Initial import of libdpf.

Co-authored-by: Cursor <cursoragent@cursor.com>

include/dpf/advice_bit_iterable.hpp

@@ -0,0 +1,426 @@
+/// @file dpf/advice_bit_iterable.hpp
+/// @brief defines `dpf::advice_bit_iterable` and associated helpers
+/// @details A `dpf::advice_bit_iterable` is a convenience class that wraps an
+///          existing iterable type to provide a new iterable over advice bits
+///          (i.e., over the least-significant bit of each element). The `begin`
+///          and `end` member functions of the `dpf::advice_bit_iterable` class
+///          each return `LegacyForwardIterator`s compatible with standard
+///          library algorithms and range-based loops.
+///
+///          In addition to `dpf::advice_bit_iterable`, this file defines the
+///          following helper functions:
+///            - `advice_bits_of`: wraps an iterable type to simplify notation
+///              for range-based loops. For example, it lets you write
+///               \code{cpp}
+///               for (auto b : advice_bits_of(my_iterable)) foo(b);
+///               \endcode
+///               instead of
+///               \code{cpp}
+///               advice_bit_iterable advice_bits{my_iterable};
+///               for (auto b : advice_bits) foo(b);
+///               \endcode
+///            - `for_each_advice_bit`: iterate through and apply a given
+///              function to each advice bit
+///            - `bit_array_from_advice_bits`: constructs a
+///              `dpf::dynamic_bit_array` that holds the advice bits of the
+///              underlying iterable.
+/// @author Ryan Henry <ryan.henry@ucalgary.ca>
+/// @author Christopher Jiang <christopher.jiang@ucalgary.ca>
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_ADVICE_BIT_ITERABLE_HPP__
+#define LIBDPF_INCLUDE_DPF_ADVICE_BIT_ITERABLE_HPP__
+
+#include "hedley/hedley.h"
+
+#include <cstddef>
+#include <cstring>
+#include <type_traits>
+#include <iterator>
+#include <memory>
+#include <algorithm>
+#include <array>
+
+#include "simde/simde/x86/avx2.h"
+#include "portable-snippets/exact-int/exact-int.h"
+
+#include "dpf/utils.hpp"
+#include "dpf/bit_array.hpp"
+
+namespace dpf
+{
+
+namespace detail
+{
+
+template <typename Iterator>
+struct extract_bit_simde_node
+{
+    bool operator()(Iterator it) const
+    {
+        auto buf = reinterpret_cast<const char *>(&*it);
+        return buf[0] & 1;
+    }
+};
+
+template <typename NodeT, typename Iterator>
+struct extract_bit;
+
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+template <typename Iterator>
+struct extract_bit<simde__m128i, Iterator>
+  : public extract_bit_simde_node<Iterator> { };
+
+template <typename Iterator>
+struct extract_bit<simde__m256i, Iterator>
+  : public extract_bit_simde_node<Iterator> { };
+HEDLEY_PRAGMA(GCC diagnostic pop)
+
+}  // namespace detail
+
+template <typename WrappedIteratorType>
+class advice_bit_iterable_const_iterator;
+
+template <typename Iterable>
+class advice_bit_iterable
+{
+  public:
+    using wrapped_iterator_type = typename Iterable::iterator_type;
+    using const_iterator
+        = advice_bit_iterable_const_iterator<wrapped_iterator_type>;
+
+    explicit advice_bit_iterable(const Iterable & iterable)
+      : begin_{std::begin(iterable)}, end_{std::end(iterable)}
+    { }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    const_iterator begin() const noexcept
+    {
+        return const_iterator(begin_);
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    const_iterator cbegin() const noexcept
+    {
+        return begin();
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    const_iterator end() const noexcept
+    {
+        return const_iterator(end_);
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    const_iterator cend() const noexcept
+    {
+        return end();
+    }
+
+  private:
+    const wrapped_iterator_type begin_, end_;
+};  // class dpf::advice_bit_iterable
+
+template <typename WrappedIteratorType>
+class advice_bit_iterable_const_iterator
+{
+  public:
+    using iterator_traits = std::iterator_traits<WrappedIteratorType>;
+    using wrapped_type = WrappedIteratorType;
+    using value_type = bool;
+    using reference = value_type;
+    using const_reference = reference;
+    using pointer = std::add_pointer_t<reference>;
+    using iterator_category = typename iterator_traits::iterator_category;
+    using size_type = std::size_t;
+    using difference_type = std::ptrdiff_t;
+    using node_type = typename iterator_traits::value_type;
+
+    HEDLEY_ALWAYS_INLINE
+    constexpr
+    explicit advice_bit_iterable_const_iterator(const wrapped_type & it) noexcept
+        : it_{it}
+    { }
+
+    HEDLEY_ALWAYS_INLINE
+    constexpr
+    advice_bit_iterable_const_iterator(advice_bit_iterable_const_iterator &&)
+        = default;
+
+    HEDLEY_ALWAYS_INLINE
+    constexpr
+    advice_bit_iterable_const_iterator(
+        const advice_bit_iterable_const_iterator &) = default;
+
+    advice_bit_iterable_const_iterator & operator=(
+        const advice_bit_iterable_const_iterator &) = default;
+
+    advice_bit_iterable_const_iterator & operator=(
+        advice_bit_iterable_const_iterator &&) = default;
+
+    ~advice_bit_iterable_const_iterator() = default;
+
+    HEDLEY_PURE
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr reference operator*() const noexcept
+    {
+        return bit(it_);
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr advice_bit_iterable_const_iterator & operator++() noexcept
+    {
+        ++it_;
+        return *this;
+    }
+
+    HEDLEY_NO_THROW
+    advice_bit_iterable_const_iterator operator++(int) noexcept
+    {
+        auto tmp = *this;
+        advice_bit_iterable_const_iterator::operator++();
+        return tmp;
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    advice_bit_iterable_const_iterator & operator--() noexcept
+    {
+        --it_;
+        return *this;
+    }
+
+    HEDLEY_NO_THROW
+    advice_bit_iterable_const_iterator operator--(int) noexcept
+    {
+        auto tmp = *this;
+        advice_bit_iterable_const_iterator::operator--();
+        return tmp;
+    }
+
+    advice_bit_iterable_const_iterator & operator+=(std::size_t n) noexcept
+    {
+        it_ += n;
+        return *this;
+    }
+
+    advice_bit_iterable_const_iterator operator+(std::size_t n) const noexcept
+    {
+        return advice_bit_iterable_const_iterator(it_ + n);
+    }
+
+    advice_bit_iterable_const_iterator & operator-=(std::size_t n) noexcept
+    {
+        it_ -= n;
+        return *this;
+    }
+
+    advice_bit_iterable_const_iterator operator-(std::size_t n) const noexcept
+    {
+        return advice_bit_iterable_const_iterator(it_ - n);
+    }
+
+    difference_type
+    operator-(advice_bit_iterable_const_iterator rhs) const noexcept
+    {
+        return it_ - rhs.it_;
+    }
+
+    reference operator[](std::size_t i) const noexcept
+    {
+        return bit(it_ + i);
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr bool
+    operator==(const advice_bit_iterable_const_iterator & rhs) const noexcept
+    {
+        return it_ == rhs.it_;
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr bool
+    operator<(const advice_bit_iterable_const_iterator & rhs) const noexcept
+    {
+        return it_ < rhs.it_;
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr bool
+    operator!=(const advice_bit_iterable_const_iterator & rhs) const noexcept
+    {
+        return !(*this == rhs);
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr bool
+    operator>(const advice_bit_iterable_const_iterator & rhs) const noexcept
+    {
+        return rhs < *this;
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr bool
+    operator<=(const advice_bit_iterable_const_iterator & rhs) const noexcept
+    {
+        return !(rhs < *this);
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr bool
+    operator>=(const advice_bit_iterable_const_iterator & rhs) const noexcept
+    {
+        return !(*this < rhs);
+    }
+
+  private:
+    wrapped_type it_;
+    static constexpr auto bit = detail::extract_bit<node_type, wrapped_type>{};
+};  // class dpf::advice_bit_iterable_const_iterator
+
+template <typename Iterable>
+dpf::advice_bit_iterable<Iterable> advice_bits_of(const Iterable & iterable)
+{
+    return advice_bit_iterable<Iterable>{iterable};
+}
+
+template <typename Iterable,
+          typename UnaryFunction>
+void for_each_advice_bit(const Iterable & iterable, UnaryFunction f)
+{
+    for (auto i : advice_bits_of(iterable)) f(i);
+}
+
+namespace detail
+{
+
+template <typename Iterator>
+auto bit_array_from_advice_bits_small(Iterator first, Iterator last,
+    std::size_t bits)
+{
+    auto ret = dynamic_bit_array(bits);
+
+    auto curbit = ret.begin();
+    for (; first != last; ++first)
+    {
+        (*curbit++).assign(*first);
+    }
+
+    return ret;
+}
+
+template <typename Iterator>
+auto bit_array_from_advice_bits_simde(Iterator first, Iterator last,
+    std::size_t bits)
+{
+    using simde_type = simde__m256i;
+    using simde_ptr = simde_type *;
+    static_assert(CHAR_BIT == 8, "CHAR_BIT not equal to 8");
+
+    auto ret = dynamic_bit_array(bits);
+    std::size_t bits_per_byte = CHAR_BIT,
+                bytes = (bits-1)/bits_per_byte + 1,
+                bits_per_word = ret.bits_per_word,
+                bits_per_simde = dpf::utils::bitlength_of_v<simde_type>,
+                bytes_per_simde = sizeof(simde_type),
+                words_per_simde = bits_per_simde / bits_per_word;
+
+    std::size_t curbits = 0, pos = 0;
+    std::array<char, 32> in = {0};
+    std::array<psnip_uint32_t, 8> out;
+    while (curbits < bits)
+    {
+        simde_type simde = {0, 0, 0, 0};
+
+        std::size_t i = 0;
+        for (; i < bits_per_byte && curbits < bits; ++i)
+        {
+            for (std::size_t j = 0; j < 32 && curbits < bits; ++j, ++curbits)
+            {
+                in[j] = *first++;
+            }
+            auto tmp = reinterpret_cast<simde_ptr>(std::data(in));
+            simde = simde_mm256_or_si256(
+                simde_mm256_slli_epi64(simde, 1),
+                simde_mm256_loadu_si256(tmp));
+        }
+
+        // algorithm expects "first bit" to be MSB in each 8-bit block at next step
+        for (std::size_t j = i; j < bits_per_byte; ++j)
+        {
+            simde = simde_mm256_slli_epi64(simde, 1);
+        }
+
+        for (std::size_t j = 0; j < i; ++j)
+        {
+            out[j] = simde_mm256_movemask_epi8(simde);
+            simde = simde_mm256_slli_epi64(simde, 1);
+        }
+
+        auto dst = reinterpret_cast<char *>(
+            std::addressof(ret.data(pos++ * words_per_simde)));
+        auto src = reinterpret_cast<char *>(std::data(out));
+        std::memcpy(dst, src, std::min(bytes_per_simde, bytes));
+        bytes -= bytes_per_simde;
+    }
+
+    return ret;
+}
+
+}  // namespace detail
+
+template <std::size_t NbitsCrossover = 1 << 4,
+          typename Iterator>
+auto
+bit_array_from_advice_bits(const advice_bit_iterable<Iterator> & advice_bits)
+{
+    auto first = std::begin(advice_bits), last = std::end(advice_bits);
+    std::size_t bits = std::distance(first, last);
+
+    if (bits < NbitsCrossover)
+    {
+        return detail::bit_array_from_advice_bits_small(first, last, bits);
+    }
+    else
+    {
+        return detail::bit_array_from_advice_bits_simde(first, last, bits);
+    }
+}
+
+}  // namespace dpf
+
+namespace std
+{
+
+template <typename Iterator>
+struct iterator_traits<dpf::advice_bit_iterable_const_iterator<Iterator>>
+{
+  private:
+    using type = dpf::advice_bit_iterable_const_iterator<Iterator>;
+  public:
+    using iterator_category = typename type::iterator_category;
+    using difference_type = typename type::difference_type;
+    using value_type = typename type::value_type;
+    using reference = typename type::reference;
+    using const_reference = typename type::const_reference;
+    using pointer = typename type::pointer;
+};
+
+}  // namespace std
+
+#endif  // LIBDPF_INCLUDE_DPF_ADVICE_BIT_ITERABLE_HPP__

include/dpf/aligned_allocator.hpp

@@ -0,0 +1,227 @@
+/// @file dpf/aligned_allocator.hpp
+/// @brief defines an allocator that aligns memory allocations to a specified alignment
+/// @details The `dpf::aligned_allocator` class template is used to allocate
+///          uninitialized memory with a specified alignment for all `libdpf++`
+///          buffers and memoizers, if no user-specified allocator is
+///          provided. It is stateless, so all instances of the allocator are
+///          interchangeable. The alignment is specified by the Alignment
+///          parameter, which must be a power of two (default:
+///          `dpf::utils::max_align_v`).
+///
+///          The allocator supports the `dpf::aligned_allocator::allocate()` function for allocating
+///          aligned, yet uninitialized memory and the `dpf::aligned_allocator::deallocate()` function
+///          for freeing the same. It also includes a convenient
+///          `dpf::aligned_allocator::allocate_unique_ptr()` function that returns a `std::unique_ptr`
+///          to the output of a call to `dpf::aligned_allocator::allocate()`.
+/// @author Ryan Henry <ryan.henry@ucalgary.ca>
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_ALIGNED_ALLOCATOR_HPP__
+#define LIBDPF_INCLUDE_DPF_ALIGNED_ALLOCATOR_HPP__
+
+#include <cstddef>
+#include <cstdlib>
+#include <type_traits>
+#include <memory>
+#include <limits>
+#include <new>
+
+#include "hedley/hedley.h"
+#include "simde/simde/x86/avx2.h"
+
+namespace dpf
+{
+
+/// @brief an allocator that allocates aligned memory
+/// @details The `dpf::aligned_allocator` class template is the default memory
+///          allocator used by all `libdpf++` buffers and memoizers, if no
+///          user-specified allocator is provided. It allocates uninitialized
+///          storage whose alignment is specified by `Alignment` and whose
+///          size is an integral multiple of `sizeof(T)`. The allocator is
+///          stateless; that is, all instances of the given allocator are
+///          interchangeable and can deallocate memory allocated by any other
+///          instance of the same allocator type.
+/// @tparam T the type to allocate
+/// @tparam Alignment specifies the alignment (default: `dpf::utils::max_align`).'
+///         The program is ill-formed if `Alignment` is not a power of 2.
+template <typename T,
+          std::size_t Alignment = alignof(T)>
+class aligned_allocator
+{
+  private:
+    /// @brief a `deleter` functor for use by `std::unique_ptr<T[]>` to free
+    ///        memory allocated when the `std::unique_ptr<T[]>` was
+    ///        constructed
+    template <typename Pointer>
+    struct deleter
+    {
+        constexpr void operator()(Pointer p) const noexcept { free(p); }
+    };
+  public:
+    using value_type = T;
+    using size_type = std::size_t;
+    using difference_type = std::ptrdiff_t;
+    using pointer = value_type *;
+    using unique_ptr = std::unique_ptr<value_type[], deleter<pointer>>;
+    using const_pointer = const value_type *;
+    using reference = value_type &;
+    using const_reference = const value_type &;
+    static constexpr size_type alignment = Alignment;
+
+    /// @brief class whose member `other` is a typedef of
+    ///        `dpf::aligned_allocator` for some type `U` with alignment `A`.
+    /// @tparam U the type to rebind to
+    /// @tparam A the alignment of the rebound allocator
+    template <typename U, size_type A = alignment> struct rebind
+    {
+        using other = aligned_allocator<U, A>;
+    };
+
+    /// @name Constructors
+    /// @brief Constructs the default allocator. Since the default allocator
+    ///        is stateless, the constructors have no visible effect.
+    /// @{
+
+    /// @brief Default constructor
+    /// @details Constructs an instance of `dpf::aligned_allocator`.
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    constexpr aligned_allocator() noexcept = default;
+
+    /// @brief Copy constructor
+    /// @details Constructs an instance of `dpf::aligned_allocator` from another
+    ///          using copy semantics.
+    /// @param other another `dpf::aligned_allocator` to construct with
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    constexpr aligned_allocator(const aligned_allocator & other) noexcept
+        = default;
+
+    /// @brief Move constructor
+    /// @details Constructs an instance of `dpf::aligned_allocator` from another
+    ///          using move semantics.
+    /// @param other another `dpf::aligned_allocator` to construct with
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    constexpr aligned_allocator(aligned_allocator && other) noexcept = default;
+
+    /// @}
+
+    /// @{
+    aligned_allocator & operator=(const aligned_allocator &) noexcept = default;
+
+    aligned_allocator & operator=(aligned_allocator &&) noexcept = default;
+    /// @}
+
+    /// @brief D'tor
+    /// @details Destroys an instance of `dpf::aligned_allocator`.
+    ~aligned_allocator() = default;
+
+    /// @brief returns the largest supported allocation size
+    /// @details Returns the maximum theoretically possible value of `num`,
+    ///          for which the call `allocate(num)` could succeed.
+    /// @note This function returns the maximum number of elements that can
+    ///       be allocated, not the maximum allocation size in bytes
+    /// @return The maximum supported allocation size.
+    constexpr size_type max_size() const noexcept
+    {
+        return std::numeric_limits<size_type>::max() / sizeof(value_type);
+    }
+
+    /// @brief allocates aligned, yet uninitialized storage
+    /// @details Allocates `num * sizeof(T)` bytes of uninitialized
+    ///          storage by invoking
+    ///          `std::aligned_alloc(alignment, num * sizeof(T))`.
+    /// @param num the number of instances of `T` to allocate storage for
+    /// @return Pointer to the first element of an array of `num` instaces
+    ///         of type `T` whose elements have not been constructed yet.
+    /// @throws std::bad_array_new_length if `max_size() < num`
+    /// @throws std::bad_alloc if allocation fails.
+    HEDLEY_WARN_UNUSED_RESULT
+    HEDLEY_MALLOC
+    HEDLEY_RETURNS_NON_NULL
+    constexpr
+    pointer allocate(size_type num, const void * /*hint*/ = nullptr) const
+    {
+        if (max_size() < num)
+        {
+            throw std::bad_array_new_length();
+        }
+
+        // C11 `aligned_alloc` requires the size to be a multiple of the
+        // alignment. Round up so odd element counts (or odd sizeof(T))
+        // do not pass a non-conforming size and corrupt the heap.
+        const size_type bytes = num * sizeof(T);
+        const size_type aligned_bytes =
+            (bytes + (alignment - 1)) & ~(alignment - 1);
+        void * ptr = std::aligned_alloc(alignment, aligned_bytes);
+
+        if (ptr == nullptr)
+        {
+            throw std::bad_alloc();
+        }
+
+        return assume_aligned(static_cast<pointer>(ptr));
+    }
+
+    /// @brief allocates and constructs a `std::unqiue_ptr<T[]>` to aligned, yet
+    ///        uninitialized storage
+    /// @details Allocates `num * sizeof(T)` bytes of uninitialized
+    ///          storage by invoking `allocate(size_type, const void *)` and
+    ///          returns a `std::unique_ptr<T[]>` that owns it.
+    /// @param num the number of instances of `T` to allocate storage for
+    /// @return An `std::unique_ptr<T[]>` owning the pointer to the first
+    ///         element of an array of `num` instaces of type `T` whose elements
+    ///         have not been constructed yet.
+    /// @throws std::bad_array_new_length if `max_size() < num`
+    /// @throws std::bad_alloc if allocation fails.
+    HEDLEY_ALWAYS_INLINE
+    constexpr auto allocate_unique_ptr(size_type num) const
+    {
+        return unique_ptr{allocate(num)};
+    }
+
+    /// @brief deallocates storage
+    /// @details Deallocates the storage referenced by the pointer `p`,
+    ///          which must be a pointer obtained by an earlier call to
+    ///          `allocate()`.
+    /// @param p pointer obtained from allocate()
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    constexpr void deallocate(pointer p, size_type /*num*/ = 0) const noexcept
+    {
+        free(p);
+    }
+
+    /// @brief informs the compiler that a pointer is aligned
+    /// @details Informs the implementation that the object ptr points to is
+    ///        aligned to at least `alignment`. The implementation may use
+    ///        this information to generate more efficient code, but it might
+    ///        only make this assumption if the object is accessed via the
+    ///        return value of `assume_aligned`.
+    ///
+    ///        The behavior is undefined if `ptr` does not point to an object
+    ///        of type `T` (ignoring cv-qualification at every level), or if the
+    ///        object's alignment is not at least `Alignment`.
+    /// @note It is up to the program to ensure that the alignment assumption
+    ///       actually holds. A call to `assume_aligned` does not cause the
+    ///       compiler to verify or enforce this.
+    /// @param ptr the pointer
+    /// @return `ptr`
+    HEDLEY_WARN_UNUSED_RESULT
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_CONST
+    HEDLEY_NO_THROW
+    HEDLEY_NON_NULL(1)
+    constexpr static auto assume_aligned(pointer ptr) noexcept
+    {
+        return static_cast<pointer>(
+            __builtin_assume_aligned(ptr, alignment));
+    }
+};
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_ALIGNED_ALLOCATOR_HPP__

include/dpf/bit.hpp

@@ -0,0 +1,266 @@
+/// @file dpf/bit.hpp
+/// @brief defines `dpf::bit` and associated helpers
+/// @details A `dpf::bit` is a binary type whose representation can be packed
+///          into one bit. It is implemented as an `enum` with two values:
+///          `zero` and `one`. This type is intended for us as an [output type](@ref output_types)
+///          for a DPF, in which case leaf nodes will be packed in much
+///          the ways as in an `std::bitset` or `std::vector<bool>`.
+///
+///          In addition to `dpf::bit`, this file defines three overloaded
+///          variants of a `dpf::to_bit` function that respectively convert
+///          a `bool, a `char`, or (the least significant bit of) an `int` to
+///          a `dpf::bit`. Likewise, it defines `dpf::to_string` to convert
+///          a `dpf::bit` into an `std::string`. Finally, it overloads stream
+///          input and output operators (`<<` and `>>`) for `dpf::bit`.
+/// @author Ryan Henry <ryan.henry@ucalgary.ca>
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_BIT_HPP__
+#define LIBDPF_INCLUDE_DPF_BIT_HPP__
+
+#include <cstddef>
+#include <type_traits>
+#include <limits>
+#include <stdexcept>
+#include <string>
+#include <memory>
+#include <ostream>
+#include <istream>
+
+#include "hedley/hedley.h"
+
+#include "dpf/utils.hpp"
+
+/// @brief the dpf namespace
+namespace dpf
+{
+
+/// @brief binary type whose representation can be packed into one bit
+enum bit : bool
+{
+    zero = false,  ///< `0`, `false`, "unset", "off"
+    one = true     ///< `1`, `true`, "set", "on"
+};
+
+/// @brief converts a value to a `dpf::bit`
+/// @{
+
+/// @brief converts (the lsb of) an `int` to a `dpf::bit`
+/// @details Convert an `int` to a `dpf::bit`. The resulting `dpf::bit` is
+///          equal to `dpf::bit::one` if the *least-significant bit* of
+///          `value` is `1` and `dpf::bit::zero` otherwise.
+/// @param value the `int` to convert
+/// @returns `static_cast<dpf::bit>(value & 1)`
+HEDLEY_CONST
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+static constexpr dpf::bit to_bit(int value) noexcept
+{
+    return static_cast<dpf::bit>(value & 1);
+}
+
+/// @brief converts the least-significant bit of an integer literal to a `dpf::bit`
+/// @details This overload exists so `operator""_bit` does not select the
+///          character converter, which is an exact match for `unsigned long long`.
+HEDLEY_CONST
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+static constexpr dpf::bit to_bit(unsigned long long value) noexcept
+{
+    return static_cast<dpf::bit>(value & 1ull);
+}
+
+/// @brief converts a `bool` to a `dpf::bit`
+/// @details Convert a `bool` to a `dpf::bit`. The resulting `dpf::bit` is
+///          equal to `dpf::bit::one` if `value==true` and `dpf::bit::zero`
+///          otherwise.
+/// @param value the `bool` to convert
+/// @returns `static_cast<dpf::bit>(value)`
+HEDLEY_CONST
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+static constexpr dpf::bit to_bit(bool value) noexcept
+{
+    return static_cast<dpf::bit>(value);
+}
+
+/// @brief converts a character to a `dpf::bit`
+/// @details Convert a character to a `dpf::bit`. The resulting `dpf::bit` is
+///          equal to `dpf::bit::one` if `value==one` and `dpf::bit::zero`
+///          otherwise.
+/// @param value the character to convert
+/// @param zero character used to represent `0` (default: ``CharT('0')``)
+/// @param one character used to represent `1` (default: ``CharT('1')``)
+/// @returns `static_cast<dpf::bit>(0)` if `value==0` or
+///          `static_cast<dpf::bit>(1)` if `value==1`
+/// @throws std::domain_error if `value != zero && value != one`
+template <typename CharT,
+          typename Traits = std::char_traits<CharT>>
+HEDLEY_ALWAYS_INLINE
+static constexpr dpf::bit to_bit(
+    CharT value,
+    CharT zero = CharT('0'),
+    CharT one = CharT('1'))
+{
+    if (!Traits::eq(value, zero) && !Traits::eq(value, one))
+    {
+        throw std::domain_error("Unrecognized character");
+    }
+    return Traits::eq(value, zero) ? dpf::bit::zero : dpf::bit::one;
+}
+
+/// @}
+
+/// @brief converts a `dpf::bit` to a `std::basic_string`
+/// @details Converts the contents of a `dpf::bit` to a `std::string` for
+///          human-friendly printing. Uses `zero` to represent the value
+///          `0` and `one` to the value `1`.
+/// @param value the `dpf::bit` to convert
+/// @param zero character to use to represent `false`/`0` (default: ``CharT('0')``)
+/// @param one character to use to represent `true`/`1` (default: ``CharT('1')``)
+/// @return `(value == 0) ? zero : one`
+template <typename CharT = char,
+          typename Traits = std::char_traits<CharT>,
+          typename Allocator = std::allocator<CharT>>
+static std::basic_string<CharT, Traits, Allocator> to_string(
+    dpf::bit value,
+    CharT zero = CharT('0'),
+    CharT one = CharT('1'))
+{
+    auto ch = (value == dpf::bit::zero) ? zero : one;
+    return std::basic_string<CharT>(1, ch, Allocator{});
+}
+
+/// @brief performs stream input and output on `dpf::bit`s
+/// @{
+
+/// @brief performs stream output on a `dpf::bit`
+/// @details Writes a `dpf::bit` to the character stream `os` as if by first
+///          converting it to a `std::basic_string<CharT, Traits>` using
+///          `dpf::to_string()`, and then writing it into `os` using `operator<<`
+///          (which is a `FormattedOutputFunction` for strings). The
+///          characters to use for zero and one are obtained from the
+///          currently-imbued locale by calling `os.widen()` with `0` and `1`
+///          as the arguments.
+/// @param os a character output stream
+/// @param value the `dpf::bit` to insert into the output stream
+/// @return `os`
+template <typename CharT,
+          typename Traits>
+std::basic_ostream<CharT, Traits> &
+operator<<(std::basic_ostream<CharT, Traits> & os, const dpf::bit & value)
+{
+    return os << to_string<CharT, Traits>(value, os.widen('0'),
+        os.widen('1'));
+}
+
+/// @brief performs stream input on a `dpf::bit`
+/// @details Extracts one character from `is` and attempts to convert it to
+///          a `dpf::bit` using `dpf::to_bit()`. If successful, the result is
+///          stored in `value`. The characters to use for zero and one are
+///          obtained from the currently-imbued locale by calling `is.widen()`
+///          with `0` and `1` as the arguments.
+/// @param is a character input stream
+/// @param value the `dpf::bit` to extract from the input stream
+/// @return `is`
+template <typename CharT,
+          typename Traits>
+std::basic_istream<CharT, Traits> &
+operator>>(std::basic_istream<CharT, Traits> & is, dpf::bit & value)
+{
+    try
+    {
+        value = to_bit<CharT>(is.get(), is.widen('0'), is.widen('1'));
+    }
+    catch(const std::exception & e)
+    {
+        is.setstate(std::ios::failbit);
+    }
+    return is;
+}
+
+/// @}
+
+inline constexpr dpf::bit operator+(dpf::bit lhs, dpf::bit rhs) noexcept
+{
+    return static_cast<dpf::bit>(static_cast<bool>(lhs) ^ static_cast<bool>(rhs));
+}
+
+/// @brief GF(2) subtraction. Identical to `operator+`.
+inline constexpr dpf::bit operator-(dpf::bit lhs, dpf::bit rhs) noexcept
+{
+    return lhs + rhs;
+}
+
+namespace utils
+{
+
+/// @brief specializes `dpf::utils::bitlength_of` for `dpf::bit`
+template <>
+struct bitlength_of<dpf::bit>
+  : public std::integral_constant<std::size_t, 1> { };
+
+template <typename NodeT>
+struct bitlength_of_output<dpf::bit, NodeT>
+  : public std::integral_constant<std::size_t, 1> { };
+
+template <>
+struct is_packed_subbyte<dpf::bit> : std::true_type {};
+
+template <>
+struct packed_lane_bits<dpf::bit>
+  : public std::integral_constant<std::size_t, 1> {};
+
+template <>
+struct make_from_integral_value<dpf::bit>
+{
+    constexpr dpf::bit operator()(bool val) const noexcept
+    {
+        return val ? dpf::bit::one : dpf::bit::zero;
+    }
+};
+
+}  // namespace utils
+
+namespace literals
+{
+
+namespace bit
+{
+
+constexpr static auto operator "" _bit(unsigned long long int x) { return dpf::to_bit(x); }
+
+}  // namespace bit
+
+}  // namespace literals
+
+}  // namespace dpf
+
+namespace std
+{
+
+/// @{
+
+/// @brief specializes `std::numeric_limits` for `dpf::bit`
+template<> class numeric_limits<dpf::bit>
+    : public numeric_limits<bool> { };
+
+/// @brief specializes `std::numeric_limits` for `dpf::bit const`
+template<> class numeric_limits<dpf::bit const>
+    : public numeric_limits<dpf::bit> {};
+
+/// @brief specializes `std::numeric_limits` for `dpf::bit volatile`
+template<> class numeric_limits<dpf::bit volatile>
+    : public numeric_limits<dpf::bit> {};
+
+/// @brief specializes `std::numeric_limits` for `dpf::bit const volatile`
+template<> class numeric_limits<dpf::bit const volatile>
+    : public numeric_limits<dpf::bit> {};
+
+/// @}
+
+}  // namespace std
+
+#endif  // LIBDPF_INCLUDE_DPF_BIT_HPP__

include/dpf/buffered_prg.hpp

@@ -0,0 +1,349 @@
+/// @file dpf/buffered_prg.hpp
+/// @brief Buffered, seekable lanes over a libdpf PRG.
+/// @details The default PRG is `dpf::prg::aes128`. Any PRG with
+///          `block_type`, `eval(seed, pos)`, and `eval(seed, out, count, pos)`
+///          can be substituted. A master block is the recorded seed. Role `r`
+///          owns two streams: values are `PRG::eval(master, r)` and share
+///          masks are `PRG::eval(tweaked_master, r)`. Element `index` is a
+///          contiguous run of blocks on that stream, so a forward scan is one
+///          multi-block `eval`.
+/// @copyright Copyright (c) 2019-2026 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_BUFFERED_PRG_HPP__
+#define LIBDPF_INCLUDE_DPF_BUFFERED_PRG_HPP__
+
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <map>
+#include <stdexcept>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+#include "hedley/hedley.h"
+
+#include "dpf/aligned_allocator.hpp"
+#include "dpf/prg.hpp"
+#include "dpf/random.hpp"
+
+namespace dpf
+{
+namespace randomness
+{
+
+namespace detail
+{
+
+template <typename PRG>
+typename PRG::block_type mask_master(typename PRG::block_type master) noexcept
+{
+    unsigned char raw[sizeof(master)];
+    std::memcpy(raw, &master, sizeof(master));
+    raw[sizeof(master) - 1] ^= 0x01u;
+    typename PRG::block_type out{};
+    std::memcpy(&out, raw, sizeof(out));
+    return out;
+}
+
+template <typename PRG, typename T>
+struct lane_codec
+{
+    static_assert(std::is_trivially_copyable_v<T>,
+        "prg lanes require a trivially copyable value type");
+
+    using block_type = typename PRG::block_type;
+
+    static T at(block_type seed, std::uint64_t index)
+    {
+        T out{};
+        fill(seed, index, &out, 1u);
+        return out;
+    }
+
+    /// Element `index` is the packed byte range `[index * sizeof(T), ...)`.
+    static void fill(block_type seed, std::uint64_t index, T * out, std::size_t count)
+    {
+        if (count == 0)
+            return;
+        constexpr std::uint64_t block_bytes = sizeof(block_type);
+        std::uint64_t byte_off = index * static_cast<std::uint64_t>(sizeof(T));
+        std::uint64_t nbytes = static_cast<std::uint64_t>(count) * sizeof(T);
+        std::uint64_t start = byte_off / block_bytes;
+        std::uint64_t end = byte_off + nbytes;
+        std::uint64_t nblocks = (end + block_bytes - 1u) / block_bytes - start;
+        if (start > static_cast<std::uint64_t>(UINT32_MAX)
+            || nblocks > static_cast<std::uint64_t>(UINT32_MAX)
+            || start > static_cast<std::uint64_t>(UINT32_MAX) - nblocks)
+        {
+            throw std::invalid_argument("prg lane index is out of range");
+        }
+        HEDLEY_PRAGMA(GCC diagnostic push)
+        HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+        dpf::aligned_allocator<block_type> alloc;
+        auto blocks = alloc.allocate_unique_ptr(static_cast<std::size_t>(nblocks));
+        HEDLEY_PRAGMA(GCC diagnostic pop)
+        PRG::eval(seed, blocks.get(), static_cast<psnip_uint32_t>(nblocks),
+            static_cast<psnip_uint32_t>(start));
+        auto * bytes = reinterpret_cast<const unsigned char *>(blocks.get());
+        std::memcpy(out, bytes + static_cast<std::size_t>(byte_off % block_bytes),
+            static_cast<std::size_t>(nbytes));
+    }
+};
+
+template <typename PRG, typename T>
+struct buffered_slot
+{
+    using block_type = typename PRG::block_type;
+
+    explicit buffered_slot(block_type seed, std::size_t buffer_elems)
+      : seed_(seed),
+        buffer_(buffer_elems),
+        absolute_pos_(0u),
+        filled_(0u),
+        next_(0u)
+    {
+        refill(0u);
+    }
+
+    T get()
+    {
+        if (next_ >= filled_)
+            refill(absolute_pos_);
+        T v = buffer_[next_];
+        ++next_;
+        ++absolute_pos_;
+        return v;
+    }
+
+    void fill(T * out, std::size_t count)
+    {
+        std::size_t written = 0u;
+        while (written < count)
+        {
+            if (next_ >= filled_)
+                refill(absolute_pos_);
+            std::size_t available = filled_ - next_;
+            std::size_t take = available < count - written ? available : count - written;
+            std::memcpy(out + written, buffer_.data() + next_, take * sizeof(T));
+            written += take;
+            next_ += take;
+            absolute_pos_ += static_cast<std::uint64_t>(take);
+        }
+    }
+
+    T at(std::uint64_t index) const
+    {
+        return lane_codec<PRG, T>::at(seed_, index);
+    }
+
+    std::uint64_t sampled() const noexcept { return absolute_pos_; }
+
+private:
+    void refill(std::uint64_t at_elem)
+    {
+        next_ = 0u;
+        filled_ = buffer_.size();
+        lane_codec<PRG, T>::fill(seed_, at_elem, buffer_.data(), buffer_.size());
+    }
+
+    block_type seed_;
+    std::vector<T> buffer_;
+    std::uint64_t absolute_pos_;
+    std::size_t filled_;
+    std::size_t next_;
+};
+
+}  // namespace detail
+
+template <typename PRG = dpf::prg::aes128>
+typename PRG::block_type sample_master_seed()
+{
+    return dpf::uniform_sample<typename PRG::block_type>();
+}
+
+/// Fixed lanes. Lane `I` is `PRG::eval(master, I)`.
+template <typename PRG, typename... Ts>
+class buffered_prg
+{
+public:
+    using prg_type = PRG;
+    using seed_type = typename PRG::block_type;
+    static constexpr std::size_t stream_count = sizeof...(Ts);
+
+    explicit buffered_prg(std::size_t per_stream_buffer_elems = 1024u)
+      : seed_(sample_master_seed<PRG>()),
+        buffers_(make_buffers(per_stream_buffer_elems))
+    { }
+
+    explicit buffered_prg(seed_type seed, std::size_t per_stream_buffer_elems = 1024u)
+      : seed_(seed),
+        buffers_(make_buffers(per_stream_buffer_elems))
+    { }
+
+    const seed_type & seed() const noexcept { return seed_; }
+
+    template <std::size_t I>
+    auto get()
+    {
+        static_assert(I < stream_count, "stream index out of range");
+        return std::get<I>(buffers_).get();
+    }
+
+    template <std::size_t I>
+    void fill(std::tuple_element_t<I, std::tuple<Ts...>> * out, std::size_t count)
+    {
+        static_assert(I < stream_count, "stream index out of range");
+        std::get<I>(buffers_).fill(out, count);
+    }
+
+    template <std::size_t I>
+    auto at(std::uint64_t index) const
+    {
+        static_assert(I < stream_count, "stream index out of range");
+        return std::get<I>(buffers_).at(index);
+    }
+
+    template <std::size_t I>
+    std::uint64_t sampled() const noexcept
+    {
+        static_assert(I < stream_count, "stream index out of range");
+        return std::get<I>(buffers_).sampled();
+    }
+
+private:
+    template <std::size_t... I>
+    std::tuple<detail::buffered_slot<PRG, Ts>...>
+    make_buffers_impl(std::size_t n, std::index_sequence<I...>) const
+    {
+        return std::tuple<detail::buffered_slot<PRG, Ts>...>(
+            detail::buffered_slot<PRG, Ts>(
+                PRG::eval(seed_, static_cast<psnip_uint32_t>(I)), n)...);
+    }
+
+    std::tuple<detail::buffered_slot<PRG, Ts>...> make_buffers(std::size_t n) const
+    {
+        if (n == 0)
+            throw std::invalid_argument("per_stream_buffer_elems must be positive");
+        return make_buffers_impl(n, std::make_index_sequence<stream_count>{});
+    }
+
+    seed_type seed_;
+    std::tuple<detail::buffered_slot<PRG, Ts>...> buffers_;
+};
+
+template <typename... Ts>
+using aes_buffered_prg = buffered_prg<dpf::prg::aes128, Ts...>;
+
+/// Dynamic lanes of one value type. `value_at(role, index)` and
+/// `mask_at(role, index)` are independent of call order. A window cache
+/// refills from the requested index.
+template <typename T, typename PRG = dpf::prg::aes128>
+class lane_table
+{
+public:
+    using prg_type = PRG;
+    using seed_type = typename PRG::block_type;
+    using value_type = T;
+
+    explicit lane_table(std::size_t window = 256u)
+      : lane_table(sample_master_seed<PRG>(), window)
+    { }
+
+    explicit lane_table(seed_type seed, std::size_t window = 256u)
+      : seed_(seed),
+        mask_seed_(detail::mask_master<PRG>(seed)),
+        window_(window)
+    {
+        if (window_ == 0)
+            throw std::invalid_argument("prg lane window must be positive");
+    }
+
+    lane_table(const lane_table &) = delete;
+    lane_table & operator=(const lane_table &) = delete;
+    lane_table(lane_table &&) = default;
+    lane_table & operator=(lane_table &&) = default;
+
+    const seed_type & seed() const noexcept { return seed_; }
+
+    T value_at(std::uint32_t role, std::uint64_t index) const
+    {
+        return cached(role, true, index);
+    }
+
+    T mask_at(std::uint32_t role, std::uint64_t index) const
+    {
+        return cached(role, false, index);
+    }
+
+    void fill_values(std::uint32_t role, std::uint64_t index, T * out, std::size_t n) const
+    {
+        if (n == 0)
+            return;
+        detail::lane_codec<PRG, T>::fill(slot(role).value_seed, index, out, n);
+    }
+
+    void fill_masks(std::uint32_t role, std::uint64_t index, T * out, std::size_t n) const
+    {
+        if (n == 0)
+            return;
+        detail::lane_codec<PRG, T>::fill(slot(role).mask_seed, index, out, n);
+    }
+
+private:
+    using block_type = typename PRG::block_type;
+
+    struct slot_pair
+    {
+        block_type value_seed{};
+        block_type mask_seed{};
+        std::vector<T> value_cache;
+        std::vector<T> mask_cache;
+        std::uint64_t value_base = 0;
+        std::uint64_t mask_base = 0;
+        bool value_hot = false;
+        bool mask_hot = false;
+    };
+
+    slot_pair & slot(std::uint32_t role) const
+    {
+        auto it = slots_.find(role);
+        if (it != slots_.end())
+            return it->second;
+        slot_pair created;
+        created.value_seed = PRG::eval(seed_, role);
+        created.mask_seed = PRG::eval(mask_seed_, role);
+        created.value_cache.resize(window_);
+        created.mask_cache.resize(window_);
+        auto inserted = slots_.emplace(role, std::move(created));
+        return inserted.first->second;
+    }
+
+    T cached(std::uint32_t role, bool values, std::uint64_t index) const
+    {
+        slot_pair & s = slot(role);
+        std::vector<T> & buf = values ? s.value_cache : s.mask_cache;
+        std::uint64_t & base = values ? s.value_base : s.mask_base;
+        bool & hot = values ? s.value_hot : s.mask_hot;
+        block_type stream = values ? s.value_seed : s.mask_seed;
+        if (hot && index >= base && index < base + buf.size())
+            return buf[static_cast<std::size_t>(index - base)];
+        base = index;
+        hot = true;
+        detail::lane_codec<PRG, T>::fill(stream, index, buf.data(), buf.size());
+        return buf[0];
+    }
+
+    seed_type seed_;
+    seed_type mask_seed_;
+    std::size_t window_;
+    mutable std::map<std::uint32_t, slot_pair> slots_;
+};
+
+}  // namespace randomness
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_BUFFERED_PRG_HPP__

include/dpf/dcf.hpp

@@ -0,0 +1,352 @@
+/// @file dpf/dcf.hpp
+/// @brief Comparison-channel specs and GGM path-sum helpers for libdpf.
+/// @details `lt`/`leq`/`gt`/`geq` (+ `_at`) take `(if_true, if_false=0)`.
+///          Eval walks the same GGM tree as the DPF (per-level value CWs).
+///          `eq` / `eq_at` are synonyms for ordinary point placements.
+/// @copyright Copyright (c) 2019-2026 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license.
+
+#ifndef LIBDPF_INCLUDE_DPF_DCF_HPP__
+#define LIBDPF_INCLUDE_DPF_DCF_HPP__
+
+#include <cstddef>
+#include <cstdint>
+#include <type_traits>
+#include <utility>
+#include <limits>
+
+#include "hedley/hedley.h"
+#include "simde/simde/x86/avx2.h"
+
+#include "dpf/utils.hpp"
+#include "dpf/bit.hpp"
+#include "dpf/xor_wrapper.hpp"
+#include "dpf/twiddle.hpp"
+
+namespace dpf
+{
+
+/// Comparison kind for the optional DCF channel on a key.
+enum class cmp_kind : uint8_t
+{
+    lt = 0,
+    leq = 1,
+    gt = 2,
+    geq = 3
+};
+
+enum class cmp_trivial : uint8_t
+{
+    none = 0,
+    always_true = 1,
+    always_false = 2
+};
+
+namespace detail
+{
+namespace dcf_impl
+{
+
+template <typename Beta>
+Beta default_false() noexcept
+{
+    if constexpr (std::is_same_v<Beta, dpf::bit>)
+        return dpf::bit::zero;
+    else
+        return Beta{};
+}
+
+template <typename Beta>
+uint64_t beta_delta_u64(const Beta & if_true, const Beta & if_false,
+    uint64_t mask) noexcept
+{
+    if constexpr (std::is_same_v<Beta, dpf::bit>)
+    {
+        const uint64_t t = static_cast<bool>(if_true) ? 1ULL : 0ULL;
+        const uint64_t f = static_cast<bool>(if_false) ? 1ULL : 0ULL;
+        return (t ^ f) & mask;
+    }
+    else if constexpr (dpf::utils::is_xor_wrapper_v<Beta>)
+    {
+        return (static_cast<uint64_t>(if_true) ^ static_cast<uint64_t>(if_false))
+            & mask;
+    }
+    else
+    {
+        return (static_cast<uint64_t>(if_true)
+            - static_cast<uint64_t>(if_false)) & mask;
+    }
+}
+
+template <typename Beta>
+uint64_t beta_to_u64_simple(const Beta & beta, uint64_t mask) noexcept
+{
+    if constexpr (std::is_same_v<Beta, dpf::bit>)
+        return (static_cast<bool>(beta) ? 1ULL : 0ULL) & mask;
+    else
+        return static_cast<uint64_t>(beta) & mask;
+}
+
+template <typename Beta>
+Beta sub_beta(const Beta & a, const Beta & b) noexcept
+{
+    if constexpr (std::is_same_v<Beta, dpf::bit>)
+        return dpf::bit{static_cast<bool>(a) ^ static_cast<bool>(b)};
+    else if constexpr (dpf::utils::is_xor_wrapper_v<Beta>)
+        return Beta{static_cast<uint64_t>(a) ^ static_cast<uint64_t>(b)};
+    else
+        return static_cast<Beta>(a - b);
+}
+
+template <typename Beta>
+Beta u64_to_beta(uint64_t v) noexcept
+{
+    if constexpr (std::is_same_v<Beta, dpf::bit>)
+        return dpf::bit{static_cast<bool>(v & 1u)};
+    else
+        return static_cast<Beta>(v);
+}
+
+inline uint64_t default_mask_for_bits(std::size_t out_bits) noexcept
+{
+    if (out_bits >= 64)
+        return ~0ULL;
+    if (out_bits == 0)
+        return 0ULL;
+    return (1ULL << out_bits) - 1ULL;
+}
+
+HEDLEY_ALWAYS_INLINE
+uint64_t neg_m(uint64_t x, uint64_t mask) noexcept
+{
+    return (0ULL - x) & mask;
+}
+
+HEDLEY_ALWAYS_INLINE
+uint64_t sgn_m(uint8_t t1, uint64_t x, uint64_t mask) noexcept
+{
+    return t1 ? neg_m(x, mask) : x;
+}
+
+/// Convert a GGM node to a group element (low 64 bits, control bits cleared).
+HEDLEY_ALWAYS_INLINE
+uint64_t convert_node(simde__m128i n, uint64_t mask) noexcept
+{
+    return static_cast<uint64_t>(
+        simde_mm_cvtsi128_si64(dpf::unset_lo_2bits(n))) & mask;
+}
+
+/// Draw the group-width blind `r` used to split the `cmp_addend` share.
+/// `sample` yields one interior block; only `popcount(mask)` live bits are
+/// kept, so the blind (and thus the addend share) never needs a full padded
+/// `uint64_t` on the wire. Dealer and Doerner–Shelat gen call this with the
+/// same block source so their keys stay byte-identical (matched tapes).
+template <typename BlockSampler>
+HEDLEY_ALWAYS_INLINE
+uint64_t sample_addend_blind(uint64_t mask, BlockSampler && sample) noexcept
+{
+    return convert_node(dpf::unset_lo_2bits(sample()), mask);
+}
+
+/// One level of value CW on GGM children. Updates running `Va`.
+/// `ai` is the keep-path bit of the (effective) threshold.
+inline uint64_t make_value_cw(simde__m128i c0L, simde__m128i c0R,
+    simde__m128i c1L, simde__m128i c1R, uint8_t t0, uint8_t t1, int ai,
+    uint64_t & Va, uint64_t beta, uint64_t mask) noexcept
+{
+    (void)t0;
+    uint64_t v0K, v1K, v0Lo, v1Lo;
+    if (ai == 0)
+    {
+        v0K = convert_node(c0L, mask);
+        v1K = convert_node(c1L, mask);
+        v0Lo = convert_node(c0R, mask);
+        v1Lo = convert_node(c1R, mask);
+    }
+    else
+    {
+        v0K = convert_node(c0R, mask);
+        v1K = convert_node(c1R, mask);
+        v0Lo = convert_node(c0L, mask);
+        v1Lo = convert_node(c1L, mask);
+    }
+    uint64_t vcw = sgn_m(t1,
+        (v1Lo + neg_m(v0Lo, mask) + neg_m(Va, mask)) & mask, mask);
+    // Lose-left (ai==1) is the x<α diverge: plant β there.
+    if (ai == 1)
+        vcw = (vcw + sgn_m(t1, beta, mask)) & mask;
+    Va = (Va + neg_m(v1K, mask) + v0K + sgn_m(t1, vcw, mask)) & mask;
+    return vcw;
+}
+
+/// Final leaf value CW. `on_path` is the payload reconstructed when the query
+/// stays on α's path through all levels (0 for strict lt/geq; β for leq/gt).
+inline uint64_t make_final_cw(simde__m128i s0, simde__m128i s1, uint8_t t1,
+    uint64_t Va, uint64_t mask, uint64_t on_path = 0) noexcept
+{
+    uint64_t c0 = convert_node(s0, mask);
+    uint64_t c1 = convert_node(s1, mask);
+    return sgn_m(t1,
+        (c1 + neg_m(c0, mask) + neg_m(Va, mask) + on_path) & mask, mask);
+}
+
+}  // namespace dcf_impl
+
+/// Comparison metadata on an incremental key (value CWs live on the key).
+/// Payload δ = if_true − if_false is dealer-known and baked into `value_cw` /
+/// `cw_last` only — never stored clear on the key (traditional DPF hiding).
+/// The second output value (`if_false`) is held as a per-party additive share
+/// on the key (`cmp_addend`), not as a public constant.
+struct cmp_meta
+{
+    int nbits = 0;                 // comparison prefix length
+    uint64_t mask = 0;
+    cmp_kind kind = cmp_kind::lt;
+    cmp_trivial trivial = cmp_trivial::none;
+    bool eval_as_ge = false;       // invert path-sum (geq / gt)
+    bool include_eq = false;       // plant δ on the α-path leaf (leq / gt)
+    bool active = false;
+
+    bool empty() const noexcept { return !active; }
+};
+
+/// Backward-compatible alias while call sites migrate.
+using cmp_channel = cmp_meta;
+
+}  // namespace detail
+
+// ---------------------------------------------------------------------------
+// Comparison specs: lt/leq/gt/geq (+ _at)
+// ---------------------------------------------------------------------------
+
+template <cmp_kind Kind, typename Beta>
+struct cmp_pack
+{
+    static constexpr bool is_cmp = true;
+    static constexpr cmp_kind kind = Kind;
+    static constexpr std::size_t prefix = 0;
+    using beta_type = Beta;
+    Beta if_true;
+    Beta if_false;
+
+    explicit cmp_pack(Beta t, Beta f = detail::dcf_impl::default_false<Beta>())
+      : if_true{std::move(t)}, if_false{std::move(f)} { }
+};
+
+template <std::size_t N, cmp_kind Kind, typename Beta>
+struct cmp_at_pack
+{
+    static constexpr bool is_cmp = true;
+    static constexpr cmp_kind kind = Kind;
+    static constexpr std::size_t prefix = N;
+    using beta_type = Beta;
+    Beta if_true;
+    Beta if_false;
+
+    explicit cmp_at_pack(Beta t, Beta f = detail::dcf_impl::default_false<Beta>())
+      : if_true{std::move(t)}, if_false{std::move(f)} { }
+};
+
+template <typename Beta>
+inline auto lt(Beta t, Beta f = detail::dcf_impl::default_false<std::decay_t<Beta>>())
+{
+    return cmp_pack<cmp_kind::lt, std::decay_t<Beta>>(std::move(t), std::move(f));
+}
+template <typename Beta>
+inline auto leq(Beta t, Beta f = detail::dcf_impl::default_false<std::decay_t<Beta>>())
+{
+    return cmp_pack<cmp_kind::leq, std::decay_t<Beta>>(std::move(t), std::move(f));
+}
+template <typename Beta>
+inline auto gt(Beta t, Beta f = detail::dcf_impl::default_false<std::decay_t<Beta>>())
+{
+    return cmp_pack<cmp_kind::gt, std::decay_t<Beta>>(std::move(t), std::move(f));
+}
+template <typename Beta>
+inline auto geq(Beta t, Beta f = detail::dcf_impl::default_false<std::decay_t<Beta>>())
+{
+    return cmp_pack<cmp_kind::geq, std::decay_t<Beta>>(std::move(t), std::move(f));
+}
+
+template <std::size_t N, typename Beta>
+inline auto lt_at(Beta t, Beta f = detail::dcf_impl::default_false<std::decay_t<Beta>>())
+{
+    return cmp_at_pack<N, cmp_kind::lt, std::decay_t<Beta>>(std::move(t), std::move(f));
+}
+template <std::size_t N, typename Beta>
+inline auto leq_at(Beta t, Beta f = detail::dcf_impl::default_false<std::decay_t<Beta>>())
+{
+    return cmp_at_pack<N, cmp_kind::leq, std::decay_t<Beta>>(std::move(t), std::move(f));
+}
+template <std::size_t N, typename Beta>
+inline auto gt_at(Beta t, Beta f = detail::dcf_impl::default_false<std::decay_t<Beta>>())
+{
+    return cmp_at_pack<N, cmp_kind::gt, std::decay_t<Beta>>(std::move(t), std::move(f));
+}
+template <std::size_t N, typename Beta>
+inline auto geq_at(Beta t, Beta f = detail::dcf_impl::default_false<std::decay_t<Beta>>())
+{
+    return cmp_at_pack<N, cmp_kind::geq, std::decay_t<Beta>>(std::move(t), std::move(f));
+}
+
+// ---------------------------------------------------------------------------
+// Equality specs: eq / eq_at
+// ---------------------------------------------------------------------------
+
+template <typename Beta>
+struct eq_pack
+{
+    static constexpr bool is_eq = true;
+    static constexpr std::size_t prefix = 0;
+    using beta_type = Beta;
+    Beta if_true;
+    Beta if_false;
+
+    explicit eq_pack(Beta t, Beta f = detail::dcf_impl::default_false<Beta>())
+      : if_true{std::move(t)}, if_false{std::move(f)} { }
+};
+
+template <std::size_t N, typename Beta>
+struct eq_at_pack
+{
+    static constexpr bool is_eq = true;
+    static constexpr std::size_t prefix = N;
+    using beta_type = Beta;
+    Beta if_true;
+    Beta if_false;
+
+    explicit eq_at_pack(Beta t, Beta f = detail::dcf_impl::default_false<Beta>())
+      : if_true{std::move(t)}, if_false{std::move(f)} { }
+};
+
+template <typename Beta>
+inline auto eq(Beta t, Beta f = detail::dcf_impl::default_false<std::decay_t<Beta>>())
+{
+    return eq_pack<std::decay_t<Beta>>(std::move(t), std::move(f));
+}
+template <std::size_t N, typename Beta>
+inline auto eq_at(Beta t, Beta f = detail::dcf_impl::default_false<std::decay_t<Beta>>())
+{
+    return eq_at_pack<N, std::decay_t<Beta>>(std::move(t), std::move(f));
+}
+
+template <typename T> struct is_cmp_spec : std::false_type {};
+template <cmp_kind K, typename B> struct is_cmp_spec<cmp_pack<K, B>> : std::true_type {};
+template <std::size_t N, cmp_kind K, typename B>
+struct is_cmp_spec<cmp_at_pack<N, K, B>> : std::true_type {};
+template <typename T>
+inline constexpr bool is_cmp_spec_v = is_cmp_spec<T>::value;
+
+template <typename T> struct is_eq_spec : std::false_type {};
+template <typename B> struct is_eq_spec<eq_pack<B>> : std::true_type {};
+template <std::size_t N, typename B>
+struct is_eq_spec<eq_at_pack<N, B>> : std::true_type {};
+template <typename T>
+inline constexpr bool is_eq_spec_v = is_eq_spec<T>::value;
+
+template <typename T>
+inline constexpr bool is_dcf_spec_v = is_cmp_spec_v<T>;
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_DCF_HPP__

include/dpf/doerner_shelat.hpp

@@ -0,0 +1,542 @@
+/// @file dpf/doerner_shelat.hpp
+/// @brief Doerner–Shelat generation of a dealer DPF key.
+/// @details Two XOR shares of the point are walked level by level. Correction
+///          words, advice bits, seeds, and leaves are the ones `make_dpf`
+///          would emit for the XOR of those shares, the same roots, and the
+///          same beaver coins. Beaver pads used to hide the path bit cancel
+///          and are not part of the key. Pad randomness must not come from
+///          `uniform_fill` if the beaver tape is being matched.
+/// @copyright Copyright (c) 2019-2026 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_DOERNER_SHELAT_HPP__
+#define LIBDPF_INCLUDE_DPF_DOERNER_SHELAT_HPP__
+
+#include <cstddef>
+#include <cstdint>
+#include <type_traits>
+#include <utility>
+
+#include "hedley/hedley.h"
+#include "simde/simde/x86/avx2.h"
+
+#include "dpf/dpf_key.hpp"
+#include "dpf/random.hpp"
+#include "dpf/dcf.hpp"
+
+namespace dpf
+{
+
+/// Roots and the Beaver-pad stream for one Doerner–Shelat generation.
+/// `root` is called twice, same as `make_dpf`: party 0 clears the low bit of
+/// the first sample, party 1 sets the low bit of the second.
+template <typename RootSampler, typename PadRng>
+struct ds_randomness
+{
+    RootSampler root;
+    PadRng pad;
+};
+
+namespace detail
+{
+
+struct urandom_pad_rng
+{
+    simde__m128i block()
+    {
+        return dpf::uniform_sample<simde__m128i>();
+    }
+
+    uint8_t bit()
+    {
+        return static_cast<uint8_t>(dpf::uniform_sample<unsigned char>() & 1u);
+    }
+};
+
+struct ds_cw_party
+{
+    simde__m128i rand;
+    simde__m128i gamma;
+    uint8_t bit;
+};
+
+struct ds_cw_pads
+{
+    ds_cw_party p0;
+    ds_cw_party p1;
+};
+
+struct ds_blind
+{
+    simde__m128i msg;
+    uint8_t bit;
+};
+
+struct ds_and_pads
+{
+    uint8_t a0;
+    uint8_t a1;
+    simde__m128i b0_share, b1_share, c0_share, c1_share;
+};
+
+struct ds_and_shares
+{
+    simde__m128i z0;
+    simde__m128i z1;
+};
+
+HEDLEY_ALWAYS_INLINE
+simde__m128i ds_xor(simde__m128i a, simde__m128i b) noexcept
+{
+    return simde_mm_xor_si128(a, b);
+}
+
+HEDLEY_ALWAYS_INLINE
+simde__m128i ds_gate(uint8_t bit, simde__m128i block) noexcept
+{
+    return dpf::get_if(block, bit & 1u);
+}
+
+template <typename PadRng>
+ds_cw_pads ds_sample_cw(PadRng & pad)
+{
+    ds_cw_pads p{};
+    const simde__m128i zero = simde_mm_setzero_si128();
+    p.p0.rand = pad.block();
+    p.p1.rand = pad.block();
+    p.p0.bit = static_cast<uint8_t>(pad.bit() & 1u);
+    p.p1.bit = static_cast<uint8_t>(pad.bit() & 1u);
+    p.p0.gamma = p.p1.bit ? p.p0.rand : zero;
+    p.p1.gamma = p.p0.bit ? p.p1.rand : zero;
+    return p;
+}
+
+template <typename PadRng>
+ds_and_pads ds_sample_and(PadRng & pad)
+{
+    ds_and_pads p{};
+    const uint8_t a = static_cast<uint8_t>(pad.bit() & 1u);
+    const simde__m128i B = pad.block();
+    const simde__m128i C = ds_gate(a, B);
+    p.a0 = static_cast<uint8_t>(pad.bit() & 1u);
+    p.a1 = static_cast<uint8_t>(a ^ p.a0);
+    p.b0_share = pad.block();
+    p.b1_share = ds_xor(B, p.b0_share);
+    p.c0_share = pad.block();
+    p.c1_share = ds_xor(C, p.c0_share);
+    return p;
+}
+
+HEDLEY_ALWAYS_INLINE
+simde__m128i ds_cw_share(simde__m128i L, simde__m128i R, uint8_t my_bit,
+    const ds_cw_party & mine, const ds_blind & their) noexcept
+{
+    simde__m128i out = ds_xor(R, mine.gamma);
+    if (my_bit & 1u)
+    {
+        out = ds_xor(out, ds_xor(ds_xor(L, R), their.msg));
+    }
+    if (their.bit & 1u)
+    {
+        out = ds_xor(out, mine.rand);
+    }
+    return out;
+}
+
+inline void ds_cw_blinds(const ds_cw_pads & p,
+    simde__m128i L0, simde__m128i R0, uint8_t bit0,
+    simde__m128i L1, simde__m128i R1, uint8_t bit1,
+    ds_blind & b0, ds_blind & b1) noexcept
+{
+    b0.bit = static_cast<uint8_t>(bit0 ^ p.p0.bit);
+    b1.bit = static_cast<uint8_t>(bit1 ^ p.p1.bit);
+    b0.msg = ds_xor(ds_xor(L0, R0), p.p0.rand);
+    b1.msg = ds_xor(ds_xor(L1, R1), p.p1.rand);
+}
+
+inline simde__m128i ds_cw_outs(const ds_cw_pads & p,
+    simde__m128i L0, simde__m128i R0, uint8_t bit0,
+    simde__m128i L1, simde__m128i R1, uint8_t bit1,
+    const ds_blind & b0, const ds_blind & b1) noexcept
+{
+    return ds_xor(
+        ds_cw_share(L0, R0, bit0, p.p0, b1),
+        ds_cw_share(L1, R1, bit1, p.p1, b0));
+}
+
+inline uint8_t ds_open_advice(simde__m128i L0, simde__m128i R0, uint8_t bit0,
+    simde__m128i L1, simde__m128i R1, uint8_t bit1) noexcept
+{
+    const uint8_t a00 = static_cast<uint8_t>(dpf::get_lo_bit(L0) ^ bit0);
+    const uint8_t a01 = static_cast<uint8_t>(dpf::get_lo_bit(R0) ^ bit0);
+    const uint8_t a10 = static_cast<uint8_t>(dpf::get_lo_bit(L1) ^ bit1);
+    const uint8_t a11 = static_cast<uint8_t>(dpf::get_lo_bit(R1) ^ bit1);
+    const uint8_t t0 = static_cast<uint8_t>(a00 ^ a10 ^ 1u);
+    const uint8_t t1 = static_cast<uint8_t>(a01 ^ a11);
+    return static_cast<uint8_t>((t1 << 1) | (t0 & 1u));
+}
+
+inline void ds_next_terms(simde__m128i L, simde__m128i R, uint8_t advice,
+    simde__m128i cw, uint8_t tpack, simde__m128i & M, simde__m128i & base) noexcept
+{
+    const simde__m128i D = ds_xor(L, R);
+    const uint8_t t0 = static_cast<uint8_t>(tpack & 1u);
+    const uint8_t t1 = static_cast<uint8_t>((tpack >> 1) & 1u);
+    const simde__m128i lo = dpf::set_lo_bit(simde_mm_setzero_si128(), 1);
+    const simde__m128i DT = ds_gate(static_cast<uint8_t>(t0 ^ t1), lo);
+    const simde__m128i cw_base = ds_xor(dpf::unset_lo_bit(cw), ds_gate(t0, lo));
+    M = (advice & 1u) ? ds_xor(D, DT) : D;
+    base = (advice & 1u) ? ds_xor(L, cw_base) : L;
+}
+
+inline ds_and_shares ds_and_open(const ds_and_pads & p, simde__m128i M,
+    uint8_t b_recv) noexcept
+{
+    const simde__m128i e = ds_xor(ds_xor(M, p.b0_share), p.b1_share);
+    const uint8_t d = static_cast<uint8_t>((b_recv ^ p.a1) ^ p.a0);
+    ds_and_shares z;
+    z.z0 = ds_xor(ds_xor(ds_xor(ds_gate(d, e), ds_gate(d, p.b0_share)),
+        ds_gate(p.a0, e)), p.c0_share);
+    z.z1 = ds_xor(ds_xor(ds_gate(d, p.b1_share), ds_gate(p.a1, e)), p.c1_share);
+    return z;
+}
+
+inline simde__m128i ds_deliver(uint8_t b_exp, simde__m128i base, simde__m128i M,
+    const ds_and_shares & z) noexcept
+{
+    const simde__m128i local = ds_xor(base, ds_gate(b_exp, M));
+    return ds_xor(ds_xor(local, z.z0), z.z1);
+}
+
+/// Per-level messages prepared before the CW protocol runs (blinds + pads).
+struct ds_level_blinds
+{
+    ds_cw_pads cwp;
+    ds_and_pads and0;
+    ds_and_pads and1;
+    ds_blind b0;
+    ds_blind b1;
+    simde__m128i L0, R0, L1, R1;
+    uint8_t bit0;
+    uint8_t bit1;
+};
+
+/// Opened CW, advice, and AND products delivered by a `CwProtocol`.
+struct ds_level_open
+{
+    simde__m128i cw;
+    uint8_t advice;
+    ds_and_shares z0;
+    ds_and_shares z1;
+    uint64_t value_cw = 0;  // public after open when cmp is active at this level
+};
+
+/// Running comparison-gen state shared across DS levels (Va residual).
+/// When `track_coeff` is set (wildcard cmp payload), a parallel β = 1
+/// accumulator `Va1` is advanced alongside `Va` so the gen can stash
+/// `value_cw(1) − value_cw(β)` coefficients for a later `assign_cmp`.
+struct ds_cmp_gen_state
+{
+    bool active = false;
+    std::size_t nbits = 0;
+    uint64_t mask = 0;
+    uint64_t beta = 0;
+    bool include_eq = false;
+    cmp_trivial trivial = cmp_trivial::none;
+    uint64_t Va = 0;
+    unsigned __int128 thresh = 0;
+    bool track_coeff = false;
+    uint64_t Va1 = 0;
+    uint64_t last_vcw_coeff = 0;
+};
+
+/// Local joint simulation: today's `ds_cw_outs` / `ds_open_advice` / `ds_and_open`.
+/// An MPC backend would send `blinds` and return the same `ds_level_open` shape.
+template <typename PadRng>
+struct local_cw_protocol
+{
+    PadRng & pads;
+
+    ds_level_blinds prepare_level(simde__m128i L0, simde__m128i R0, uint8_t bit0,
+        simde__m128i L1, simde__m128i R1, uint8_t bit1)
+    {
+        ds_level_blinds b;
+        b.cwp = ds_sample_cw(pads);
+        b.and0 = ds_sample_and(pads);
+        b.and1 = ds_sample_and(pads);
+        b.L0 = L0;
+        b.R0 = R0;
+        b.L1 = L1;
+        b.R1 = R1;
+        b.bit0 = bit0;
+        b.bit1 = bit1;
+        ds_cw_blinds(b.cwp, L0, R0, bit0, L1, R1, bit1, b.b0, b.b1);
+        return b;
+    }
+
+    ds_level_open complete_level(std::size_t /*level*/, const ds_level_blinds & b,
+        simde__m128i M0, simde__m128i M1, uint8_t rec0, uint8_t rec1)
+    {
+        ds_level_open out;
+        out.advice = ds_open_advice(b.L0, b.R0, b.bit0, b.L1, b.R1, b.bit1);
+        out.cw = ds_cw_outs(b.cwp, b.L0, b.R0, b.bit0, b.L1, b.R1, b.bit1,
+            b.b0, b.b1);
+        out.z0 = ds_and_open(b.and0, M0, rec0);
+        out.z1 = ds_and_open(b.and1, M1, rec1);
+        return out;
+    }
+
+    /// Open CW + advice only (AND pads stay in `blinds` for a later open).
+    std::pair<simde__m128i, uint8_t> open_cw(const ds_level_blinds & b) noexcept
+    {
+        return {ds_cw_outs(b.cwp, b.L0, b.R0, b.bit0, b.L1, b.R1, b.bit1,
+                    b.b0, b.b1),
+            ds_open_advice(b.L0, b.R0, b.bit0, b.L1, b.R1, b.bit1)};
+    }
+
+    /// Open the public value CW for this level (local: clear convert+make_value_cw).
+    /// MPC backends open additive shares of the same word.
+    uint64_t open_value_cw(const ds_level_blinds & b, uint8_t adv0, uint8_t adv1,
+        int ai, uint64_t & Va, uint64_t beta, uint64_t mask) noexcept
+    {
+        return dcf_impl::make_value_cw(b.L0, b.R0, b.L1, b.R1, adv0,
+            adv1, ai, Va, beta, mask);
+    }
+
+    ds_and_shares open_and(const ds_and_pads & p, simde__m128i M,
+        uint8_t b_recv) noexcept
+    {
+        return ds_and_open(p, M, b_recv);
+    }
+
+    /// Open the final comparison leaf CW. Wraps `make_final_cw` so the
+    /// Doerner–Shelat gen does not call it directly on reconstructed seeds;
+    /// an MPC backend would open additive shares of the same word.
+    uint64_t open_final_cw(simde__m128i s0, simde__m128i s1, uint8_t t1,
+        uint64_t Va, uint64_t mask, uint64_t on_path) noexcept
+    {
+        return dcf_impl::make_final_cw(s0, s1, t1, Va, mask, on_path);
+    }
+
+    /// Draw the group-width `cmp_addend` blind. Local joint simulation reuses
+    /// the shared root sampler so the blind matches the dealer's; an MPC
+    /// backend would instead pull a group-width element from the pad stream.
+    template <typename BlockSampler>
+    uint64_t sample_addend_blind(uint64_t mask, BlockSampler && sample) noexcept
+    {
+        return dcf_impl::sample_addend_blind(mask,
+            std::forward<BlockSampler>(sample));
+    }
+
+    /// Open a group of leaf correction words for one prefix group. In this
+    /// local joint simulation both XOR shares of the point are present, so the
+    /// point is reconstructed *inside* the protocol and handed to `leaf_fn`
+    /// (which runs `make_leaves` for the group). The Doerner–Shelat gen never
+    /// forms `x = x0 ^ x1` at its own call site; an MPC backend would instead
+    /// run a per-group leaf CW exchange that never reveals `x`.
+    template <typename InputT, typename LeafFn>
+    void open_leaf_group(InputT x0, InputT x1, LeafFn && leaf_fn)
+    {
+        std::forward<LeafFn>(leaf_fn)(utils::xor_input_shares(x0, x1));
+    }
+};
+
+/// Generation-side level state (seeds / home bits). Not an eval path memoizer.
+template <typename NodeT>
+struct ds_gen_state
+{
+    NodeT inbox[2];
+    int home[2];
+    NodeT root0;
+    NodeT root1;
+
+    void init(NodeT r0, NodeT r1) noexcept
+    {
+        root0 = r0;
+        root1 = r1;
+        inbox[0] = r0;
+        inbox[1] = r1;
+        home[0] = 0;
+        home[1] = 1;
+    }
+
+    NodeT & seed0() noexcept { return inbox[home[0]]; }
+    NodeT & seed1() noexcept { return inbox[home[1]]; }
+    const NodeT & seed0() const noexcept { return inbox[home[0]]; }
+    const NodeT & seed1() const noexcept { return inbox[home[1]]; }
+};
+
+/// One interior level: expand, protocol open, advance both party seeds.
+/// When `cmp` is non-null and active for `level`, also opens `value_cw` via
+/// the protocol (no second PRG expand outside).
+template <typename InteriorPRG, typename CwProtocol, typename NodeT,
+          typename InputT, typename AdviceT>
+void ds_advance_level(ds_gen_state<NodeT> & st, InputT x0, InputT x1,
+    InputT mask, std::size_t level, CwProtocol & proto, NodeT & cw_out,
+    AdviceT & advice_out, uint64_t * value_cw_out = nullptr,
+    ds_cmp_gen_state * cmp = nullptr)
+{
+    // Integral bridge so bit extraction works for `keyword` / `modint` /
+    // signed / bitstring the same way dealer gen does via `mask & x`.
+    constexpr auto to_int = utils::to_integral_type<InputT>{};
+    const auto mi = to_int(mask);
+    const uint8_t bit0 = static_cast<uint8_t>(!!(mi & to_int(x0)));
+    const uint8_t bit1 = static_cast<uint8_t>(!!(mi & to_int(x1)));
+
+    NodeT s0 = st.seed0();
+    NodeT s1 = st.seed1();
+    const uint8_t adv0 = static_cast<uint8_t>(
+        dpf::get_lo_bit_and_clear_lo_2bits(s0));
+    const uint8_t adv1 = static_cast<uint8_t>(
+        dpf::get_lo_bit_and_clear_lo_2bits(s1));
+    const auto c0 = InteriorPRG::eval01(s0);
+    const auto c1 = InteriorPRG::eval01(s1);
+
+    auto blinds = proto.prepare_level(c0[0], c0[1], bit0, c1[0], c1[1], bit1);
+
+    if (value_cw_out != nullptr && cmp != nullptr && cmp->active
+        && cmp->trivial == cmp_trivial::none && level < cmp->nbits)
+    {
+        const int ai = static_cast<int>(
+            (cmp->thresh >> (cmp->nbits - 1 - level)) & 1);
+        *value_cw_out = proto.open_value_cw(blinds, adv0, adv1, ai, cmp->Va,
+            cmp->beta, cmp->mask);
+        if (cmp->track_coeff)
+        {
+            // Affine coefficient: same level with β = 1 on a parallel Va.
+            const uint64_t v1 = proto.open_value_cw(blinds, adv0, adv1, ai,
+                cmp->Va1, 1ULL, cmp->mask);
+            cmp->last_vcw_coeff =
+                (v1 + dcf_impl::neg_m(*value_cw_out, cmp->mask)) & cmp->mask;
+        }
+    }
+
+    auto [cw, tpack] = proto.open_cw(blinds);
+
+    const uint8_t exp0 = st.home[0] == 0 ? bit0 : bit1;
+    const uint8_t rec0 = st.home[0] == 0 ? bit1 : bit0;
+    const uint8_t exp1 = st.home[1] == 0 ? bit0 : bit1;
+    const uint8_t rec1 = st.home[1] == 0 ? bit1 : bit0;
+
+    NodeT M0, base0, M1, base1;
+    ds_next_terms(c0[0], c0[1], adv0, cw, tpack, M0, base0);
+    ds_next_terms(c1[0], c1[1], adv1, cw, tpack, M1, base1);
+    const NodeT nxt0 =
+        ds_deliver(exp0, base0, M0, proto.open_and(blinds.and0, M0, rec0));
+    const NodeT nxt1 =
+        ds_deliver(exp1, base1, M1, proto.open_and(blinds.and1, M1, rec1));
+
+    st.home[0] ^= 1;
+    st.home[1] ^= 1;
+    st.inbox[st.home[0]] = nxt0;
+    st.inbox[st.home[1]] = nxt1;
+
+    cw_out = cw;
+    advice_out = tpack;
+}
+
+template <typename T>
+struct is_ds_randomness : std::false_type {};
+
+template <typename RootSampler, typename PadRng>
+struct is_ds_randomness<ds_randomness<RootSampler, PadRng>> : std::true_type {};
+
+template <typename T, typename = void>
+struct is_cw_protocol : std::false_type {};
+
+template <typename PadRng>
+struct is_cw_protocol<local_cw_protocol<PadRng>, void> : std::true_type {};
+
+template <typename T>
+struct is_cw_protocol<T,
+    std::void_t<decltype(std::declval<T &>().prepare_level(
+                    simde_mm_setzero_si128(), simde_mm_setzero_si128(),
+                    uint8_t{}, simde_mm_setzero_si128(),
+                    simde_mm_setzero_si128(), uint8_t{})),
+                decltype(std::declval<T &>().open_cw(
+                    std::declval<const ds_level_blinds &>()))>>
+    : std::true_type {};
+
+template <typename ...Ts>
+struct first_is_cw_protocol : std::false_type {};
+
+template <typename T, typename ...Rest>
+struct first_is_cw_protocol<T, Rest...>
+    : is_cw_protocol<std::decay_t<T>> {};
+
+template <typename InteriorPRG,
+          typename ExteriorPRG,
+          typename InputT,
+          typename OutputT,
+          typename ...OutputTs,
+          typename RootSampler,
+          typename CwProtocol>
+auto make_dpf_doerner_shelat_impl(InputT x0, InputT x1,
+    RootSampler & root_sampler, CwProtocol & proto, OutputT && y,
+    OutputTs && ...ys)
+{
+    static_assert(!dpf::is_wildcard_v<InputT>,
+        "Doerner–Shelat gen takes XOR shares of a concrete point");
+    static_assert(!dpf::is_secret_share_v<InputT>,
+        "Doerner–Shelat: pass additive_share of xor_wrapper, or raw XOR shares");
+    static_assert(sizeof(typename InteriorPRG::block_type) == sizeof(simde__m128i),
+        "Doerner–Shelat gen uses the AES-block interior node");
+
+    using dpf_type = utils::dpf_type_t<InteriorPRG, ExteriorPRG, InputT,
+                                       OutputT, OutputTs...>;
+    using node = typename dpf_type::interior_node;
+    using input_type = typename dpf_type::input_type;
+    constexpr auto depth = dpf_type::depth;
+
+    utils::flip_msb_if_signed_integral(x0);
+
+    const node root0 = dpf::unset_lo_bit(static_cast<node>(root_sampler()));
+    const node root1 = dpf::set_lo_bit(static_cast<node>(root_sampler()));
+
+    ds_gen_state<node> st;
+    st.init(root0, root1);
+
+    typename dpf_type::correction_words_array correction_words{};
+    typename dpf_type::correction_advice_array correction_advice{};
+
+    auto mask = dpf_type::msb_mask;
+    for (std::size_t level = 0; level < depth; ++level, mask >>= 1)
+    {
+        ds_advance_level<InteriorPRG>(st, x0, x1, mask, level, proto,
+            correction_words[level], correction_advice[level]);
+    }
+
+    const node parent0 = st.seed0();
+    const node parent1 = st.seed1();
+    const bool sign0 = dpf::get_lo_bit(parent0);
+
+    input_type x = utils::xor_input_shares(x0, x1);
+    auto built = dpf::make_leaves<ExteriorPRG>(x,
+        dpf::unset_lo_2bits(parent0), dpf::unset_lo_2bits(parent1), sign0,
+        std::size_t{0}, std::forward<OutputT>(y), std::forward<OutputTs>(ys)...);
+
+    input_type off0{};
+    input_type off1{};
+    return dpf::make_party_key_pair(
+        dpf_type{root0, correction_words, correction_advice,
+            built.first.first, built.first.second, off0},
+        dpf_type{root1, correction_words, correction_advice,
+            built.second.first, built.second.second, off1});
+}
+
+}  // namespace detail
+
+/// Local CW protocol (pads cancel; same keys as dealer when roots match).
+template <typename PadRng>
+using local_cw_protocol = detail::local_cw_protocol<PadRng>;
+
+template <typename NodeT>
+using ds_gen_state = detail::ds_gen_state<NodeT>;
+
+// Public `make_dpf_doerner_shelat(x0, x1, ...)` lives in incremental.hpp so
+// classic and `at<>` / mixed-width packs share one entry point.
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_DOERNER_SHELAT_HPP__

include/dpf/emplace.hpp

@@ -0,0 +1,252 @@
+/// @file dpf/emplace.hpp
+/// @brief Defines various template structures for efficient emplacement of
+///        `dpf::dpf_key` objects.
+/// @details Provides specialized template structures for the in-place
+///          construction ("emplacing") of `dpf::dpf_key` objects into different
+///          types of pre-allocated storage including smart pointers
+///          (`std::unique_ptr` and `std::shared_ptr`), `std::optional`,
+///          `std::variant`, raw pointers, and `std::reference_wrapper`s, as
+///          well as containers that support `emplace_back`. The goal is to
+///          facilitate efficient construction and storage of `dpf::dpf_key`
+///          objects received from a dealer over a socket.
+///
+///          The emplacement functionalities are specialized for different
+///          storage types to handle their unique construction requirements.
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_EMPLACE_HPP__
+#define LIBDPF_INCLUDE_DPF_EMPLACE_HPP__
+
+#include "hedley/hedley.h"
+
+#include <variant>
+#include <memory>
+#include <optional>
+#include <functional>
+
+namespace dpf
+{
+
+namespace utils
+{
+
+/// @brief Emplaces a `dpf::dpf_key` object into the specified, pre-allocated memory.
+/// @tparam DpfKey The concrete specialization of `dpf::dpf_key` to construct.
+/// @param storage Reference to the container where the `dpf::dpf_key` object will be emplaced.
+/// @param root The root node used by the `dpf::dpf_key`.
+/// @param correction_words Correction words array for the `dpf::dpf_key`.
+/// @param correction_advice Correction advice array for the `dpf::dpf_key`.
+/// @param leaves Leaf-node tuple for the `dpf::dpf_key`.
+/// @param beavers Beaver tuple for the `dpf::dpf_key`.
+/// @param offset_share The offset share (default: `0`).
+
+/// @defgroup EmplaceFunctions Emplace Functions
+/// @brief Group of functions for emplacing `dpf::dpf_key` objects into
+///        pre-allocated memory.
+/// @{
+
+template <typename DpfKey, typename T>
+struct dpf_emplacer
+{
+    using dpf_key = DpfKey;
+    using interior_node = typename DpfKey::interior_node;
+    using correction_words_array = typename DpfKey::correction_words_array;
+    using correction_advice_array = typename DpfKey::correction_advice_array;
+    using leaf_tuple = typename DpfKey::leaf_tuple;
+    using beaver_tuple = typename DpfKey::beaver_tuple;
+    using input_type = typename DpfKey::input_type;
+
+    /// @brief Generic version is intentionally left undefined.
+    static auto emplace(T & storage,
+        const interior_node & root,
+        const correction_words_array & correction_words,
+        const correction_advice_array & correction_advice,
+        const leaf_tuple & leaves,
+        const beaver_tuple & beavers,
+        const input_type & offset_share);
+};
+
+/// @brief Specialization for `std::unique_ptr`.
+template <typename DpfKey>
+struct dpf_emplacer<DpfKey, std::unique_ptr<DpfKey>>
+{
+    using dpf_key = DpfKey;
+    using interior_node = typename DpfKey::interior_node;
+    using correction_words_array = typename DpfKey::correction_words_array;
+    using correction_advice_array = typename DpfKey::correction_advice_array;
+    using leaf_tuple = typename DpfKey::leaf_tuple;
+    using beaver_tuple = typename DpfKey::beaver_tuple;
+    using input_type = typename DpfKey::input_type;
+
+    HEDLEY_ALWAYS_INLINE
+    static auto emplace(
+        std::unique_ptr<DpfKey> & storage,
+        const interior_node & root,
+        const correction_words_array & correction_words,
+        const correction_advice_array & correction_advice,
+        const leaf_tuple & leaves,
+        const beaver_tuple & beavers,
+        const input_type & offset_share)
+    {
+        storage.reset(new DpfKey(root, correction_words, correction_advice, leaves, beavers, offset_share));
+    }
+};
+
+template <typename DpfKey>
+struct dpf_emplacer<DpfKey, std::shared_ptr<DpfKey>>
+{
+    using dpf_key = DpfKey;
+    using interior_node = typename DpfKey::interior_node;
+    using correction_words_array = typename DpfKey::correction_words_array;
+    using correction_advice_array = typename DpfKey::correction_advice_array;
+    using leaf_tuple = typename DpfKey::leaf_tuple;
+    using beaver_tuple = typename DpfKey::beaver_tuple;
+    using input_type = typename DpfKey::input_type;
+
+    HEDLEY_ALWAYS_INLINE
+    static auto emplace(
+        std::shared_ptr<DpfKey> & storage,
+        const interior_node & root,
+        const correction_words_array & correction_words,
+        const correction_advice_array & correction_advice,
+        const leaf_tuple & leaves,
+        const beaver_tuple & beavers,
+        const input_type & offset_share)
+    {
+        storage = std::make_shared<DpfKey>(root, correction_words, correction_advice, leaves, beavers, offset_share);
+    }
+};
+
+template <typename DpfKey>
+struct dpf_emplacer<DpfKey, std::optional<DpfKey>>
+{
+    using dpf_key = DpfKey;
+    using interior_node = typename DpfKey::interior_node;
+    using correction_words_array = typename DpfKey::correction_words_array;
+    using correction_advice_array = typename DpfKey::correction_advice_array;
+    using leaf_tuple = typename DpfKey::leaf_tuple;
+    using beaver_tuple = typename DpfKey::beaver_tuple;
+    using input_type = typename DpfKey::input_type;
+
+    HEDLEY_ALWAYS_INLINE
+    static auto emplace(
+        std::optional<DpfKey> & storage,
+        const interior_node & root,
+        const correction_words_array & correction_words,
+        const correction_advice_array & correction_advice,
+        const leaf_tuple & leaves,
+        const beaver_tuple & beavers,
+        const input_type & offset_share)
+    {
+        storage.emplace(root, correction_words, correction_advice, leaves, beavers, offset_share);
+    }
+};
+
+template <typename DpfKey, typename ...Ts>
+struct dpf_emplacer<DpfKey, std::variant<Ts...>>
+{
+    using dpf_key = DpfKey;
+    using interior_node = typename DpfKey::interior_node;
+    using correction_words_array = typename DpfKey::correction_words_array;
+    using correction_advice_array = typename DpfKey::correction_advice_array;
+    using leaf_tuple = typename DpfKey::leaf_tuple;
+    using beaver_tuple = typename DpfKey::beaver_tuple;
+    using input_type = typename DpfKey::input_type;
+
+    HEDLEY_ALWAYS_INLINE
+    static auto emplace(
+        std::variant<Ts...> & storage,
+        const interior_node & root,
+        const correction_words_array & correction_words,
+        const correction_advice_array & correction_advice,
+        const leaf_tuple & leaves,
+        const beaver_tuple & beavers,
+        const input_type & offset_share)
+    {
+        storage.template emplace<DpfKey>(root, correction_words, correction_advice, leaves, beavers, offset_share);
+    }
+};
+
+template <typename DpfKey>
+struct dpf_emplacer<DpfKey, DpfKey *>
+{
+    using dpf_key = DpfKey;
+    using interior_node = typename DpfKey::interior_node;
+    using correction_words_array = typename DpfKey::correction_words_array;
+    using correction_advice_array = typename DpfKey::correction_advice_array;
+    using leaf_tuple = typename DpfKey::leaf_tuple;
+    using beaver_tuple = typename DpfKey::beaver_tuple;
+    using input_type = typename DpfKey::input_type;
+
+    HEDLEY_ALWAYS_INLINE
+    static auto emplace(
+        DpfKey * storage,
+        const interior_node & root,
+        const correction_words_array & correction_words,
+        const correction_advice_array & correction_advice,
+        const leaf_tuple & leaves,
+        const beaver_tuple & beavers,
+        const input_type & offset_share)
+    {
+        ::new (storage) DpfKey(root, correction_words, correction_advice, leaves, beavers, offset_share);
+    }
+};
+
+template <typename DpfKey>
+struct dpf_emplacer<DpfKey, std::reference_wrapper<DpfKey>>
+{
+    using dpf_key = DpfKey;
+    using interior_node = typename DpfKey::interior_node;
+    using correction_words_array = typename DpfKey::correction_words_array;
+    using correction_advice_array = typename DpfKey::correction_advice_array;
+    using leaf_tuple = typename DpfKey::leaf_tuple;
+    using beaver_tuple = typename DpfKey::beaver_tuple;
+    using input_type = typename DpfKey::input_type;
+
+    HEDLEY_ALWAYS_INLINE
+    static auto emplace(
+        std::reference_wrapper<DpfKey> storage,
+        const interior_node & root,
+        const correction_words_array & correction_words,
+        const correction_advice_array & correction_advice,
+        const leaf_tuple & leaves,
+        const beaver_tuple & beavers,
+        const input_type & offset_share)
+    {
+        storage.get().~DpfKey(); 
+        ::new (&storage.get()) DpfKey(root, correction_words, correction_advice, leaves, beavers, offset_share);
+    }
+};
+
+/// @}
+
+template <typename DpfKey, typename ContainerT>
+struct dpf_back_emplacer
+{
+    using dpf_key = DpfKey;
+    using interior_node = typename DpfKey::interior_node;
+    using correction_words_array = typename DpfKey::correction_words_array;
+    using correction_advice_array = typename DpfKey::correction_advice_array;
+    using leaf_tuple = typename DpfKey::leaf_tuple;
+    using beaver_tuple = typename DpfKey::beaver_tuple;
+    using input_type = typename DpfKey::input_type;
+
+    static auto emplace_back(ContainerT & storage,
+        const interior_node & root,
+        const correction_words_array & correction_words,
+        const correction_advice_array & correction_advice,
+        const leaf_tuple & leaves,
+        const beaver_tuple & beavers,
+        const input_type & offset_share)
+    {
+        storage.emplace_back(root, correction_words, correction_advice, leaves, beavers, offset_share);
+    }
+};
+
+}  // namespace utils
+
+}  // namespace std
+
+#endif  // LIBDPF_INCLUDE_DPF_EMPLACE_HPP__

include/dpf/eval_common.hpp

@@ -0,0 +1,169 @@
+/// @file dpf/eval_common.hpp
+/// @brief
+/// @details
+/// @author Ryan Henry <ryan.henry@ucalgary.ca>
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_EVAL_COMMON_HPP__
+#define LIBDPF_INCLUDE_DPF_EVAL_COMMON_HPP__
+
+#include "hedley/hedley.h"
+
+#include <cstddef>
+#include <cstring>
+#include <limits>
+#include <stdexcept>
+#include <type_traits>
+
+#include "dpf/leaf_node.hpp"
+#include "dpf/secret_share.hpp"
+
+namespace dpf
+{
+
+/// Sentinel: `dpf_output` converts to a bare `OutputT` (no party tag).
+inline constexpr std::size_t no_party = std::numeric_limits<std::size_t>::max();
+
+/// Eval result type for a leaf output of `KeyT`: subtractive share when the
+/// key is party-tagged, otherwise the concrete output type.
+template <typename KeyT, typename OutputT, bool = is_party_key_v<KeyT>>
+struct eval_leaf_result
+{
+    using type = OutputT;
+};
+template <typename KeyT, typename OutputT>
+struct eval_leaf_result<KeyT, OutputT, true>
+{
+    using type = subtractive_share<OutputT, party_of_v<KeyT>>;
+};
+template <typename KeyT, typename OutputT>
+using eval_leaf_result_t = typename eval_leaf_result<KeyT, OutputT>::type;
+
+/// Eval result type for a comparison output of `KeyT`: additive share when
+/// the key is party-tagged, otherwise `Beta`.
+template <typename KeyT, typename Beta, bool = is_party_key_v<KeyT>>
+struct eval_cmp_result
+{
+    using type = Beta;
+};
+template <typename KeyT, typename Beta>
+struct eval_cmp_result<KeyT, Beta, true>
+{
+    using type = additive_share<Beta, party_of_v<KeyT>>;
+};
+template <typename KeyT, typename Beta>
+using eval_cmp_result_t = typename eval_cmp_result<KeyT, Beta>::type;
+
+template <typename OutputT,
+          typename NodeT,
+          std::size_t Party = no_party>
+struct alignas(utils::max_align_v) dpf_output
+{
+    using result_type = std::conditional_t<
+        Party == no_party,
+        OutputT,
+        subtractive_share<OutputT, Party>>;
+
+    dpf_output(const dpf_output &) = default;
+    dpf_output(dpf_output &&) noexcept = default;
+    dpf_output & operator=(const dpf_output &) = default;
+    dpf_output & operator=(dpf_output &&) noexcept = default;
+    ~dpf_output() = default;
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_PURE
+    constexpr operator result_type() const
+    {
+        OutputT v = extract_leaf<NodeT, OutputT>(node, offset);
+        if constexpr (Party == no_party)
+            return v;
+        else
+            return subtractive_share<OutputT, Party>::from_raw(v);
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_PURE
+    auto operator*() const
+    {
+        return static_cast<result_type>(*this);
+    }
+
+    NodeT node;
+    std::size_t offset;
+
+  private:
+    dpf_output(NodeT leaf_node, std::size_t off)
+      : node{leaf_node}, offset{off} { }
+
+  public:
+    template <typename Output,
+              typename Input,
+              typename Node>
+    friend auto make_dpf_output(const Node & node, Input x);
+
+    template <std::size_t P,
+              typename Output,
+              typename Input,
+              typename Node>
+    friend auto make_dpf_output(const Node & node, Input x);
+};
+
+/// Copy one packed leaf into a buffer whose element type may differ
+/// from `LeafT` (bit arrays store `word_type`, not the exterior node).
+/// Byte destination keeps the store free of strict-aliasing UB.
+template <typename LeafT, typename Buffer>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+void store_leaf_bytes(Buffer && buf, std::size_t index, const LeafT & leaf) noexcept
+{
+    auto * dst = reinterpret_cast<unsigned char *>(utils::data(buf))
+        + index * sizeof(LeafT);
+    std::memcpy(dst, std::addressof(leaf), sizeof(LeafT));
+}
+
+template <typename Output,
+          typename Input,
+          typename Node>
+auto make_dpf_output(const Node & node, Input x)
+{
+    return dpf_output<concrete_type_t<Output>, Node>{node,
+        offset_within_block<concrete_type_t<Output>, Node>(x)};
+}
+
+template <std::size_t Party,
+          typename Output,
+          typename Input,
+          typename Node>
+auto make_dpf_output(const Node & node, Input x)
+{
+    return dpf_output<concrete_type_t<Output>, Node, Party>{node,
+        offset_within_block<concrete_type_t<Output>, Node>(x)};
+}
+
+/// Wrap a raw leaf node into a party-tagged `dpf_output` when `KeyT` is a
+/// `party_key`, otherwise a bare `dpf_output`.
+template <typename KeyT, typename Output, typename Input, typename Node>
+auto make_eval_dpf_output(const Node & node, Input x)
+{
+    if constexpr (is_party_key_v<KeyT>)
+        return make_dpf_output<party_of_v<KeyT>, Output>(node, x);
+    else
+        return make_dpf_output<Output>(node, x);
+}
+
+/// Wrap a raw comparison `Beta` value as an additive share when `KeyT` is a
+/// `party_key`.
+template <typename KeyT, typename Beta>
+auto make_eval_cmp_result(Beta raw) noexcept
+{
+    if constexpr (is_party_key_v<KeyT>)
+        return additive_share<Beta, party_of_v<KeyT>>::from_raw(raw);
+    else
+        return raw;
+}
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_EVAL_COMMON_HPP__

include/dpf/eval_full.hpp

@@ -0,0 +1,152 @@
+/// @file dpf/eval_full.hpp
+/// @brief
+/// @details
+/// @author Ryan Henry <ryan.henry@ucalgary.ca>
+/// @author Christopher Jiang <christopher.jiang@ucalgary.ca>
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_EVAL_FULL_HPP__
+#define LIBDPF_INCLUDE_DPF_EVAL_FULL_HPP__
+
+#include <portable-snippets/builtin/builtin.h>
+#include "hedley/hedley.h"
+
+#include <cstddef>
+#include <type_traits>
+#include <utility>
+#include <limits>
+
+#include "dpf/dpf_key.hpp"
+#include "dpf/eval_common.hpp"
+#include "dpf/eval_target.hpp"
+#include "dpf/output_buffer.hpp"
+#include "dpf/interval_memoizer.hpp"
+#include "dpf/rotation_iterable.hpp"
+#include "dpf/subinterval_iterable.hpp"
+
+namespace dpf
+{
+
+namespace internal
+{
+
+template <std::size_t ...Is,
+          typename DpfKey,
+          typename OutputBuffers,
+          typename IntervalMemoizer,
+          std::size_t ...IIs,
+          std::enable_if_t<dpf::is_wildcard_v<typename DpfKey::raw_input_type>, bool> = false>
+auto eval_full(const DpfKey & dpf, OutputBuffers && outbufs,
+    IntervalMemoizer && memoizer, std::index_sequence<IIs...>)
+{
+    using dpf_type = DpfKey;
+    using input_type = typename dpf_type::input_type;
+    auto offset = dpf.offset_x(0);  // N.B.: throws if dpf is not ready
+
+    dpf::internal::eval_interval_impl<Is...>(dpf,
+            std::numeric_limits<input_type>::min(),
+            std::numeric_limits<input_type>::max(),
+            outbufs, memoizer, std::make_index_sequence<sizeof...(Is)>());
+
+    return utils::make_tuple(dpf::rotation_iterable(std::begin(utils::get<IIs>(outbufs)), std::end(utils::get<IIs>(outbufs)), offset)...);
+}
+
+template <std::size_t ...Is,
+          typename DpfKey,
+          typename OutputBuffers,
+          typename IntervalMemoizer,
+          std::size_t ...IIs,
+          std::enable_if_t<!dpf::is_wildcard_v<typename DpfKey::raw_input_type>, bool> = false>
+auto eval_full(const DpfKey & dpf, OutputBuffers && outbufs,
+    IntervalMemoizer && memoizer, std::index_sequence<IIs...>)
+{
+    using dpf_type = DpfKey;
+    using input_type = typename dpf_type::input_type;
+
+    dpf::internal::eval_interval_impl<Is...>(dpf,
+            std::numeric_limits<input_type>::min(),
+            std::numeric_limits<input_type>::max(),
+            outbufs, memoizer, std::make_index_sequence<sizeof...(Is)>());
+
+    return utils::make_tuple(
+        subinterval_iterable(std::begin(utils::get<IIs>(outbufs)),
+            utils::size(utils::get<IIs>(outbufs)),
+            std::size_t{0},
+            utils::get<IIs>(outbufs).size() - 1,
+            std::size_t{0},
+            std::size_t{0})...);
+}
+
+}  // namespace internal
+
+template <std::size_t I = 0,
+          std::size_t ...Is,
+          typename DpfKey,
+          typename OutputBuffers,
+          typename IntervalMemoizer,
+          std::enable_if_t<looks_like_dpf_key_v<DpfKey> && !is_multilevel_key_v<DpfKey>, bool> = true>
+HEDLEY_ALWAYS_INLINE
+auto eval_full(const DpfKey & dpf, OutputBuffers && outbufs,
+    IntervalMemoizer && memoizer)
+{
+    assert_not_wildcard_output<I, Is...>(dpf);
+
+    return internal::eval_full<I, Is...>(dpf, outbufs, memoizer, std::make_index_sequence<1+sizeof...(Is)>());
+}
+
+template <std::size_t I = 0,
+          std::size_t ...Is,
+          typename DpfKey,
+          typename OutputBuffers,
+          std::enable_if_t<looks_like_dpf_key_v<DpfKey> && !is_multilevel_key_v<DpfKey>, bool> = true,
+          std::enable_if_t<!std::is_base_of_v<
+              dpf::interval_memoizer_base<unwrap_party_key_t<DpfKey>>,
+              std::decay_t<OutputBuffers>>, bool> = true>
+HEDLEY_ALWAYS_INLINE
+auto eval_full(const DpfKey & dpf, OutputBuffers & outbufs)  // NOLINT(runtime/references)
+{
+    using input_type = typename DpfKey::input_type;
+    return eval_full<I, Is...>(dpf, outbufs,
+        dpf::make_basic_full_memoizer(dpf));
+}
+
+template <std::size_t I = 0,
+          std::size_t ...Is,
+          typename DpfKey,
+          typename IntervalMemoizer,
+          std::enable_if_t<looks_like_dpf_key_v<DpfKey> && !is_multilevel_key_v<DpfKey>, bool> = true,
+          std::enable_if_t<std::is_base_of_v<
+              dpf::interval_memoizer_base<unwrap_party_key_t<DpfKey>>,
+              std::decay_t<IntervalMemoizer>>, bool> = true>
+HEDLEY_ALWAYS_INLINE
+auto eval_full(const DpfKey & dpf,
+    IntervalMemoizer && memoizer)
+{
+    auto outbufs = utils::make_tuple(
+        make_output_buffer_for_full<I>(dpf),
+        make_output_buffer_for_full<Is>(dpf)...);
+
+    // moving `outbufs` is allowed as the `outbufs` are `std::vectors`
+    //   the underlying data remains on the heap
+    //   and thus the data the iterable refers to is still valid
+    auto iterable = eval_full<I, Is...>(dpf, outbufs, memoizer);
+    return std::make_pair(std::move(outbufs), std::move(iterable));
+}
+
+template <std::size_t I = 0,
+          std::size_t ...Is,
+          typename DpfKey,
+          std::enable_if_t<looks_like_dpf_key_v<DpfKey> && !is_multilevel_key_v<DpfKey>, bool> = true>
+HEDLEY_ALWAYS_INLINE
+auto eval_full(const DpfKey & dpf)
+{
+    using input_type = typename DpfKey::input_type;
+    return eval_full<I, Is...>(dpf,
+        dpf::make_basic_full_memoizer(dpf));
+}
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_EVAL_FULL_HPP__

include/dpf/eval_inner_product.hpp

@@ -0,0 +1,461 @@
+/// @file dpf/eval_inner_product.hpp
+/// @brief Full / interval DPF evaluation that reduces against a public
+///        weight vector instead of materializing the output.
+/// @details Same interior + batched exterior AES as `eval_interval`, but
+///          each packed leaf is multiply-accumulated into a scalar:
+///          additive outputs sum `DPF(x) * w[x]`, XOR outputs xor
+///          `DPF(x) & w[x]`. A prepared memoizer skips the interior walk
+///          so the tree can be expanded before the weights exist.
+
+#ifndef LIBDPF_INCLUDE_DPF_EVAL_INNER_PRODUCT_HPP__
+#define LIBDPF_INCLUDE_DPF_EVAL_INNER_PRODUCT_HPP__
+
+#include <array>
+#include <cstddef>
+#include <cstring>
+#include <limits>
+#include <stdexcept>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+
+#include "hedley/hedley.h"
+#include <portable-snippets/exact-int/exact-int.h>
+#include <simde/simde/x86/avx2.h>
+
+#include "dpf/dpf_key.hpp"
+#include "dpf/eval_interval.hpp"
+#include "dpf/eval_target.hpp"
+#include "dpf/leaf_node.hpp"
+#include "dpf/twiddle.hpp"
+#include "dpf/utils.hpp"
+#include "dpf/xor_wrapper.hpp"
+
+namespace dpf
+{
+
+namespace internal
+{
+
+template <typename T>
+struct is_xor_wrapper : std::false_type {};
+
+template <typename T>
+struct is_xor_wrapper<dpf::xor_wrapper<T>> : std::true_type {};
+
+template <typename T>
+inline constexpr bool is_xor_wrapper_v = is_xor_wrapper<T>::value;
+
+template <typename W>
+HEDLEY_ALWAYS_INLINE
+auto weight_as_u64(W && w, std::size_t i)
+{
+    return static_cast<psnip_uint64_t>(w[i]);
+}
+
+HEDLEY_ALWAYS_INLINE
+simde__m128i load_weight_pair_u64(psnip_uint64_t lo, psnip_uint64_t hi)
+{
+    return simde_mm_set_epi64x(static_cast<int64_t>(hi),
+                               static_cast<int64_t>(lo));
+}
+
+HEDLEY_ALWAYS_INLINE
+simde__m128i mullo_epi64x2(simde__m128i a, simde__m128i b)
+{
+#if defined(__AVX512DQ__) && defined(__AVX512VL__)
+    return _mm_mullo_epi64(a, b);
+#else
+    psnip_uint64_t av[2], bv[2];
+    std::memcpy(av, &a, sizeof(av));
+    std::memcpy(bv, &b, sizeof(bv));
+    av[0] *= bv[0];
+    av[1] *= bv[1];
+    simde__m128i r;
+    std::memcpy(&r, av, sizeof(r));
+    return r;
+#endif
+}
+
+template <typename NodeT,
+          typename OutputsTuple,
+          std::size_t ...Is>
+struct ip_prg_range
+{
+    static constexpr std::size_t pos_min
+        = const_min_size<block_offset_of_leaf_v<Is, NodeT, OutputsTuple>...>::value;
+    static constexpr std::size_t pos_end
+        = const_max_size<(block_offset_of_leaf_v<Is, NodeT, OutputsTuple>
+            + block_length_of_leaf_v<std::tuple_element_t<Is, OutputsTuple>, NodeT>)...>::value;
+    static constexpr std::size_t count = pos_end - pos_min;
+};
+
+template <typename OutputT>
+struct ip_accum
+{
+    using output_type = OutputT;
+    static constexpr bool xor_mode = is_xor_wrapper_v<OutputT>;
+    static constexpr bool simd64 = (sizeof(OutputT) == 8);
+
+    simde__m128i vacc = simde_mm_setzero_si128();
+    output_type scalar{};
+
+    template <typename LeafT, typename W>
+    HEDLEY_ALWAYS_INLINE
+    void mac(const LeafT & leaf, std::size_t base, std::size_t opl, W && w)
+    {
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+        if constexpr (simd64 && std::is_same_v<LeafT, simde__m128i>)
+HEDLEY_PRAGMA(GCC diagnostic pop)
+        {
+            if (HEDLEY_LIKELY(opl == 2))
+            {
+                simde__m128i ww = load_weight_pair_u64(
+                    weight_as_u64(w, base),
+                    weight_as_u64(w, base + 1));
+                if constexpr (xor_mode)
+                {
+                    vacc = simde_mm_xor_si128(vacc,
+                        simde_mm_and_si128(leaf, ww));
+                }
+                else
+                {
+                    vacc = simde_mm_add_epi64(vacc, mullo_epi64x2(leaf, ww));
+                }
+                return;
+            }
+        }
+
+        for (std::size_t p = 0; p < opl; ++p)
+        {
+            output_type val;
+            if constexpr (utils::is_packed_subbyte_v<output_type>)
+            {
+                val = extract_leaf<std::remove_cv_t<LeafT>, output_type>(leaf, p);
+            }
+            else
+            {
+                std::memcpy(&val,
+                    reinterpret_cast<const unsigned char *>(std::addressof(leaf))
+                        + p * sizeof(output_type),
+                    sizeof(val));
+            }
+            const auto wt = weight_as_u64(w, base + p);
+            if constexpr (xor_mode)
+            {
+                scalar = output_type{static_cast<typename output_type::value_type>(
+                    static_cast<psnip_uint64_t>(scalar)
+                    ^ (static_cast<psnip_uint64_t>(val) & wt))};
+            }
+            else if constexpr (utils::is_packed_subbyte_v<output_type>
+                && !std::is_same_v<output_type, dpf::bit>)
+            {
+                constexpr unsigned mask
+                    = (1u << utils::packed_lane_bits_v<output_type>) - 1u;
+                const auto wlane = static_cast<output_type>(
+                    static_cast<unsigned>(wt) & mask);
+                scalar = scalar + val * wlane;
+            }
+            else
+            {
+                scalar = static_cast<output_type>(
+                    static_cast<psnip_uint64_t>(scalar)
+                    + static_cast<psnip_uint64_t>(val) * wt);
+            }
+        }
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    output_type finish() const
+    {
+        if constexpr (simd64)
+        {
+            psnip_uint64_t lanes[2];
+            std::memcpy(lanes, &vacc, sizeof(lanes));
+            if constexpr (xor_mode)
+            {
+                return output_type{static_cast<typename output_type::value_type>(
+                    (lanes[0] ^ lanes[1])
+                    ^ static_cast<psnip_uint64_t>(scalar))};
+            }
+            else
+            {
+                return output_type{
+                    lanes[0] + lanes[1]
+                    + static_cast<psnip_uint64_t>(scalar)};
+            }
+        }
+        return scalar;
+    }
+};
+
+template <std::size_t ...Is,
+          typename DpfKey,
+          typename Weights,
+          typename IntervalMemoizer,
+          typename IntegralT,
+          std::size_t ...IIs>
+void eval_inner_product_exterior(const DpfKey & dpf, IntegralT from_node,
+    IntegralT to_node, Weights && weights, IntervalMemoizer && memoizer,
+    std::index_sequence<IIs...>,
+    std::tuple<ip_accum<typename DpfKey::concrete_output_type<Is>>...> & accs,
+    std::size_t start = 0)
+{
+    assert_not_wildcard_output<Is...>(dpf);
+    if (HEDLEY_UNLIKELY(to_node < from_node && to_node != IntegralT{0}))
+        throw std::runtime_error("to_node<from_node");
+
+    using node_type = typename DpfKey::exterior_node;
+    using outputs_tuple = typename DpfKey::concrete_outputs_tuple;
+    using range = ip_prg_range<node_type, outputs_tuple, Is...>;
+    constexpr std::size_t opl = DpfKey::outputs_per_leaf;
+
+    std::size_t nodes_in_interval = static_cast<std::size_t>(to_node - from_node);
+    // `to_node == 0` is the saturated exclusive end; the subtraction is the
+    // leaf count. A real inverted range is rejected above.
+    auto *nodes = memoizer[DpfKey::depth];
+    auto cws = std::make_tuple(std::get<Is>(dpf.leaf_nodes).get()...);
+
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    auto apply_masks = [&](std::size_t k, const node_type & node,
+        const node_type * HEDLEY_RESTRICT masks)
+    {
+        auto apply_output = [&](auto out_index, auto buf_index)
+        {
+            constexpr std::size_t out_i = decltype(out_index)::value;
+            constexpr std::size_t buf_i = decltype(buf_index)::value;
+            using output_type = typename DpfKey::concrete_output_type<out_i>;
+            using leaf_type = dpf::leaf_node_t<node_type, output_type>;
+            constexpr auto pos = block_offset_of_leaf_v<out_i, node_type, outputs_tuple>;
+            leaf_type mask;
+            std::memcpy(&mask, masks + (pos - range::pos_min), sizeof(leaf_type));
+            // Subtractive share: CW_if_t − mask so reconstruct(y0, y1) = y0 − y1 = β.
+            auto leaf = dpf::subtract_leaf<output_type>(
+                get_if_lo_bit(std::get<buf_i>(cws), node), mask);
+            std::get<buf_i>(accs).mac(leaf, k * opl, opl,
+                utils::get<buf_i>(weights));
+        };
+        (apply_output(std::integral_constant<std::size_t, Is>{},
+                      std::integral_constant<std::size_t, IIs>{}), ...);
+    };
+
+    std::size_t j = 0, k = start;
+    if constexpr (range::count == 2 && range::pos_min == 0)
+    {
+        for (; j + 4 <= nodes_in_interval; j += 4, k += 4)
+        {
+            alignas(node_type) node_type seeds[4];
+            alignas(node_type) node_type left[4];
+            alignas(node_type) node_type right[4];
+            DPF_UNROLL_LOOP
+            for (std::size_t t = 0; t < 4; ++t)
+            {
+                seeds[t] = utils::to_exterior_node<node_type>(
+                    unset_lo_2bits(nodes[j + t]));
+            }
+            DpfKey::exterior_prg::eval01_x4(seeds, left, right);
+            DPF_UNROLL_LOOP
+            for (std::size_t t = 0; t < 4; ++t)
+            {
+                node_type masks[2] = {left[t], right[t]};
+                apply_masks(k + t, nodes[j + t], masks);
+            }
+        }
+    }
+    else if constexpr (range::count == 1)
+    {
+        const auto pos = static_cast<psnip_uint32_t>(range::pos_min);
+        for (; j + 8 <= nodes_in_interval; j += 8, k += 8)
+        {
+            alignas(node_type) node_type seeds[8];
+            alignas(node_type) node_type masks[8];
+            DPF_UNROLL_LOOP
+            for (std::size_t t = 0; t < 8; ++t)
+            {
+                seeds[t] = utils::to_exterior_node<node_type>(
+                    unset_lo_2bits(nodes[j + t]));
+            }
+            DpfKey::exterior_prg::eval_x8(seeds, masks, pos);
+            DPF_UNROLL_LOOP
+            for (std::size_t t = 0; t < 8; ++t)
+            {
+                apply_masks(k + t, nodes[j + t], &masks[t]);
+            }
+        }
+        for (; j + 4 <= nodes_in_interval; j += 4, k += 4)
+        {
+            alignas(node_type) node_type seeds[4];
+            alignas(node_type) node_type masks[4];
+            DPF_UNROLL_LOOP
+            for (std::size_t t = 0; t < 4; ++t)
+            {
+                seeds[t] = utils::to_exterior_node<node_type>(
+                    unset_lo_2bits(nodes[j + t]));
+            }
+            DpfKey::exterior_prg::eval_x4(seeds, masks, pos);
+            DPF_UNROLL_LOOP
+            for (std::size_t t = 0; t < 4; ++t)
+            {
+                apply_masks(k + t, nodes[j + t], &masks[t]);
+            }
+        }
+    }
+
+    DPF_UNROLL_LOOP
+    for (; j < nodes_in_interval; ++j, ++k)
+    {
+        const auto & node = nodes[j];
+        auto seed = utils::to_exterior_node<node_type>(unset_lo_2bits(node));
+        std::array<node_type, range::count> masks;
+        DpfKey::exterior_prg::eval(seed, masks.data(),
+            static_cast<psnip_uint32_t>(range::count),
+            static_cast<psnip_uint32_t>(range::pos_min));
+        apply_masks(k, node, masks.data());
+    }
+HEDLEY_PRAGMA(GCC diagnostic pop)
+}
+
+template <typename DpfKey,
+          typename InputT,
+          typename IntervalMemoizer>
+void eval_prepare_nodes(const DpfKey & dpf, InputT from, InputT to,
+    IntervalMemoizer && memoizer)
+{
+    using dpf_type = DpfKey;
+    using integral_type = typename DpfKey::integral_type;
+
+    utils::flip_msb_if_signed_integral(from);
+    utils::flip_msb_if_signed_integral(to);
+
+    integral_type from_node = utils::get_from_node<dpf_type>(from);
+    integral_type to_node = utils::get_to_node<dpf_type>(to);
+    auto segs = utils::split_leaf_nodes(from_node, to_node, dpf.depth);
+    // The memoizer keeps one interval. A wrap is two intervals, and walking
+    // the first clobbers the second, so only a single segment can be cached.
+    if (segs.n == 1)
+    {
+        eval_interval_interior(dpf, segs.seg[0].from_node, segs.seg[0].to_node,
+            memoizer);
+    }
+}
+
+template <std::size_t ...Is,
+          typename DpfKey,
+          typename InputT,
+          typename Weights,
+          typename IntervalMemoizer,
+          std::size_t ...IIs>
+auto eval_inner_product_impl(const DpfKey & dpf, InputT from, InputT to,
+    Weights && weights, IntervalMemoizer && memoizer,
+    std::index_sequence<IIs...>)
+{
+    using dpf_type = DpfKey;
+    using integral_type = typename DpfKey::integral_type;
+
+    utils::flip_msb_if_signed_integral(from);
+    utils::flip_msb_if_signed_integral(to);
+
+    integral_type from_node = utils::get_from_node<dpf_type>(from);
+    integral_type to_node = utils::get_to_node<dpf_type>(to);
+    auto segs = utils::split_leaf_nodes(from_node, to_node, dpf.depth);
+
+    auto accs = std::make_tuple(
+        ip_accum<typename DpfKey::concrete_output_type<Is>>{}...);
+    auto idxs = std::index_sequence<IIs...>{};
+
+    std::size_t start = 0;
+    for (std::size_t s = 0; s < segs.n; ++s)
+    {
+        const auto & seg = segs.seg[s];
+        eval_interval_interior(dpf, seg.from_node, seg.to_node, memoizer);
+        eval_inner_product_exterior<Is...>(dpf, seg.from_node, seg.to_node,
+            weights, memoizer, idxs, accs, start);
+        start += seg.count;
+    }
+
+    if constexpr (sizeof...(Is) == 1)
+    {
+        return std::get<0>(accs).finish();
+    }
+    else
+    {
+        return std::make_tuple(std::get<IIs>(accs).finish()...);
+    }
+}
+
+}  // namespace internal
+
+/// Expand the interior tree for `[from, to]`. A wrapping interval is left
+/// cold: the memoizer holds one half, and walking the first half of the later
+/// inner product would clobber a cached second half. Safe to call before the
+/// weight vector exists; a subsequent inner-product on the same memoizer
+/// skips the interior AES when the interval did not wrap.
+template <typename DpfKey,
+          typename InputT,
+          typename IntervalMemoizer>
+HEDLEY_ALWAYS_INLINE
+void eval_prepare_interval(const DpfKey & dpf, InputT from, InputT to,
+    IntervalMemoizer && memoizer)
+{
+    internal::eval_prepare_nodes(dpf, dpf.offset_x(from), dpf.offset_x(to),
+        memoizer);
+}
+
+template <typename DpfKey,
+          typename IntervalMemoizer>
+HEDLEY_ALWAYS_INLINE
+void eval_prepare_full(const DpfKey & dpf, IntervalMemoizer && memoizer)
+{
+    using input_type = typename DpfKey::input_type;
+    eval_prepare_interval(dpf,
+        std::numeric_limits<input_type>::min(),
+        std::numeric_limits<input_type>::max(),
+        memoizer);
+}
+
+/// `sum_x DPF_I(x) * w[x]` (additive) or `xor_x DPF_I(x) & w[x]` (XOR).
+/// `w[j]` is the weight for the `j`-th output in the interval, matching
+/// `eval_interval`'s destination layout. Multiple `Is` take a tuple of
+/// weight ranges and return a tuple of accumulators; a single `I` takes
+/// one range and returns one accumulator.
+template <std::size_t I = 0,
+          std::size_t ...Is,
+          typename DpfKey,
+          typename InputT,
+          typename Weights,
+          typename IntervalMemoizer,
+          std::enable_if_t<looks_like_dpf_key_v<DpfKey>
+              && !is_multilevel_key_v<DpfKey>, bool> = true>
+HEDLEY_ALWAYS_INLINE
+auto eval_inner_product(const DpfKey & dpf, InputT from, InputT to,
+    Weights && weights, IntervalMemoizer && memoizer)
+{
+    assert_not_wildcard_output<I, Is...>(dpf);
+    return internal::eval_inner_product_impl<I, Is...>(
+        dpf, dpf.offset_x(from), dpf.offset_x(to),
+        weights, memoizer, std::make_index_sequence<1 + sizeof...(Is)>{});
+}
+
+template <std::size_t I = 0,
+          std::size_t ...Is,
+          typename DpfKey,
+          typename Weights,
+          typename IntervalMemoizer,
+          std::enable_if_t<looks_like_dpf_key_v<DpfKey>
+              && !is_multilevel_key_v<DpfKey>, bool> = true>
+HEDLEY_ALWAYS_INLINE
+auto eval_full_inner_product(const DpfKey & dpf, Weights && weights,
+    IntervalMemoizer && memoizer)
+{
+    using input_type = typename DpfKey::input_type;
+    return eval_inner_product<I, Is...>(dpf,
+        std::numeric_limits<input_type>::min(),
+        std::numeric_limits<input_type>::max(),
+        weights, memoizer);
+}
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_EVAL_INNER_PRODUCT_HPP__

include/dpf/eval_interval.hpp

@@ -0,0 +1,462 @@
+/// @file dpf/eval_interval.hpp
+/// @brief
+/// @details
+/// @author Ryan Henry <ryan.henry@ucalgary.ca>
+/// @author Christopher Jiang <christopher.jiang@ucalgary.ca>
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_EVAL_INTERVAL_HPP__
+#define LIBDPF_INCLUDE_DPF_EVAL_INTERVAL_HPP__
+
+#include <portable-snippets/builtin/builtin.h>
+#include <portable-snippets/exact-int/exact-int.h>
+#include "hedley/hedley.h"
+
+#include <cstddef>
+#include <cstring>
+#include <stdexcept>
+#include <array>
+#include <tuple>
+#include <type_traits>
+#include <iterator>
+#include <utility>
+
+#include "dpf/dpf_key.hpp"
+#include "dpf/eval_common.hpp"
+#include "dpf/eval_target.hpp"
+#include "dpf/output_buffer.hpp"
+#include "dpf/interval_memoizer.hpp"
+#include "dpf/subinterval_iterable.hpp"
+
+namespace dpf
+{
+
+namespace internal
+{
+
+template <typename DpfKey,
+          typename IntervalMemoizer,
+          typename IntegralT = typename DpfKey::integral_type>
+inline auto eval_interval_interior(const DpfKey & dpf, IntegralT from_node,
+    IntegralT to_node, IntervalMemoizer & memoizer,  // NOLINT(runtime/references)
+    std::size_t to_level = DpfKey::depth)
+{
+    using dpf_type = DpfKey;
+    using integral_type = typename DpfKey::integral_type;
+    using node_type = typename DpfKey::interior_node;
+
+    // level_index represents the current level being built
+    // level_index = 0 => root
+    // level_index = depth => last layer of interior nodes
+    std::size_t level_index = memoizer.assign_interval(dpf, from_node, to_node);
+    std::size_t nodes_at_level = memoizer.get_nodes_at_level();
+    integral_type mask = utils::get_node_mask<dpf_type>(dpf.msb_mask, level_index);
+
+    for (; level_index <= to_level; level_index = memoizer.advance_level(), nodes_at_level = memoizer.get_nodes_at_level(), mask>>=1)
+    {
+        std::size_t i = 0, j = 0;
+        bool from_offset = mask & from_node,
+             to_offset = from_offset ^ (nodes_at_level & 1);
+        const node_type cw[2] = {
+            dpf.correction_word(level_index-1, 0),
+            dpf.correction_word(level_index-1, 1)
+        };
+
+        auto *prev = memoizer[level_index-1];
+        auto *curr = memoizer[level_index];
+
+        // process node which only requires a right traversal
+        if (from_offset == true)
+        {
+            curr[i++] = dpf_type::traverse_interior(prev[j++], cw[1], 1);
+        }
+        // process all nodes which require both a left traversal and a right traversal
+        const std::size_t both_end = nodes_at_level - to_offset;
+        while (i + 8 <= both_end)
+        {
+            alignas(node_type) node_type parents[4];
+            alignas(node_type) node_type left[4];
+            alignas(node_type) node_type right[4];
+            DPF_UNROLL_LOOP
+            for (std::size_t t = 0; t < 4; ++t)
+            {
+                parents[t] = prev[j + t];
+            }
+            dpf_type::traverse_interior01_x4(parents, cw[0], cw[1], left, right);
+            DPF_UNROLL_LOOP
+            for (std::size_t t = 0; t < 4; ++t)
+            {
+                curr[i + 2 * t]     = left[t];
+                curr[i + 2 * t + 1] = right[t];
+            }
+            i += 8;
+            j += 4;
+        }
+        DPF_UNROLL_LOOP
+        for (; i < both_end;)
+        {
+            auto cur_node = prev[j++];
+            auto kids = dpf_type::traverse_interior01(cur_node, cw[0], cw[1]);
+            curr[i++] = kids[0];
+            curr[i++] = kids[1];
+        }
+        // process node which only requires a left traversal
+        if (to_offset == true)
+        {
+            curr[i] = dpf_type::traverse_interior(prev[j], cw[0], 0);
+        }
+    }
+}
+
+template <std::size_t I,
+          typename DpfKey,
+          typename OutputBuffer,
+          typename IntervalMemoizer,
+          typename IntegralT = typename DpfKey::integral_type>
+inline auto eval_interval_exterior(const DpfKey & dpf, IntegralT from_node,
+    IntegralT to_node, OutputBuffer && outbuf, IntervalMemoizer && memoizer,
+    std::size_t start = 0)
+{
+    assert_not_wildcard_output<I>(dpf);
+    if (HEDLEY_UNLIKELY(to_node < from_node && to_node != IntegralT{0}))
+        throw std::runtime_error("to_node<from_node");
+
+    using dpf_type = DpfKey;
+    using output_type = typename DpfKey::concrete_output_type<I>;
+
+    std::size_t nodes_in_interval = static_cast<std::size_t>(to_node - from_node);
+
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    auto cw = std::get<I>(dpf.leaf_nodes).get();
+    auto *nodes = memoizer[dpf_type::depth];
+    DPF_UNROLL_LOOP
+    for (std::size_t j = 0, k = start; j < nodes_in_interval; ++j, ++k)
+    {
+        auto leaf = dpf.template traverse_exterior<I>(nodes[j],
+            get_if_lo_bit(cw, nodes[j]));
+        if constexpr (utils::is_packed_subbyte_v<output_type>)
+        {
+            store_leaf_bytes(outbuf, k, leaf);
+        }
+        else
+        {
+            std::memcpy(&outbuf[k*dpf_type::outputs_per_leaf], &leaf,
+                sizeof(output_type) * dpf_type::outputs_per_leaf);
+        }
+    }
+HEDLEY_PRAGMA(GCC diagnostic pop)
+}
+
+template <std::size_t I,
+          typename DpfKey,
+          typename OutputBuffer,
+          typename LeafT>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+void store_interval_leaf(OutputBuffer && outbuf, std::size_t k, const LeafT & leaf) noexcept
+{
+    using dpf_type = DpfKey;
+    using output_type = typename DpfKey::concrete_output_type<I>;
+    if constexpr (utils::is_packed_subbyte_v<output_type>)
+    {
+        store_leaf_bytes(outbuf, k, leaf);
+    }
+    else
+    {
+        std::memcpy(&outbuf[k * dpf_type::outputs_per_leaf], &leaf,
+            sizeof(output_type) * dpf_type::outputs_per_leaf);
+    }
+}
+
+/// One pass over the leaf-level interior nodes. When the selected output
+/// indices occupy a contiguous PRG-position range, a single batched
+/// `ExteriorPRG::eval` produces every output's leaf mask.
+template <std::size_t ...Is,
+          typename DpfKey,
+          typename OutputBuffers,
+          typename IntervalMemoizer,
+          typename IntegralT,
+          std::size_t ...IIs>
+inline void eval_interval_exterior_fused(const DpfKey & dpf, IntegralT from_node,
+    IntegralT to_node, OutputBuffers && outbufs, IntervalMemoizer && memoizer,
+    std::index_sequence<IIs...>, std::size_t start = 0)
+{
+    assert_not_wildcard_output<Is...>(dpf);
+    if (HEDLEY_UNLIKELY(to_node < from_node && to_node != IntegralT{0}))
+        throw std::runtime_error("to_node<from_node");
+
+    using node_type = typename DpfKey::exterior_node;
+    using outputs_tuple = typename DpfKey::concrete_outputs_tuple;
+    using range = leaf_prg_range<node_type, outputs_tuple, Is...>;
+
+    std::size_t nodes_in_interval = static_cast<std::size_t>(to_node - from_node);
+    auto *nodes = memoizer[DpfKey::depth];
+    auto cws = std::make_tuple(std::get<Is>(dpf.leaf_nodes).get()...);
+
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    auto apply_masks = [&](std::size_t k, const node_type & node,
+        const node_type * HEDLEY_RESTRICT masks)
+    {
+        auto apply_output = [&](auto out_index, auto buf_index)
+        {
+            constexpr std::size_t out_i = decltype(out_index)::value;
+            constexpr std::size_t buf_i = decltype(buf_index)::value;
+            using output_type = typename DpfKey::concrete_output_type<out_i>;
+            using leaf_type = dpf::leaf_node_t<node_type, output_type>;
+            constexpr auto pos = block_offset_of_leaf_v<out_i, node_type, outputs_tuple>;
+            leaf_type mask;
+            std::memcpy(&mask, masks + (pos - range::pos_min), sizeof(leaf_type));
+            // Subtractive share: CW_if_t − mask so reconstruct(y0, y1) = y0 − y1 = β.
+            auto leaf = dpf::subtract_leaf<output_type>(
+                get_if_lo_bit(std::get<buf_i>(cws), node), mask);
+            store_interval_leaf<out_i, DpfKey>(utils::get<buf_i>(outbufs), k, leaf);
+        };
+        (apply_output(std::integral_constant<std::size_t, Is>{},
+                      std::integral_constant<std::size_t, IIs>{}), ...);
+    };
+
+    std::size_t j = 0, k = start;
+    if constexpr (range::count == 2 && range::pos_min == 0)
+    {
+        for (; j + 4 <= nodes_in_interval; j += 4, k += 4)
+        {
+            alignas(node_type) node_type seeds[4];
+            alignas(node_type) node_type left[4];
+            alignas(node_type) node_type right[4];
+            DPF_UNROLL_LOOP
+            for (std::size_t t = 0; t < 4; ++t)
+            {
+                seeds[t] = utils::to_exterior_node<node_type>(
+                    unset_lo_2bits(nodes[j + t]));
+            }
+            DpfKey::exterior_prg::eval01_x4(seeds, left, right);
+            DPF_UNROLL_LOOP
+            for (std::size_t t = 0; t < 4; ++t)
+            {
+                node_type masks[2] = {left[t], right[t]};
+                apply_masks(k + t, nodes[j + t], masks);
+            }
+        }
+    }
+    else if constexpr (range::count == 1)
+    {
+        const auto pos = static_cast<psnip_uint32_t>(range::pos_min);
+        for (; j + 8 <= nodes_in_interval; j += 8, k += 8)
+        {
+            alignas(node_type) node_type seeds[8];
+            alignas(node_type) node_type masks[8];
+            DPF_UNROLL_LOOP
+            for (std::size_t t = 0; t < 8; ++t)
+            {
+                seeds[t] = utils::to_exterior_node<node_type>(
+                    unset_lo_2bits(nodes[j + t]));
+            }
+            DpfKey::exterior_prg::eval_x8(seeds, masks, pos);
+            DPF_UNROLL_LOOP
+            for (std::size_t t = 0; t < 8; ++t)
+            {
+                apply_masks(k + t, nodes[j + t], &masks[t]);
+            }
+        }
+        for (; j + 4 <= nodes_in_interval; j += 4, k += 4)
+        {
+            alignas(node_type) node_type seeds[4];
+            alignas(node_type) node_type masks[4];
+            DPF_UNROLL_LOOP
+            for (std::size_t t = 0; t < 4; ++t)
+            {
+                seeds[t] = utils::to_exterior_node<node_type>(
+                    unset_lo_2bits(nodes[j + t]));
+            }
+            DpfKey::exterior_prg::eval_x4(seeds, masks, pos);
+            DPF_UNROLL_LOOP
+            for (std::size_t t = 0; t < 4; ++t)
+            {
+                apply_masks(k + t, nodes[j + t], &masks[t]);
+            }
+        }
+    }
+
+    DPF_UNROLL_LOOP
+    for (; j < nodes_in_interval; ++j, ++k)
+    {
+        const auto & node = nodes[j];
+        auto seed = utils::to_exterior_node<node_type>(unset_lo_2bits(node));
+        std::array<node_type, range::count> masks;
+        DpfKey::exterior_prg::eval(seed, masks.data(),
+            static_cast<psnip_uint32_t>(range::count),
+            static_cast<psnip_uint32_t>(range::pos_min));
+        apply_masks(k, node, masks.data());
+    }
+HEDLEY_PRAGMA(GCC diagnostic pop)
+}
+
+template <std::size_t ...Is,
+          typename DpfKey,
+          typename OutputBuffers,
+          typename IntervalMemoizer,
+          typename IntegralT,
+          std::size_t ...IIs>
+HEDLEY_ALWAYS_INLINE
+void eval_interval_exterior_all(const DpfKey & dpf, IntegralT from_node,
+    IntegralT to_node, OutputBuffers && outbufs, IntervalMemoizer && memoizer,
+    std::index_sequence<IIs...> idxs, std::size_t start = 0)
+{
+    using node_type = typename DpfKey::exterior_node;
+    using outputs_tuple = typename DpfKey::concrete_outputs_tuple;
+    using range = leaf_prg_range<node_type, outputs_tuple, Is...>;
+    if constexpr (range::is_contiguous)
+    {
+        eval_interval_exterior_fused<Is...>(dpf, from_node, to_node, outbufs,
+            memoizer, idxs, start);
+    }
+    else
+    {
+        (eval_interval_exterior<Is>(dpf, from_node, to_node,
+            utils::get<IIs>(outbufs), memoizer, start), ...);
+    }
+}
+
+template <std::size_t ...Is,
+          typename DpfKey,
+          typename InputT,
+          typename OutputBuffers,
+          typename IntervalMemoizer,
+          std::size_t ...IIs>
+auto eval_interval_impl(const DpfKey & dpf, InputT from, InputT to,
+    OutputBuffers && outbufs, IntervalMemoizer && memoizer,
+    std::index_sequence<IIs...>)
+{
+    using dpf_type = DpfKey;
+    using integral_type = typename DpfKey::integral_type;
+
+    utils::flip_msb_if_signed_integral(from);
+    utils::flip_msb_if_signed_integral(to);
+
+    integral_type from_node = utils::get_from_node<dpf_type>(from),
+        to_node = utils::get_to_node<dpf_type>(to);
+    auto segs = utils::split_leaf_nodes(from_node, to_node, dpf.depth);
+
+    auto idxs = std::index_sequence<IIs...>{};
+    std::size_t start = 0;
+    for (std::size_t s = 0; s < segs.n; ++s)
+    {
+        const auto & seg = segs.seg[s];
+        internal::eval_interval_interior(dpf, seg.from_node, seg.to_node, memoizer);
+        eval_interval_exterior_all<Is...>(dpf, seg.from_node, seg.to_node, outbufs,
+            memoizer, idxs, start);
+        start += seg.count;
+    }
+}
+
+template <std::size_t ...Is,
+          typename DpfKey,
+          typename InputT,
+          typename OutputBuffers,
+          typename IntervalMemoizer,
+          std::size_t ...IIs>
+auto eval_interval(const DpfKey & dpf, InputT from, InputT to,
+    OutputBuffers && outbufs, IntervalMemoizer && memoizer,
+    std::index_sequence<IIs...>)
+{
+    using dpf_type = DpfKey;
+    constexpr auto mod_pow_2 = utils::mod_pow_2<InputT>{};
+    constexpr auto to_integral_t = utils::to_integral_type<InputT>{};
+    constexpr auto bits = utils::bitlength_of_v<InputT>;
+
+    eval_interval_impl<Is...>(dpf, from, to, outbufs, memoizer, std::make_index_sequence<sizeof...(Is)>());
+
+    // `to_integral_type` widens to at least `size_t`. Subtracting in that
+    // wider type loses wrap-around of a narrower input domain (e.g. int16
+    // intervals that increment across 0). Mask back to the domain width so
+    // `subinterval_iterable` length matches the inclusive [from, to] walk.
+    auto from_i = to_integral_t(from);
+    auto span = to_integral_t(to) - from_i;
+    if constexpr (bits < utils::bitlength_of_v<decltype(span)>)
+    {
+        span &= (decltype(span){1} << bits) - 1;
+    }
+    auto from_sz = static_cast<std::size_t>(from_i);
+    auto to_sz = from_sz + static_cast<std::size_t>(span);
+
+    return utils::make_tuple(subinterval_iterable(std::begin(utils::get<IIs>(outbufs)), utils::size(utils::get<IIs>(outbufs)), from_sz, to_sz, mod_pow_2(from, dpf_type::lg_outputs_per_leaf), dpf_type::outputs_per_leaf)...);
+}
+
+}  // namespace internal
+
+template <std::size_t I = 0,
+          std::size_t ...Is,
+          typename DpfKey,
+          typename InputT,
+          typename OutputBuffers,
+          typename IntervalMemoizer = dpf::basic_interval_memoizer<DpfKey>,
+          std::enable_if_t<looks_like_dpf_key_v<DpfKey> && !is_multilevel_key_v<DpfKey>, bool> = true>
+HEDLEY_ALWAYS_INLINE
+auto eval_interval(const DpfKey & dpf, InputT from, InputT to,
+    OutputBuffers & outbufs, IntervalMemoizer && memoizer)  // NOLINT(runtime/references)
+{
+    assert_not_wildcard_output<I, Is...>(dpf);
+
+    return internal::eval_interval<I, Is...>(dpf, dpf.offset_x(from), dpf.offset_x(to), outbufs, memoizer, std::make_index_sequence<1+sizeof...(Is)>());
+}
+
+template <std::size_t I = 0,
+          std::size_t ...Is,
+          typename DpfKey,
+          typename InputT,
+          typename OutputBuffers,
+          std::enable_if_t<looks_like_dpf_key_v<DpfKey> && !is_multilevel_key_v<DpfKey>, bool> = true,
+          std::enable_if_t<!std::is_base_of_v<
+              dpf::interval_memoizer_base<unwrap_party_key_t<DpfKey>>,
+              std::decay_t<OutputBuffers>>, bool> = true>
+HEDLEY_ALWAYS_INLINE
+auto eval_interval(const DpfKey & dpf, InputT from, InputT to,
+    OutputBuffers & outbufs)  // NOLINT(runtime/references)
+{
+    return eval_interval<I, Is...>(dpf, from, to, outbufs,
+        dpf::make_basic_interval_memoizer<DpfKey>(from, to));
+}
+
+template <std::size_t I = 0,
+          std::size_t ...Is,
+          typename DpfKey,
+          typename InputT,
+          typename IntervalMemoizer,
+          std::enable_if_t<looks_like_dpf_key_v<DpfKey> && !is_multilevel_key_v<DpfKey>, bool> = true,
+          std::enable_if_t<std::is_base_of_v<
+              dpf::interval_memoizer_base<unwrap_party_key_t<DpfKey>>,
+              std::decay_t<IntervalMemoizer>>, bool> = true>
+HEDLEY_ALWAYS_INLINE
+auto eval_interval(const DpfKey & dpf, InputT from, InputT to,
+    IntervalMemoizer && memoizer)
+{
+    auto outbufs = utils::make_tuple(
+        make_output_buffer_for_interval<I>(dpf, from, to),
+        make_output_buffer_for_interval<Is>(dpf, from, to)...);
+
+    // moving `outbufs` is allowed as the `outbufs` are `std::vectors`
+    //   the underlying data remains on the heap
+    //   and thus the data the iterable refers to is still valid
+    auto iterable = eval_interval<I, Is...>(dpf, from, to, outbufs, memoizer);
+    return std::make_pair(std::move(outbufs), std::move(iterable));
+}
+
+template <std::size_t I = 0,
+          std::size_t ...Is,
+          typename DpfKey,
+          typename InputT,
+          std::enable_if_t<looks_like_dpf_key_v<DpfKey> && !is_multilevel_key_v<DpfKey>, bool> = true>
+HEDLEY_ALWAYS_INLINE
+auto eval_interval(const DpfKey & dpf, InputT from, InputT to)
+{
+    return eval_interval<I, Is...>(dpf, from, to,
+        dpf::make_basic_interval_memoizer<DpfKey>(from, to));
+}
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_EVAL_INTERVAL_HPP__

include/dpf/eval_point.hpp

@@ -0,0 +1,109 @@
+/// @file dpf/eval_point.hpp
+/// @brief
+/// @details
+/// @author Ryan Henry <ryan.henry@ucalgary.ca>
+/// @author Christopher Jiang <christopher.jiang@ucalgary.ca>
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_EVAL_POINT_HPP__
+#define LIBDPF_INCLUDE_DPF_EVAL_POINT_HPP__
+
+#include <portable-snippets/builtin/builtin.h>
+#include "hedley/hedley.h"
+
+#include <cstddef>
+#include <tuple>
+
+#include "dpf/dpf_key.hpp"
+#include "dpf/eval_common.hpp"
+#include "dpf/eval_target.hpp"
+#include "dpf/path_memoizer.hpp"
+
+namespace dpf
+{
+
+namespace internal
+{
+
+template <typename DpfKey,
+          typename InputT,
+          typename PathMemoizer>
+inline auto eval_point_interior(const DpfKey & dpf, InputT && x, PathMemoizer && path)
+{
+    using dpf_type = DpfKey;
+
+    auto level_index = detail::path_resume_for_level(path, dpf, x, dpf.depth);
+
+    DPF_UNROLL_LOOP
+    for (auto mask = dpf.msb_mask>>(level_index-1);
+        level_index <= dpf.depth; ++level_index, mask>>=1)
+    {
+        bool bit = !!(mask & x);
+        auto cw = dpf.correction_word(level_index-1, bit);
+        path[level_index] = dpf_type::traverse_interior(path[level_index-1], cw, bit);
+    }
+    detail::path_note_filled_to(path, dpf.depth);
+}
+
+template <std::size_t I,
+          typename DpfKey,
+          typename PathMemoizer>
+inline auto eval_point_exterior(const DpfKey & dpf, PathMemoizer && path)
+{
+    assert_not_wildcard_output<I>(dpf);
+
+    auto interior = path[dpf.depth];
+    return dpf.template traverse_exterior<I>(interior);
+}
+
+template <std::size_t I,
+          typename DpfKey,
+          typename InputT,
+          typename PathMemoizer>
+HEDLEY_ALWAYS_INLINE
+auto eval_point(const DpfKey & dpf, InputT && x, PathMemoizer && path)
+{
+    utils::flip_msb_if_signed_integral(x);
+    internal::eval_point_interior(dpf, x, path);
+    return internal::eval_point_exterior<I>(dpf, path);
+}
+
+}  // namespace internal
+
+template <std::size_t I = 0,
+          typename DpfKey,
+          typename InputT,
+          typename PathMemoizer = dpf::nonmemoizing_path_memoizer<DpfKey>,
+          std::enable_if_t<looks_like_dpf_key_v<DpfKey> && !is_multilevel_key_v<DpfKey>, bool> = true>
+HEDLEY_ALWAYS_INLINE
+auto eval_point(const DpfKey & dpf, InputT && x, PathMemoizer && path = PathMemoizer{})
+{
+    assert_not_wildcard_output<I>(dpf);
+    using output_type = typename DpfKey::concrete_output_type<I>;
+
+    auto tx = dpf.offset_x(x);
+    return make_eval_dpf_output<DpfKey, output_type>(
+        internal::eval_point<I>(dpf, tx, path), tx);
+}
+
+template <std::size_t I0,
+          std::size_t I1,
+          std::size_t ...Is,
+          typename DpfKey,
+          typename InputT,
+          typename PathMemoizer = dpf::basic_path_memoizer<DpfKey>,
+          std::enable_if_t<looks_like_dpf_key_v<DpfKey> && !is_multilevel_key_v<DpfKey>, bool> = true>
+HEDLEY_ALWAYS_INLINE
+auto eval_point(const DpfKey & dpf, InputT && x, PathMemoizer && path = PathMemoizer{})
+{
+    return std::make_tuple(
+        *eval_point<I0>(dpf, x, path),
+        *eval_point<I1>(dpf, x, path),
+        *eval_point<Is>(dpf, x, path)...);
+}
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_EVAL_POINT_HPP__

include/dpf/eval_sequence.hpp

@@ -0,0 +1,522 @@
+/// @file dpf/eval_sequence.hpp
+/// @brief
+/// @details
+/// @author Ryan Henry <ryan.henry@ucalgary.ca>
+/// @author Christopher Jiang <christopher.jiang@ucalgary.ca>
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_EVAL_SEQUENCE_HPP__
+#define LIBDPF_INCLUDE_DPF_EVAL_SEQUENCE_HPP__
+
+#include <portable-snippets/builtin/builtin.h>
+#include "hedley/hedley.h"
+
+#include <cstddef>
+#include <cstring>
+#include <type_traits>
+#include <utility>
+#include <tuple>
+#include <algorithm>
+#include <iterator>
+#include <stdexcept>
+#include <list>
+
+#include "dpf/dpf_key.hpp"
+#include "dpf/eval_common.hpp"
+#include "dpf/eval_target.hpp"
+#include "dpf/eval_point.hpp"
+#include "dpf/path_memoizer.hpp"
+#include "dpf/sequence_memoizer.hpp"
+#include "dpf/sequence_utils.hpp"
+#include "dpf/subsequence_iterable.hpp"
+#include "dpf/subinterval_iterable.hpp"
+
+namespace dpf
+{
+
+namespace internal
+{
+
+template <std::size_t ...Is,
+          typename DpfKey,
+          typename ForwardIterator,
+          typename OutputBuffers,
+          std::size_t ...IIs>
+auto eval_sequence_entire_node(const DpfKey & dpf, ForwardIterator begin, ForwardIterator end,
+    OutputBuffers && outbufs, std::index_sequence<IIs...>)
+{
+    static constexpr std::size_t outputs_per_leaf = DpfKey::outputs_per_leaf;
+
+    auto path = make_basic_path_memoizer(dpf);
+
+    std::size_t i = 0;
+    // DPF_UNROLL_LOOP
+    for (auto it = begin; it != end; ++it, ++i)
+    {
+        if constexpr(utils::is_packed_subbyte_v<typename DpfKey::concrete_output_type<0>>)
+        {
+            auto nodes = std::make_tuple(dpf::eval_point<Is>(dpf, *it, path).node...);
+            (store_leaf_bytes(utils::get<IIs>(outbufs), i, std::get<IIs>(nodes)), ...);
+        }
+        else
+        {
+            auto temp = std::make_tuple(dpf::eval_point<Is>(dpf, *it, path).node...);
+            (std::memcpy(&utils::get<IIs>(outbufs)[i*outputs_per_leaf], &utils::get<IIs>(temp), sizeof(typename DpfKey::concrete_output_type<Is>)*outputs_per_leaf), ...);
+        }
+    }
+    return utils::make_tuple(
+        dpf::subsequence_iterable<DpfKey, decltype(std::begin(utils::get<IIs>(outbufs))), ForwardIterator>(std::begin(utils::get<IIs>(outbufs)), begin, end)...);
+}
+
+template <typename Slot, typename Val>
+HEDLEY_ALWAYS_INLINE
+void assign_eval_slot(Slot && slot, Val && val)
+{
+    using val_t = std::decay_t<Val>;
+    if constexpr (is_secret_share_v<std::decay_t<Slot>>)
+    {
+        using elem_t = std::decay_t<Slot>;
+        if constexpr (is_secret_share_v<val_t>)
+            slot = elem_t::from_raw(val.raw());
+        else
+            slot = elem_t::from_raw(static_cast<typename elem_t::value_type>(val));
+    }
+    else if constexpr (is_secret_share_v<val_t>)
+    {
+        slot = val.raw();
+    }
+    else
+    {
+        slot = std::forward<Val>(val);
+    }
+}
+
+template <std::size_t ...Is,
+          typename DpfKey,
+          typename ForwardIterator,
+          typename OutputBuffers,
+          std::size_t ...IIs>
+auto eval_sequence_output_only(const DpfKey & dpf, ForwardIterator begin, ForwardIterator end, OutputBuffers && outbufs,
+    std::index_sequence<IIs...>)
+{
+    auto path = make_basic_path_memoizer(dpf);
+
+    std::size_t i = 0;
+    // DPF_UNROLL_LOOP
+    for (auto it = begin; it != end; ++it, ++i)
+    {
+        (assign_eval_slot(utils::get<IIs>(outbufs)[i],
+             *dpf::eval_point<Is>(dpf, *it, path)), ...);
+    }
+    if (i == 0)
+    {
+        return utils::make_tuple(subinterval_iterable(std::begin(utils::get<IIs>(outbufs)), utils::size(utils::get<IIs>(outbufs)), 0, 0, 0, 0, false)...);
+    }
+    return utils::make_tuple(subinterval_iterable(std::begin(utils::get<IIs>(outbufs)), utils::size(utils::get<IIs>(outbufs)), 0, i-1, 0, 0)...);
+}
+
+}  // namespace internal
+
+template <std::size_t I = 0,
+          std::size_t ...Is,
+          typename DpfKey,
+          typename ForwardIterator,
+          typename OutputBuffers,
+          typename ReturnType = return_entire_node_tag_,
+          std::enable_if_t<looks_like_dpf_key_v<DpfKey> && !is_multilevel_key_v<DpfKey>, bool> = true,
+          std::enable_if_t<std::is_base_of_v<return_type_tag_, ReturnType>, bool> = true,
+          std::enable_if_t<!std::is_base_of_v<return_type_tag_, OutputBuffers>, bool> = true>
+inline auto eval_sequence(const DpfKey & dpf, ForwardIterator begin, ForwardIterator end,
+    OutputBuffers && outbufs, ReturnType return_type = ReturnType{})
+{
+    static_assert(std::is_same_v<ReturnType, return_entire_node_tag_> ||
+                    std::is_same_v<ReturnType, return_output_only_tag_>);
+    if constexpr(std::is_same_v<ReturnType, return_entire_node_tag_>)
+    {
+        return internal::eval_sequence_entire_node<I, Is...>(dpf, begin, end, outbufs, std::make_index_sequence<1+sizeof...(Is)>{});
+    }
+    else
+    {
+        return internal::eval_sequence_output_only<I, Is...>(dpf, begin, end, outbufs, std::make_index_sequence<1+sizeof...(Is)>{});
+    }
+}
+
+template <std::size_t I = 0,
+          std::size_t ...Is,
+          typename DpfKey,
+          typename ForwardIterator,
+          typename ReturnType = return_entire_node_tag_,
+          std::enable_if_t<looks_like_dpf_key_v<DpfKey> && !is_multilevel_key_v<DpfKey>, bool> = true,
+          std::enable_if_t<std::is_base_of_v<return_type_tag_, ReturnType>, bool> = true>
+auto eval_sequence(const DpfKey & dpf, ForwardIterator begin, ForwardIterator end,
+    ReturnType return_type = ReturnType{})
+{
+    auto outbufs = utils::make_tuple(
+        make_output_buffer_for_subsequence<I>(dpf, begin, end, return_type),
+        make_output_buffer_for_subsequence<Is>(dpf, begin, end, return_type)...);
+
+    // moving `outbufs` is allowed as the `outbufs` are `std::vectors`
+    //   the underlying data remains on the heap
+    //   and thus the data the iterable refers to is still valid
+    auto iterable = eval_sequence<I, Is...>(dpf, begin, end, outbufs, return_type);
+    return std::make_pair(std::move(outbufs), std::move(iterable));
+}
+
+template <std::size_t I = 0,
+          typename DpfKey,
+          typename ForwardIterator,
+          typename OutputBuffer>
+inline auto eval_sequence_breadth_first(const DpfKey & dpf, ForwardIterator begin, ForwardIterator end, OutputBuffer && outbuf)
+{
+    assert_not_wildcard_output<I>(dpf);
+
+    using dpf_type = DpfKey;
+    using input_type = typename DpfKey::input_type;
+    using node_type = typename DpfKey::interior_node;
+    using output_type = typename DpfKey::concrete_output_type<I>;
+
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    using allocator = aligned_allocator<typename DpfKey::interior_node>;
+    using unique_ptr = typename allocator::unique_ptr;
+HEDLEY_PRAGMA(GCC diagnostic pop)
+    allocator alloc = allocator{};
+
+    if (HEDLEY_UNLIKELY(!std::is_sorted(begin, end)))
+    {
+        throw std::runtime_error("list must be sorted");
+    }
+    if (begin == end)
+    {
+        return subsequence_iterable<DpfKey, decltype(std::begin(outbuf)), ForwardIterator>(
+            std::begin(outbuf), begin, end);
+    }
+
+    auto mask = dpf_type::msb_mask;
+    std::size_t nodes_in_sequence = std::distance(begin, end);
+    unique_ptr memo{alloc.allocate_unique_ptr(nodes_in_sequence*2)};
+
+    bool curhalf = (dpf_type::depth ^ 1) & 1;
+    memo[!curhalf*nodes_in_sequence + 0] = dpf.root();
+
+    std::list<ForwardIterator> splits{begin, end};
+
+    std::size_t level_index = 1;
+    auto func = [&](const bool flip = false)
+    {
+        std::size_t i = 0, j = 0;
+        const node_type cw[2] = {
+            dpf.correction_word(level_index-1, 0),
+            dpf.correction_word(level_index-1, 1)
+        };
+        // `lower` and `upper` are always adjacent elements of `splits` with `lower` < `upper`
+        // [lower, upper) = "block"
+        for (auto upper = std::begin(splits), lower = upper++; upper != std::end(splits); lower = upper++)
+        {
+            // `upper_bound()` returns iterator to first element where the relevant bit (based on `mask`) is set
+            auto it = std::upper_bound(*lower, *upper, mask,
+                [&flip](auto a, auto b){ return static_cast<bool>(a&b) ^ flip; });
+            if (it == *lower)       // right only since first element in "block" requires right traversal
+            {
+                memo[curhalf*nodes_in_sequence + i++] = dpf_type::traverse_interior(memo[!curhalf*nodes_in_sequence + j++], cw[1], 1);
+            }
+            else if (it == *upper)  // left only since no element in "block" requires right traversal
+            {
+                memo[curhalf*nodes_in_sequence + i++] = dpf_type::traverse_interior(memo[!curhalf*nodes_in_sequence + j++], cw[0], 0);
+            }
+            else                    // both ways since some (non-lower) element within "block" requires right traversal
+            {
+                auto cur_node = memo[!curhalf*nodes_in_sequence + j++];
+                auto kids = dpf_type::traverse_interior01(cur_node, cw[0], cw[1]);
+                memo[curhalf*nodes_in_sequence + i++] = kids[0];
+                memo[curhalf*nodes_in_sequence + i++] = kids[1];
+                splits.insert(upper, it);
+            }
+        }
+    };
+    if (dpf_type::depth >= level_index)
+    {
+        func(utils::uses_signed_msb_v<input_type>);
+        ++level_index;
+        mask >>= 1;
+        curhalf =! curhalf;
+    }
+    for (; level_index <= dpf_type::depth; ++level_index, mask>>=1, curhalf=!curhalf)
+    {
+        func();
+    }
+
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    auto cw = dpf.template leaf<I>();
+    auto buf = memo.get();
+
+    constexpr auto clz = utils::countl_zero_symmetric_difference<input_type>{};
+    auto curr = begin, prev = curr;
+    for (std::size_t i = 0, j = 0; i < nodes_in_sequence; ++i)
+    {
+        j += (clz(*prev, *curr)) < dpf_type::depth;
+        auto leaf = dpf_type::template traverse_exterior<I>(buf[j],
+            get_if_lo_bit(cw, buf[j]));
+        if constexpr (utils::is_packed_subbyte_v<output_type>)
+        {
+            store_leaf_bytes(outbuf, i, leaf);
+        }
+        else
+        {
+            std::memcpy(&outbuf[i*dpf_type::outputs_per_leaf], &leaf, sizeof(output_type)*dpf_type::outputs_per_leaf);
+        }
+        prev = curr++;
+    }
+HEDLEY_PRAGMA(GCC diagnostic pop)
+
+   return subsequence_iterable<DpfKey, decltype(std::begin(outbuf)), ForwardIterator>(std::begin(outbuf), begin, end);
+}
+
+template <std::size_t I = 0,
+          typename DpfKey,
+          typename ForwardIterator>
+auto eval_sequence_breadth_first(const DpfKey & dpf, ForwardIterator begin, ForwardIterator end)
+{
+    auto outbuf = make_output_buffer_for_subsequence<I>(dpf, begin, end);
+
+    // moving `outbuf` is allowed as `outbuf` is a `std::vectors`
+    //   the underlying data remains on the heap
+    //   and thus the data the iterable refers to is still valid
+    auto iterable = eval_sequence_breadth_first<I>(dpf, begin, end, outbuf);
+    return std::make_pair(std::move(outbuf), std::move(iterable));
+}
+
+namespace internal
+{
+
+template <typename DpfKey,
+          typename SequenceMemoizer>
+inline auto eval_sequence_interior(const DpfKey & dpf, const sequence_recipe & recipe,
+    SequenceMemoizer && memoizer, std::size_t to_level = DpfKey::depth)
+{
+    using dpf_type = DpfKey;
+    using node_type = typename DpfKey::interior_node;
+
+    // level_index represents the current level being built
+    // level_index = 0 => root
+    // level_index = depth => last layer of interior nodes
+    if (recipe.num_leaf_nodes() == 0)
+        return;
+
+    std::size_t level_index = memoizer.assign_dpf(dpf, recipe);
+    std::size_t recipe_index = recipe.level_endpoints()[level_index-1];
+    std::size_t nodes_at_level = memoizer.get_nodes_at_level(level_index-1);
+
+    for (; level_index <= to_level; level_index = memoizer.advance_level(), nodes_at_level = memoizer.get_nodes_at_level(level_index-1))
+    {
+        const node_type cw[2] = {
+            dpf.correction_word(level_index-1, 0),
+            dpf.correction_word(level_index-1, 1)
+        };
+
+        auto prevbuf = memoizer[level_index-1];
+        auto currbuf = memoizer[level_index];
+
+        DPF_UNROLL_LOOP
+        for (std::size_t input_index = 0, output_index = 0; input_index < nodes_at_level; ++input_index, ++recipe_index)
+        {
+            if (memoizer.traverse_first(recipe_index) == true)
+            {
+                bool dir = memoizer.get_direction(0);
+                currbuf[output_index++] = dpf_type::traverse_interior(prevbuf[input_index], cw[dir], dir);
+            }
+            if (memoizer.traverse_second(recipe_index) == true)
+            {
+                bool dir = memoizer.get_direction(1);
+                currbuf[output_index++] = dpf_type::traverse_interior(prevbuf[input_index], cw[dir], dir);
+            }
+        }
+    }
+}
+
+template <std::size_t I,
+          typename DpfKey,
+          typename OutputBuffer,
+          typename SequenceMemoizer>
+inline auto eval_sequence_exterior_entire_node(const DpfKey & dpf, const sequence_recipe & recipe,
+    OutputBuffer && outbuf, SequenceMemoizer && memoizer)
+{
+    assert_not_wildcard_output<I>(dpf);
+
+    using dpf_type = DpfKey;
+    using output_type = typename DpfKey::concrete_output_type<I>;
+
+    auto nodes_in_interval = recipe.num_leaf_nodes();
+
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    auto buf = memoizer[dpf.depth];
+    DPF_UNROLL_LOOP
+    for (std::size_t j = 0; j < nodes_in_interval; ++j)
+    {
+        auto leaf = dpf.template traverse_exterior<I>(buf[j]);
+        if constexpr (utils::is_packed_subbyte_v<output_type>)
+        {
+            store_leaf_bytes(outbuf, j, leaf);
+        }
+        else
+        {
+            std::memcpy(&outbuf[j*dpf_type::outputs_per_leaf], &leaf, sizeof(output_type)*dpf_type::outputs_per_leaf);
+        }
+    }
+HEDLEY_PRAGMA(GCC diagnostic pop)
+}
+
+template <std::size_t I,
+          typename DpfKey,
+          typename OutputBuffer,
+          typename SequenceMemoizer>
+inline auto eval_sequence_exterior_output_only(const DpfKey & dpf, const sequence_recipe & recipe,
+    OutputBuffer && outbuf, SequenceMemoizer && memoizer)
+{
+    assert_not_wildcard_output<I>(dpf);
+
+    using dpf_type = DpfKey;
+    using output_type = typename DpfKey::concrete_output_type<I>;
+
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    auto cw = dpf.template leaf<I>();
+    using node_type = typename DpfKey::exterior_node;
+    using leaf_node_type = std::tuple_element_t<I, typename DpfKey::leaf_tuple>;
+    auto buf = memoizer[dpf.depth];
+
+    leaf_node_type node;
+    // DPF_UNROLL_LOOP
+    for (std::size_t i = 0, j = -1, prev = -1, curr;
+        i < recipe.output_indices().size();
+        prev = curr, ++i)
+    {
+        curr = recipe.output_indices()[i]/dpf_type::outputs_per_leaf;
+        if (prev != curr)
+        {
+            ++j;
+            node = dpf_type::template traverse_exterior<I>(buf[j], get_if_lo_bit(cw, buf[j]));
+        }
+        auto v = extract_leaf<node_type, output_type>(node,
+            recipe.output_indices()[i] % dpf_type::outputs_per_leaf);
+        using elem_t = std::decay_t<decltype(outbuf[i])>;
+        if constexpr (is_secret_share_v<elem_t>)
+            outbuf[i] = elem_t::from_raw(v);
+        else
+            outbuf[i] = v;
+    }
+HEDLEY_PRAGMA(GCC diagnostic pop)
+}
+
+template <std::size_t ...Is,
+          typename DpfKey,
+          typename OutputBuffers,
+          typename SequenceMemoizer,
+          typename ReturnType = return_entire_node_tag_,
+          std::enable_if_t<std::is_base_of_v<return_type_tag_, ReturnType>, bool> = true,
+          std::size_t ...IIs>
+auto eval_sequence(const DpfKey & dpf, const sequence_recipe & recipe,
+    OutputBuffers && outbufs, SequenceMemoizer && memoizer, ReturnType return_type, std::index_sequence<IIs...>)
+{
+    internal::eval_sequence_interior(dpf, recipe, memoizer);
+
+    static_assert(std::is_same_v<ReturnType, return_entire_node_tag_> ||
+                    std::is_same_v<ReturnType, return_output_only_tag_>);
+    if constexpr (std::is_same_v<ReturnType, return_entire_node_tag_>)
+    {
+        (internal::eval_sequence_exterior_entire_node<Is>(dpf, recipe, utils::get<IIs>(outbufs), memoizer), ...);
+        return utils::make_tuple(
+            recipe_subsequence_iterable(std::begin(utils::get<IIs>(outbufs)), recipe.output_indices())...);
+    }
+    else
+    {
+        (internal::eval_sequence_exterior_output_only<Is>(dpf, recipe, utils::get<IIs>(outbufs), memoizer), ...);
+        const auto nout = recipe.output_indices().size();
+        if (nout == 0)
+        {
+            return utils::make_tuple(subinterval_iterable(std::begin(utils::get<IIs>(outbufs)), utils::size(utils::get<IIs>(outbufs)), 0, 0, 0, 0, false)...);
+        }
+        return utils::make_tuple(subinterval_iterable(std::begin(utils::get<IIs>(outbufs)), utils::size(utils::get<IIs>(outbufs)), 0, nout-1, 0, 0)...);
+    }
+}
+
+}  // namespace internal
+
+template <std::size_t I = 0,
+          std::size_t ...Is,
+          typename DpfKey,
+          typename OutputBuffers,
+          typename SequenceMemoizer,
+          typename ReturnType = return_entire_node_tag_,
+          std::enable_if_t<!std::is_base_of_v<return_type_tag_, SequenceMemoizer>, bool> = true,
+          std::enable_if_t<std::is_base_of_v<return_type_tag_, ReturnType>, bool> = true>
+HEDLEY_ALWAYS_INLINE
+auto eval_sequence(const DpfKey & dpf, const sequence_recipe & recipe,
+    OutputBuffers & outbufs, SequenceMemoizer && memoizer,  // NOLINT(runtime/references)
+    ReturnType return_type = ReturnType{})
+{
+    assert_not_wildcard_output<I, Is...>(dpf);
+    assert_not_wildcard_input(dpf);
+
+    return internal::eval_sequence<I, Is...>(dpf, recipe, outbufs, memoizer, return_type, std::make_index_sequence<1+sizeof...(Is)>());
+}
+
+template <std::size_t I = 0,
+          std::size_t ...Is,
+          typename DpfKey,
+          typename OutputBuffers,
+          typename ReturnType = return_entire_node_tag_,
+          std::enable_if_t<!std::is_base_of_v<sequence_memoizer_tag_,
+              std::decay_t<OutputBuffers>>, bool> = true,
+          std::enable_if_t<std::is_base_of_v<return_type_tag_, ReturnType>, bool> = true>
+HEDLEY_ALWAYS_INLINE
+auto eval_sequence(const DpfKey & dpf, const sequence_recipe & recipe,
+    OutputBuffers & outbufs, ReturnType return_type = ReturnType{})  // NOLINT(runtime/references)
+{
+    return eval_sequence<I, Is...>(dpf, recipe, outbufs,
+        dpf::make_double_space_sequence_memoizer<DpfKey>(recipe), return_type);
+}
+
+template <std::size_t I = 0,
+          std::size_t ...Is,
+          typename DpfKey,
+          typename SequenceMemoizer,
+          typename ReturnType = return_entire_node_tag_,
+          std::enable_if_t<std::is_base_of_v<sequence_memoizer_tag_,
+              std::decay_t<SequenceMemoizer>>, bool> = true,
+          std::enable_if_t<std::is_base_of_v<return_type_tag_, ReturnType>, bool> = true>
+HEDLEY_ALWAYS_INLINE
+auto eval_sequence(const DpfKey & dpf, const sequence_recipe & recipe,
+    SequenceMemoizer && memoizer, ReturnType return_type = ReturnType{})
+{
+    auto outbufs = utils::make_tuple(
+        make_output_buffer_for_recipe_subsequence<I>(dpf, recipe, return_type),
+        make_output_buffer_for_recipe_subsequence<Is>(dpf, recipe, return_type)...);
+
+    // moving `outbufs` is allowed as the `outbufs` are `std::vectors`
+    //   the underlying data remains on the heap
+    //   and thus the data the iterable refers to is still valid
+    auto iterable = eval_sequence<I, Is...>(dpf, recipe, outbufs, memoizer, return_type);
+    return std::make_pair(std::move(outbufs), std::move(iterable));
+}
+
+template <std::size_t I = 0,
+          std::size_t ...Is,
+          typename DpfKey,
+          typename ReturnType = return_entire_node_tag_,
+          std::enable_if_t<std::is_base_of_v<return_type_tag_, ReturnType>, bool> = true>
+HEDLEY_ALWAYS_INLINE
+auto eval_sequence(const DpfKey & dpf, const sequence_recipe & recipe,
+     ReturnType return_type = ReturnType{})
+{
+    return eval_sequence<I, Is...>(dpf, recipe,
+       dpf::make_double_space_sequence_memoizer<DpfKey>(recipe), return_type);
+}
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_EVAL_SEQUENCE_HPP__

include/dpf/eval_target.hpp

@@ -0,0 +1,110 @@
+/// @file dpf/eval_target.hpp
+/// @brief Eval channel tags for the unified DPF / iDPF / DCF surface.
+/// @copyright Copyright (c) 2019-2026 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license.
+
+#ifndef LIBDPF_INCLUDE_DPF_EVAL_TARGET_HPP__
+#define LIBDPF_INCLUDE_DPF_EVAL_TARGET_HPP__
+
+#include <cstddef>
+#include <limits>
+#include <type_traits>
+
+namespace dpf
+{
+
+/// Sentinel: deduce point-slot prefix from the key (`meta[I].prefix`).
+inline constexpr std::size_t prefix_deduce =
+    std::numeric_limits<std::size_t>::max();
+
+/// Point-output channel: slot `I`, optional prefix check `N`.
+template <std::size_t I = 0, std::size_t N = prefix_deduce>
+struct out_t
+{
+    static constexpr std::size_t index = I;
+    static constexpr std::size_t prefix = N;
+    static constexpr bool prefix_fixed = (N != prefix_deduce);
+};
+
+template <std::size_t I = 0, std::size_t N = prefix_deduce>
+inline constexpr out_t<I, N> out{};
+
+/// Comparison (DCF) channel.
+struct cmp_t
+{
+};
+inline constexpr cmp_t cmp{};
+
+template <typename T>
+struct is_out : std::false_type
+{
+};
+template <std::size_t I, std::size_t N>
+struct is_out<out_t<I, N>> : std::true_type
+{
+};
+template <typename T>
+inline constexpr bool is_out_v = is_out<std::decay_t<T>>::value;
+
+template <typename T>
+struct is_cmp_target : std::bool_constant<std::is_same_v<std::decay_t<T>, cmp_t>>
+{
+};
+template <typename T>
+inline constexpr bool is_cmp_target_v = is_cmp_target<T>::value;
+
+/// True for channel tags that must not bind as the key in classic eval_*.
+template <typename T>
+inline constexpr bool is_eval_channel_tag_v =
+    is_out_v<T> || is_cmp_target_v<T>;
+
+template <typename T, typename = void>
+struct looks_like_dpf_key : std::false_type
+{
+};
+template <typename T>
+struct looks_like_dpf_key<T,
+    std::void_t<typename T::input_type, typename T::interior_node>>
+    : std::true_type
+{
+};
+template <typename T>
+inline constexpr bool looks_like_dpf_key_v =
+    looks_like_dpf_key<std::decay_t<T>>::value;
+
+template <typename T, typename = void>
+struct is_incremental_dpf_key : std::false_type
+{
+};
+template <typename T>
+struct is_incremental_dpf_key<T,
+    std::void_t<decltype(T::cmp_depth), decltype(T::meta),
+        decltype(T::deepest_output)>> : std::true_type
+{
+};
+template <typename T>
+inline constexpr bool is_incremental_dpf_key_v =
+    is_incremental_dpf_key<std::decay_t<T>>::value;
+
+/// True only for keys that must use the slot-aware (multi-level / comparison)
+/// eval path. Every key now carries a `slot_meta` table (so
+/// `is_incremental_dpf_key_v` is true for all keys), but classic single-level
+/// equal-width keys keep using the classic `eval_*` fast paths; they set
+/// `is_multilevel == false`. Multi-level (`at<N>`) and comparison keys set it
+/// to true.
+template <typename T, typename = void>
+struct is_multilevel_key : std::false_type
+{
+};
+template <typename T>
+struct is_multilevel_key<T, std::void_t<decltype(T::is_multilevel)>>
+    : std::bool_constant<T::is_multilevel>
+{
+};
+template <typename T>
+inline constexpr bool is_multilevel_key_v =
+    is_multilevel_key<std::decay_t<T>>::value;
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_EVAL_TARGET_HPP__

include/dpf/eval_unified.hpp

@@ -0,0 +1,642 @@
+/// @file dpf/eval_unified.hpp
+/// @brief Target-first eval surface for DPF / iDPF / DCF channels.
+/// @details `eval_*(out<I>, …)` and `eval_*(cmp, …)` are the public API.
+/// @copyright Copyright (c) 2019-2026 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license.
+
+#ifndef LIBDPF_INCLUDE_DPF_EVAL_UNIFIED_HPP__
+#define LIBDPF_INCLUDE_DPF_EVAL_UNIFIED_HPP__
+
+#include "hedley/hedley.h"
+
+#include <cstddef>
+#include <cstring>
+#include <algorithm>
+#include <iterator>
+#include <list>
+#include <stdexcept>
+#include <type_traits>
+#include <utility>
+
+#include <portable-snippets/exact-int/exact-int.h>
+
+#include "dpf/eval_target.hpp"
+#include "dpf/eval_point.hpp"
+#include "dpf/eval_interval.hpp"
+#include "dpf/eval_full.hpp"
+#include "dpf/eval_sequence.hpp"
+#include "dpf/sequence_recipe.hpp"
+#include "dpf/incremental.hpp"
+#include "dpf/path_memoizer.hpp"
+#include "dpf/interval_memoizer.hpp"
+#include "dpf/aligned_allocator.hpp"
+#include "dpf/leaf_node.hpp"
+
+namespace dpf
+{
+namespace detail
+{
+
+template <std::size_t I, std::size_t N, typename KeyT>
+constexpr std::size_t resolved_out_prefix() noexcept
+{
+    if constexpr (is_multilevel_key_v<KeyT>)
+    {
+        if constexpr (N != prefix_deduce)
+        {
+            static_assert(KeyT::meta[I].prefix == N,
+                "out<I,N>: N does not match key::meta[I].prefix");
+            return N;
+        }
+        else
+            return KeyT::meta[I].prefix;
+    }
+    else
+    {
+        (void)N;
+        return utils::bitlength_of_v<typename KeyT::input_type>;
+    }
+}
+
+}  // namespace detail
+
+// ---------------------------------------------------------------------------
+// eval_point(target, key, x [, path])
+// ---------------------------------------------------------------------------
+
+template <std::size_t I, std::size_t N, typename KeyT, typename QueryT,
+          typename PathMemoizer = nonmemoizing_path_memoizer<KeyT>>
+auto eval_point(out_t<I, N>, const KeyT & key, QueryT && x,
+    PathMemoizer && path = PathMemoizer{})
+{
+    if constexpr (is_multilevel_key_v<KeyT>)
+    {
+        constexpr auto pref = detail::resolved_out_prefix<I, N, KeyT>();
+        return detail::incr::eval_out_point_impl<pref, I>(key,
+            std::forward<QueryT>(x), std::forward<PathMemoizer>(path));
+    }
+    else
+    {
+        return eval_point<I>(key, std::forward<QueryT>(x),
+            std::forward<PathMemoizer>(path));
+    }
+}
+
+template <typename Beta = uint64_t, typename KeyT, typename QueryT,
+          typename PathMemoizer = basic_path_memoizer<KeyT>>
+auto eval_point(cmp_t, const KeyT & key, QueryT && x,
+    PathMemoizer && path = PathMemoizer{})
+{
+    return detail::incr::eval_cmp_point_impl<Beta>(key, std::forward<QueryT>(x),
+        std::forward<PathMemoizer>(path));
+}
+
+// ---------------------------------------------------------------------------
+// eval_interval(target, key, from, to [, buf [, memo]])
+// ---------------------------------------------------------------------------
+
+template <std::size_t I, std::size_t N, typename KeyT, typename LaneT,
+          typename OutputBuffer, typename IntervalMemoizer>
+auto eval_interval(out_t<I, N>, const KeyT & key, LaneT from, LaneT to,
+    OutputBuffer && outbuf, IntervalMemoizer && memo)
+{
+    if constexpr (is_multilevel_key_v<KeyT>)
+    {
+        constexpr auto pref = detail::resolved_out_prefix<I, N, KeyT>();
+        return detail::incr::eval_out_interval_impl<pref, I>(key, from, to,
+            std::forward<OutputBuffer>(outbuf),
+            std::forward<IntervalMemoizer>(memo));
+    }
+    else
+    {
+        return eval_interval<I>(key, from, to,
+            std::forward<OutputBuffer>(outbuf),
+            std::forward<IntervalMemoizer>(memo));
+    }
+}
+
+template <std::size_t I, std::size_t N, typename KeyT, typename LaneT,
+          typename OutputBuffer>
+auto eval_interval(out_t<I, N>, const KeyT & key, LaneT from, LaneT to,
+    OutputBuffer && outbuf)
+{
+    if constexpr (is_multilevel_key_v<KeyT>)
+    {
+        constexpr auto pref = detail::resolved_out_prefix<I, N, KeyT>();
+        return detail::incr::eval_out_interval_impl<pref, I>(key, from, to,
+            std::forward<OutputBuffer>(outbuf));
+    }
+    else
+    {
+        return eval_interval<I>(key, from, to,
+            std::forward<OutputBuffer>(outbuf));
+    }
+}
+
+template <std::size_t I, std::size_t N, typename KeyT, typename LaneT>
+auto eval_interval(out_t<I, N>, const KeyT & key, LaneT from, LaneT to)
+{
+    if constexpr (is_multilevel_key_v<KeyT>)
+    {
+        constexpr auto pref = detail::resolved_out_prefix<I, N, KeyT>();
+        return detail::incr::eval_out_interval_impl<pref, I>(key, from, to);
+    }
+    else
+    {
+        return eval_interval<I>(key, from, to);
+    }
+}
+
+template <typename Beta = uint64_t, typename KeyT, typename LaneT,
+          typename OutputBuffer>
+void eval_interval(cmp_t, const KeyT & key, LaneT from, LaneT to,
+    OutputBuffer && outbuf)
+{
+    detail::incr::eval_cmp_interval_impl<Beta>(key, from, to,
+        std::forward<OutputBuffer>(outbuf));
+}
+
+template <typename Beta = uint64_t, typename KeyT, typename LaneT,
+          typename OutputBuffer, typename IntervalMemoizer>
+void eval_interval(cmp_t, const KeyT & key, LaneT from, LaneT to,
+    OutputBuffer && outbuf, IntervalMemoizer && memo)
+{
+    detail::incr::eval_cmp_interval_impl<Beta>(key, from, to,
+        std::forward<OutputBuffer>(outbuf),
+        std::forward<IntervalMemoizer>(memo));
+}
+
+template <typename Beta = uint64_t, typename KeyT, typename LaneT>
+auto eval_interval(cmp_t, const KeyT & key, LaneT from, LaneT to)
+{
+    return detail::incr::eval_cmp_interval_impl<Beta>(key, from, to);
+}
+
+// ---------------------------------------------------------------------------
+// eval_full(target, key [, …])
+// ---------------------------------------------------------------------------
+
+template <std::size_t I, std::size_t N, typename KeyT,
+          typename OutputBuffer, typename IntervalMemoizer>
+auto eval_full(out_t<I, N>, const KeyT & key, OutputBuffer && outbuf,
+    IntervalMemoizer && memo)
+{
+    if constexpr (is_multilevel_key_v<KeyT>)
+    {
+        constexpr auto pref = detail::resolved_out_prefix<I, N, KeyT>();
+        return detail::incr::eval_out_full_impl<pref, I>(key,
+            std::forward<OutputBuffer>(outbuf),
+            std::forward<IntervalMemoizer>(memo));
+    }
+    else
+    {
+        return eval_full<I>(key, std::forward<OutputBuffer>(outbuf),
+            std::forward<IntervalMemoizer>(memo));
+    }
+}
+
+template <std::size_t I, std::size_t N, typename KeyT>
+auto eval_full(out_t<I, N>, const KeyT & key)
+{
+    if constexpr (is_multilevel_key_v<KeyT>)
+    {
+        constexpr auto pref = detail::resolved_out_prefix<I, N, KeyT>();
+        return detail::incr::eval_out_full_impl<pref, I>(key);
+    }
+    else
+    {
+        return eval_full<I>(key);
+    }
+}
+
+template <typename Beta = uint64_t, typename KeyT>
+auto eval_full(cmp_t, const KeyT & key)
+{
+    if (!key.has_cmp())
+        throw std::invalid_argument("eval_full(cmp): no comparison channel");
+    using lane_t = typename KeyT::integral_type;
+    const auto nbits = static_cast<std::size_t>(key.cmp().nbits);
+    const lane_t lo = 0;
+    const lane_t hi = (nbits >= 8 * sizeof(lane_t))
+        ? static_cast<lane_t>(~lane_t{0})
+        : static_cast<lane_t>((lane_t{1} << nbits) - 1);
+    return detail::incr::eval_cmp_interval_impl<Beta>(key, lo, hi);
+}
+
+// ---------------------------------------------------------------------------
+// eval_sequence(target, key, begin, end, buf [, path])
+// ---------------------------------------------------------------------------
+
+template <std::size_t I, std::size_t N, typename KeyT, typename ForwardIterator,
+          typename OutputBuffer,
+          typename PathMemoizer = basic_path_memoizer<KeyT>>
+auto eval_sequence(out_t<I, N>, const KeyT & key, ForwardIterator begin,
+    ForwardIterator end, OutputBuffer && outbuf,
+    PathMemoizer && path = PathMemoizer{})
+{
+    if constexpr (is_multilevel_key_v<KeyT>)
+    {
+        constexpr auto pref = detail::resolved_out_prefix<I, N, KeyT>();
+        return detail::incr::eval_out_sequence_impl<pref, I>(key, begin, end,
+            std::forward<OutputBuffer>(outbuf),
+            std::forward<PathMemoizer>(path));
+    }
+    else
+    {
+        return eval_sequence<I>(key, begin, end,
+            std::forward<OutputBuffer>(outbuf));
+    }
+}
+
+template <typename Beta = uint64_t, typename KeyT, typename ForwardIterator,
+          typename OutputBuffer,
+          typename PathMemoizer = basic_path_memoizer<KeyT>>
+void eval_sequence(cmp_t, const KeyT & key, ForwardIterator begin,
+    ForwardIterator end, OutputBuffer && outbuf,
+    PathMemoizer && path = PathMemoizer{})
+{
+    detail::incr::eval_cmp_sequence_impl<Beta>(key, begin, end,
+        std::forward<OutputBuffer>(outbuf),
+        std::forward<PathMemoizer>(path));
+}
+
+// ---------------------------------------------------------------------------
+// make_output_buffer(target, …)
+// ---------------------------------------------------------------------------
+
+template <typename Beta = uint64_t, typename KeyT>
+auto make_output_buffer(cmp_t, const KeyT & key, std::size_t n)
+{
+    return detail::incr::make_output_buffer_for_cmp_impl<Beta>(key, n);
+}
+
+template <typename Beta = uint64_t, typename KeyT, typename LaneT>
+auto make_output_buffer(cmp_t, const KeyT & key, LaneT from, LaneT to)
+{
+    return detail::incr::make_output_buffer_for_cmp_interval_impl<Beta>(
+        key, from, to);
+}
+
+template <std::size_t I, std::size_t N, typename KeyT, typename LaneT>
+auto make_output_buffer(out_t<I, N>, const KeyT & key, LaneT from, LaneT to)
+{
+    constexpr auto pref = detail::resolved_out_prefix<I, N, KeyT>();
+    return detail::incr::make_output_buffer_for_out_interval_impl<pref, I>(
+        key, from, to);
+}
+
+// ---------------------------------------------------------------------------
+// eval_inner_product(target, key, from, to, weights [, memo])
+//
+// Point-slot inner product: same interior walk as `eval_interval(out<I>, …)`
+// but each packed leaf is multiply-accumulated against a public weight vector
+// instead of being materialized. Additive outputs sum `DPF_I(x)·w[x]`; XOR
+// outputs (`bit` / `xor_wrapper`) xor `DPF_I(x) & w[x]`. Weights are indexed in
+// the slot's lane domain, matching `eval_interval`'s destination layout.
+//
+// Cmp inner product: dot of the per-point comparison path-sum shares with the
+// weights (no leaf MAC); the two parties' results reconstruct to the true dot.
+// ---------------------------------------------------------------------------
+
+namespace detail
+{
+namespace incr
+{
+
+template <typename OutputT, typename NodeT>
+struct ml_ip_accum
+{
+    static constexpr bool xor_mode =
+        std::is_same_v<OutputT, dpf::bit> || utils::is_xor_wrapper_v<OutputT>;
+    psnip_uint64_t acc = 0;
+
+    template <typename LeafT, typename W>
+    void mac(const LeafT & leaf, std::size_t base, std::size_t opl, W && w)
+    {
+        for (std::size_t p = 0; p < opl; ++p)
+        {
+            psnip_uint64_t val;
+            if constexpr (utils::is_packed_subbyte_v<OutputT>)
+            {
+                val = static_cast<psnip_uint64_t>(
+                    dpf::extract_leaf<NodeT, OutputT>(leaf, p));
+            }
+            else
+            {
+                OutputT v;
+                std::memcpy(&v,
+                    reinterpret_cast<const unsigned char *>(std::addressof(leaf))
+                        + p * sizeof(OutputT),
+                    sizeof(v));
+                if constexpr (utils::is_xor_wrapper_v<OutputT>)
+                {
+                    // `static_cast<psnip_uint64_t>(v)` is ambiguous for
+                    // `xor_wrapper` (both `operator bool` and `operator T`
+                    // are viable). Go through the concrete underlying bits.
+                    val = static_cast<psnip_uint64_t>(v.data());
+                }
+                else
+                {
+                    val = static_cast<psnip_uint64_t>(v);
+                }
+            }
+            const auto wt = static_cast<psnip_uint64_t>(w[base + p]);
+            if constexpr (xor_mode)
+                acc ^= (val & wt);
+            else if constexpr (utils::is_packed_subbyte_v<OutputT>)
+            {
+                constexpr auto mask
+                    = (static_cast<psnip_uint64_t>(1)
+                          << utils::packed_lane_bits_v<OutputT>)
+                    - 1;
+                acc = (acc + (val & mask) * (wt & mask)) & mask;
+            }
+            else
+                acc += val * wt;
+        }
+    }
+
+    OutputT finish() const
+    {
+        if constexpr (std::is_same_v<OutputT, dpf::bit>)
+            return OutputT{static_cast<bool>(acc & 1)};
+        else if constexpr (utils::is_packed_subbyte_v<OutputT>)
+            return static_cast<OutputT>(acc);
+        else if constexpr (utils::is_xor_wrapper_v<OutputT>)
+            return OutputT{static_cast<typename OutputT::value_type>(acc)};
+        else
+            return static_cast<OutputT>(acc);
+    }
+};
+
+template <std::size_t N, std::size_t I, typename KeyT, typename LaneT,
+          typename Weights, typename IntervalMemoizer>
+auto eval_out_inner_product_impl(const KeyT & dpf, LaneT from, LaneT to,
+    Weights && weights, IntervalMemoizer && memoizer)
+{
+    using key_type = KeyT;
+    static_assert(key_type::meta[I].prefix == N,
+        "out inner product: N does not match output I");
+    using output_type = typename key_type::template concrete_output_type<I>;
+    using exterior_node = typename key_type::exterior_node;
+    using integral_type = typename key_type::integral_type;
+    constexpr auto opl = key_type::template outputs_per_leaf_of<I>;
+    constexpr auto lg_opl = key_type::template lg_outputs_per_leaf_of<I>;
+    constexpr auto to_level = key_type::meta[I].tree_level;
+    constexpr auto to_int = utils::to_integral_type<LaneT>{};
+
+    utils::flip_msb_if_signed_integral(from);
+    utils::flip_msb_if_signed_integral(to);
+
+    integral_type from_node = utils::leaf_node_floor(
+        static_cast<integral_type>(to_int(from)), lg_opl);
+    integral_type to_node = utils::leaf_node_ceil_exclusive(
+        static_cast<integral_type>(to_int(to)), lg_opl);
+    const auto segs = utils::split_leaf_nodes(from_node, to_node, to_level);
+
+    ml_ip_accum<output_type, exterior_node> acc{};
+    std::size_t start = 0;
+    for (std::size_t s = 0; s < segs.n; ++s)
+    {
+        const auto & seg = segs.seg[s];
+        internal::eval_out_interval_interior<N, I>(dpf, seg.from_node,
+            seg.to_node, memoizer);
+        auto * nodes = memoizer[to_level];
+        const std::size_t count =
+            static_cast<std::size_t>(seg.to_node - seg.from_node);
+        for (std::size_t j = 0; j < count; ++j)
+        {
+            auto leaf = dpf.template traverse_exterior<I>(nodes[j]);
+            acc.mac(leaf, (start + j) * opl, opl, weights);
+        }
+        start += seg.count;
+    }
+    return acc.finish();
+}
+
+template <typename Beta = uint64_t, typename KeyT, typename LaneT,
+          typename Weights>
+Beta eval_cmp_inner_product_impl(const KeyT & dpf, LaneT from, LaneT to,
+    Weights && weights)
+{
+    if (!dpf.has_cmp())
+        throw std::invalid_argument("cmp inner product: no comparison channel");
+    if (!dpf.cmp_assigned())
+        throw std::invalid_argument(
+            "cmp inner product: wildcard payload not assigned (call assign_cmp)");
+    constexpr auto to_int = utils::to_integral_type<LaneT>{};
+    utils::flip_msb_if_signed_integral(from);
+    utils::flip_msb_if_signed_integral(to);
+    const auto nbits = static_cast<std::size_t>(dpf.cmp().nbits);
+    const uint64_t mask = dpf.cmp().mask;
+    using integral = typename KeyT::integral_type;
+    const auto a = static_cast<integral>(to_int(from));
+    const auto b = static_cast<integral>(to_int(to));
+    const auto count = cmp_inclusive_count(a, b);
+
+    constexpr std::size_t stop =
+        KeyT::cmp_depth == 0 ? KeyT::depth : KeyT::cmp_depth;
+    detail::incr::cmp_full_interval_memo<KeyT, stop> memo{count};
+    detail::incr::eval_cmp_interval_impl_interior(dpf, a, cmp_exclusive_end(b),
+        nbits, memo);
+
+    uint64_t dot = 0;
+    for (std::size_t i = 0; i < count; ++i)
+    {
+        const auto q = static_cast<integral>(a + static_cast<integral>(i));
+        const uint64_t raw =
+            detail::incr::eval_cmp_from_interval_memo(dpf, q, a, nbits, memo);
+        const uint64_t wt = static_cast<uint64_t>(weights[i]) & mask;
+        dot = (dot + ((raw & mask) * wt)) & mask;
+    }
+    return detail::dcf_impl::u64_to_beta<Beta>(dot);
+}
+
+}  // namespace incr
+}  // namespace detail
+
+template <std::size_t I, std::size_t N, typename KeyT, typename LaneT,
+          typename Weights, typename IntervalMemoizer,
+          std::enable_if_t<is_multilevel_key_v<KeyT>, bool> = true>
+auto eval_inner_product(out_t<I, N>, const KeyT & key, LaneT from, LaneT to,
+    Weights && weights, IntervalMemoizer && memo)
+{
+    constexpr auto pref = detail::resolved_out_prefix<I, N, KeyT>();
+    return detail::incr::eval_out_inner_product_impl<pref, I>(key, from, to,
+        std::forward<Weights>(weights),
+        std::forward<IntervalMemoizer>(memo));
+}
+
+template <std::size_t I, std::size_t N, typename KeyT, typename LaneT,
+          typename Weights,
+          std::enable_if_t<is_multilevel_key_v<KeyT>, bool> = true>
+auto eval_inner_product(out_t<I, N>, const KeyT & key, LaneT from, LaneT to,
+    Weights && weights)
+{
+    auto memo = make_basic_interval_memoizer<KeyT, I>(from, to);
+    constexpr auto pref = detail::resolved_out_prefix<I, N, KeyT>();
+    return detail::incr::eval_out_inner_product_impl<pref, I>(key, from, to,
+        std::forward<Weights>(weights), memo);
+}
+
+template <typename Beta = uint64_t, typename KeyT, typename LaneT,
+          typename Weights>
+Beta eval_inner_product(cmp_t, const KeyT & key, LaneT from, LaneT to,
+    Weights && weights)
+{
+    return detail::incr::eval_cmp_inner_product_impl<Beta>(key, from, to,
+        std::forward<Weights>(weights));
+}
+
+// ---------------------------------------------------------------------------
+// eval_sequence_breadth_first(out<I>, key, begin, end [, outbuf])
+//
+// Breadth-first sequence eval that stops the interior walk at `meta[I]
+// .tree_level` (the leaf level of slot `I`) instead of the full key depth.
+// `begin`/`end` are a *sorted* range of lane points in `[0, 2^N)` (top-N-bit
+// prefixes); the result is written output-only, one value per query point in
+// query order (`outbuf[i]` is the output for the `i`-th query).
+// ---------------------------------------------------------------------------
+
+namespace detail
+{
+namespace incr
+{
+
+template <std::size_t N, std::size_t I, typename KeyT,
+          typename ForwardIterator, typename OutputBuffer>
+void eval_out_sequence_breadth_first_impl(const KeyT & dpf,
+    ForwardIterator begin, ForwardIterator end, OutputBuffer && outbuf)
+{
+    using key_type = KeyT;
+    static_assert(key_type::meta[I].prefix == N,
+        "breadth-first out sequence: N does not match output I");
+    using input_type = typename key_type::input_type;
+    using node_type = typename key_type::interior_node;
+    using exterior_node = typename key_type::exterior_node;
+    using output_type = typename key_type::template concrete_output_type<I>;
+    constexpr std::size_t stop = key_type::meta[I].tree_level;
+    constexpr std::size_t lg_opl = key_type::template lg_outputs_per_leaf_of<I>;
+    constexpr std::size_t opl = std::size_t{1} << lg_opl;
+
+    if (HEDLEY_UNLIKELY(!std::is_sorted(begin, end)))
+        throw std::runtime_error("breadth-first sequence: list must be sorted");
+    if (begin == end)
+        return;
+
+    using allocator = aligned_allocator<node_type>;
+    allocator alloc{};
+    const std::size_t nseq = static_cast<std::size_t>(std::distance(begin, end));
+    auto memo = alloc.allocate_unique_ptr(nseq * 2);
+    if (HEDLEY_UNLIKELY(memo == nullptr))
+        throw std::bad_alloc{};
+
+    input_type mask = static_cast<input_type>(input_type{1} << (N - 1));
+    bool curhalf = (stop ^ 1) & 1;
+    memo[static_cast<std::size_t>(!curhalf) * nseq + 0] = dpf.root();
+
+    std::list<ForwardIterator> splits{begin, end};
+    std::size_t level_index = 1;
+    auto step = [&]() {
+        std::size_t i = 0, j = 0;
+        const node_type cw[2] = {
+            dpf.correction_word(level_index - 1, 0),
+            dpf.correction_word(level_index - 1, 1)};
+        const std::size_t cur = static_cast<std::size_t>(curhalf) * nseq;
+        const std::size_t prv = static_cast<std::size_t>(!curhalf) * nseq;
+        for (auto upper = std::begin(splits), lower = upper++;
+             upper != std::end(splits); lower = upper++)
+        {
+            auto it = std::upper_bound(*lower, *upper, mask,
+                [](auto a, auto b) { return static_cast<bool>(a & b); });
+            if (it == *lower)
+            {
+                memo[cur + i++] = key_type::traverse_interior(
+                    memo[prv + j++], cw[1], 1);
+            }
+            else if (it == *upper)
+            {
+                memo[cur + i++] = key_type::traverse_interior(
+                    memo[prv + j++], cw[0], 0);
+            }
+            else
+            {
+                auto kids = key_type::traverse_interior01(memo[prv + j++],
+                    cw[0], cw[1]);
+                memo[cur + i++] = kids[0];
+                memo[cur + i++] = kids[1];
+                splits.insert(upper, it);
+            }
+        }
+    };
+    for (; level_index <= stop;
+         ++level_index, mask >>= 1, curhalf = !curhalf)
+        step();
+
+    auto * buf = memo.get();  // deepest built level (stop) lands in half 0
+    auto curr = begin, prev = begin;
+    std::size_t j = 0;
+    for (std::size_t i = 0; i < nseq; ++i)
+    {
+        if (i > 0
+            && (static_cast<input_type>(*curr) >> lg_opl)
+                   != (static_cast<input_type>(*prev) >> lg_opl))
+            ++j;
+        auto leaf = dpf.template traverse_exterior<I>(buf[j]);
+        const std::size_t off =
+            static_cast<std::size_t>(static_cast<input_type>(*curr) & (opl - 1));
+        auto v = dpf::extract_leaf<exterior_node, output_type>(leaf, off);
+        if constexpr (is_party_key_v<KeyT>)
+            outbuf[i] = subtractive_share<output_type, party_of_v<KeyT>>::from_raw(v);
+        else
+            outbuf[i] = v;
+        prev = curr++;
+    }
+}
+
+}  // namespace incr
+}  // namespace detail
+
+template <std::size_t I, std::size_t N, typename KeyT,
+          typename ForwardIterator, typename OutputBuffer,
+          std::enable_if_t<is_multilevel_key_v<KeyT>, bool> = true>
+void eval_sequence_breadth_first(out_t<I, N>, const KeyT & key,
+    ForwardIterator begin, ForwardIterator end, OutputBuffer && outbuf)
+{
+    constexpr auto pref = detail::resolved_out_prefix<I, N, KeyT>();
+    detail::incr::eval_out_sequence_breadth_first_impl<pref, I>(key, begin, end,
+        std::forward<OutputBuffer>(outbuf));
+}
+
+template <std::size_t I, std::size_t N, typename KeyT,
+          typename ForwardIterator,
+          std::enable_if_t<is_multilevel_key_v<KeyT>, bool> = true>
+auto eval_sequence_breadth_first(out_t<I, N>, const KeyT & key,
+    ForwardIterator begin, ForwardIterator end)
+{
+    using output_type = typename KeyT::template concrete_output_type<I>;
+    const std::size_t n = static_cast<std::size_t>(std::distance(begin, end));
+    dpf::output_buffer<leaf_buffer_elem_t<KeyT, output_type>> buf(n);
+    eval_sequence_breadth_first(out_t<I, N>{}, key, begin, end, buf);
+    return buf;
+}
+
+/// Build a sequence recipe stopped at slot `I`'s tree level (prefix domain).
+template <std::size_t I, std::size_t N, typename KeyT, typename ForwardIterator,
+          std::enable_if_t<is_multilevel_key_v<KeyT>, bool> = true>
+auto make_sequence_recipe(out_t<I, N>, const KeyT & key, ForwardIterator begin,
+    ForwardIterator end)
+{
+    constexpr auto pref = detail::resolved_out_prefix<I, N, KeyT>();
+    using input_type = typename KeyT::input_type;
+    constexpr auto stop = KeyT::meta[I].tree_level;
+    constexpr auto lg = KeyT::template lg_outputs_per_leaf_of<I>;
+    const input_type lane_msb =
+        static_cast<input_type>(input_type{1} << (pref - 1));
+    (void)key;
+    return make_sequence_recipe_at<stop, lg, input_type>(lane_msb, begin, end);
+}
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_EVAL_UNIFIED_HPP__

include/dpf/geneval.hpp

@@ -0,0 +1,638 @@
+/// @file dpf/geneval.hpp
+/// @brief Fused generation and evaluation (Doerner–Shelat on the eval trie).
+/// @details `make_dpf` / `make_dpf_doerner_shelat` build a reusable key, then
+///          `eval_*` walks it. `geneval_*` does both at once: one correction
+///          word per level, opened from the XOR-reduction of the nodes the
+///          public query actually expands. While the secret path's parent is
+///          still in that trie the word matches the reusable key byte for
+///          byte (same roots, same Beaver tape). After the path leaves, the
+///          word is uniform and later outputs still reconstruct — off-path
+///          nodes are identical across the two parties, so a dummy word
+///          cancels.
+///
+///          A wildcard-input call takes additive shares of the real point and
+///          a public query. It samples a random target, runs geneval there,
+///          and shifts the query by `target - x`, which is what
+///          `offset_x` does after a wildcard key is bound to `x`.
+/// @copyright Copyright (c) 2019-2026 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_GENEVAL_HPP__
+#define LIBDPF_INCLUDE_DPF_GENEVAL_HPP__
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <iterator>
+#include <stdexcept>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+#include "hedley/hedley.h"
+#include "simde/simde/x86/avx2.h"
+
+#include "dpf/aligned_allocator.hpp"
+#include "dpf/doerner_shelat.hpp"
+#include "dpf/leaf_node.hpp"
+
+namespace dpf
+{
+
+/// Tag for a geneval whose point is known only as additive shares.
+struct wildcard_input_t
+{
+};
+
+inline constexpr wildcard_input_t wildcard_input{};
+
+/// Shares and the correction words opened along the query trie.
+/// `correction_words[i]` / `correction_advice[i]` match a reusable key at
+/// the same target for every `i < live_levels`. `leaf_live` means the
+/// target's leaf was in the trie, so `leaf` is that key's leaf word.
+template <typename Output, typename Leaf>
+struct geneval_result
+{
+    std::vector<Output> party0;
+    std::vector<Output> party1;
+    std::vector<simde__m128i, aligned_allocator<simde__m128i>> correction_words;
+    std::vector<uint8_t> correction_advice;
+    std::size_t live_levels = 0;
+    bool leaf_live = false;
+    Leaf leaf{};
+};
+
+namespace detail
+{
+
+template <typename T>
+HEDLEY_ALWAYS_INLINE
+T geneval_mod_add(T a, T b) noexcept
+{
+    using U = std::make_unsigned_t<T>;
+    U sum = static_cast<U>(static_cast<U>(a) + static_cast<U>(b));
+    T out;
+    std::memcpy(&out, &sum, sizeof(out));
+    return out;
+}
+
+template <typename T>
+HEDLEY_ALWAYS_INLINE
+T geneval_mod_sub(T a, T b) noexcept
+{
+    using U = std::make_unsigned_t<T>;
+    U diff = static_cast<U>(static_cast<U>(a) - static_cast<U>(b));
+    T out;
+    std::memcpy(&out, &diff, sizeof(out));
+    return out;
+}
+
+template <typename T>
+T geneval_flipped(T x)
+{
+    utils::flip_msb_if_signed_integral(x);
+    return x;
+}
+
+/// Leaf-node id of an already MSB-flipped input. The id is the high
+/// `depth` bits; the low `lg(outputs_per_leaf)` bits select the lane.
+template <typename Dpf>
+uint64_t geneval_leaf_id(typename Dpf::input_type x)
+{
+    return static_cast<uint64_t>(utils::get_from_node<Dpf>(x));
+}
+
+inline uint64_t geneval_prefix(uint64_t leaf, std::size_t depth, std::size_t bits)
+{
+    if (bits == 0)
+        return 0;
+    if (bits >= depth)
+        return leaf;
+    return leaf >> (depth - bits);
+}
+
+inline bool geneval_any_prefix(const std::vector<uint64_t> & leaves,
+    std::size_t depth, uint64_t id, std::size_t bits)
+{
+    if (leaves.empty())
+        return false;
+    if (bits == 0)
+        return true;
+    const std::size_t sh = depth - bits;
+    const uint64_t lo = (sh >= 64) ? 0 : (id << sh);
+    auto it = std::lower_bound(leaves.begin(), leaves.end(), lo);
+    if (it == leaves.end())
+        return false;
+    return geneval_prefix(*it, depth, bits) == id;
+}
+
+template <typename Output, typename Leaf>
+geneval_result<Output, Leaf> geneval_empty_result()
+{
+    geneval_result<Output, Leaf> out;
+    std::memset(&out.leaf, 0, sizeof(out.leaf));
+    return out;
+}
+
+template <typename InteriorPRG,
+          typename ExteriorPRG,
+          typename InputT,
+          typename OutputT,
+          typename RootSampler,
+          typename PadRng>
+auto geneval_run(InputT x0, InputT x1, const std::vector<InputT> & queries,
+    RootSampler & root_sampler, PadRng & pads, OutputT y)
+{
+    static_assert(std::is_integral_v<InputT>,
+        "geneval input shares are an integral domain");
+    static_assert(!dpf::is_wildcard_v<OutputT>,
+        "geneval output is concrete; assign a wildcard leaf on a key");
+    static_assert(utils::bitlength_of_v<InputT> <= 64,
+        "geneval leaf ids are 64-bit");
+
+    using dpf_type = utils::dpf_type_t<InteriorPRG, ExteriorPRG, InputT, OutputT>;
+    using node = typename dpf_type::interior_node;
+    using leaf_node = leaf_node_t<node, OutputT>;
+    constexpr std::size_t depth = dpf_type::depth;
+
+    if (queries.empty())
+        return geneval_empty_result<OutputT, leaf_node>();
+
+    if (queries.size() > (std::size_t{1} << 22))
+        throw std::length_error("geneval query is too large");
+
+    InputT x0c = x0;
+    InputT x1c = x1;
+    utils::flip_msb_if_signed_integral(x0c);
+    const InputT alpha = utils::xor_input_shares(x0c, x1c);
+
+    std::vector<InputT> flipped;
+    flipped.reserve(queries.size());
+    std::vector<uint64_t> leaves;
+    leaves.reserve(queries.size());
+    for (const InputT & q : queries)
+    {
+        InputT fq = geneval_flipped(q);
+        flipped.push_back(fq);
+        leaves.push_back(geneval_leaf_id<dpf_type>(fq));
+    }
+    std::vector<uint64_t> unique_leaves = leaves;
+    std::sort(unique_leaves.begin(), unique_leaves.end());
+    unique_leaves.erase(std::unique(unique_leaves.begin(), unique_leaves.end()),
+        unique_leaves.end());
+    if (unique_leaves.size() > (std::size_t{1} << 20))
+        throw std::length_error("geneval trie is too large");
+
+    const uint64_t secret_leaf = geneval_leaf_id<dpf_type>(alpha);
+
+    local_cw_protocol<PadRng> proto{pads};
+    constexpr auto to_int = utils::to_integral_type<InputT>{};
+
+    const node root0 = dpf::unset_lo_bit(static_cast<node>(root_sampler()));
+    const node root1 = dpf::set_lo_bit(static_cast<node>(root_sampler()));
+
+    struct slot
+    {
+        uint64_t id;
+        node s0;
+        node s1;
+    };
+    std::vector<slot> frontier;
+    frontier.push_back(slot{0, root0, root1});
+
+    geneval_result<OutputT, leaf_node> result;
+    std::memset(&result.leaf, 0, sizeof(result.leaf));
+    result.correction_words.reserve(depth);
+    result.correction_advice.reserve(depth);
+
+    auto mask = dpf_type::msb_mask;
+    bool still_live = true;
+    for (std::size_t level = 0; level < depth; ++level, mask >>= 1)
+    {
+        const uint8_t bit0 = static_cast<uint8_t>(!!(to_int(mask) & to_int(x0c)));
+        const uint8_t bit1 = static_cast<uint8_t>(!!(to_int(mask) & to_int(x1c)));
+        const uint64_t parent_id = geneval_prefix(secret_leaf, depth, level);
+
+        node L0 = simde_mm_setzero_si128();
+        node R0 = simde_mm_setzero_si128();
+        node L1 = simde_mm_setzero_si128();
+        node R1 = simde_mm_setzero_si128();
+        bool level_live = false;
+
+        struct exp
+        {
+            uint64_t id;
+            node s0, s1, L0, R0, L1, R1;
+        };
+        std::vector<exp> exps;
+        exps.reserve(frontier.size());
+        for (const slot & n : frontier)
+        {
+            if (n.id == parent_id)
+                level_live = true;
+            const auto c0 = InteriorPRG::eval01(dpf::unset_lo_2bits(n.s0));
+            const auto c1 = InteriorPRG::eval01(dpf::unset_lo_2bits(n.s1));
+            L0 = ds_xor(L0, c0[0]);
+            R0 = ds_xor(R0, c0[1]);
+            L1 = ds_xor(L1, c1[0]);
+            R1 = ds_xor(R1, c1[1]);
+            exps.push_back(exp{n.id, n.s0, n.s1, c0[0], c0[1], c1[0], c1[1]});
+        }
+
+        node cw;
+        uint8_t advice;
+        if (still_live && level_live)
+        {
+            auto blinds = proto.prepare_level(L0, R0, bit0, L1, R1, bit1);
+            auto opened = proto.open_cw(blinds);
+            cw = opened.first;
+            advice = opened.second;
+            ++result.live_levels;
+        }
+        else
+        {
+            still_live = false;
+            cw = pads.block();
+            const uint8_t t0 = static_cast<uint8_t>(pads.bit() & 1u);
+            const uint8_t t1 = static_cast<uint8_t>(pads.bit() & 1u);
+            advice = static_cast<uint8_t>((t1 << 1) | t0);
+        }
+        result.correction_words.push_back(cw);
+        result.correction_advice.push_back(advice);
+
+        const node cw0 = dpf::set_lo_bit(cw, advice & 1u);
+        const node cw1 = dpf::set_lo_bit(cw, (advice >> 1) & 1u);
+        const std::size_t child_bits = level + 1;
+        std::vector<slot> next;
+        next.reserve(exps.size() * 2);
+        for (const exp & e : exps)
+        {
+            const uint64_t left = e.id << 1;
+            const uint64_t right = left | 1ull;
+            if (geneval_any_prefix(unique_leaves, depth, left, child_bits))
+            {
+                next.push_back(slot{left,
+                    dpf::xor_if_lo_bit(e.L0, cw0, e.s0),
+                    dpf::xor_if_lo_bit(e.L1, cw0, e.s1)});
+            }
+            if (geneval_any_prefix(unique_leaves, depth, right, child_bits))
+            {
+                next.push_back(slot{right,
+                    dpf::xor_if_lo_bit(e.R0, cw1, e.s0),
+                    dpf::xor_if_lo_bit(e.R1, cw1, e.s1)});
+            }
+        }
+        frontier = std::move(next);
+    }
+
+    result.leaf_live = geneval_any_prefix(unique_leaves, depth, secret_leaf, depth);
+    if (result.leaf_live)
+    {
+        const slot * on = nullptr;
+        for (const slot & n : frontier)
+        {
+            if (n.id == secret_leaf)
+            {
+                on = &n;
+                break;
+            }
+        }
+        if (on == nullptr)
+            throw std::logic_error("geneval: secret leaf missing from trie");
+        const bool sign0 = dpf::get_lo_bit(on->s0);
+        auto built = dpf::make_leaves<ExteriorPRG>(alpha,
+            dpf::unset_lo_2bits(on->s0), dpf::unset_lo_2bits(on->s1), sign0,
+            std::size_t{0}, y);
+        result.leaf = std::get<0>(built.first.first);
+    }
+
+    result.party0.reserve(flipped.size());
+    result.party1.reserve(flipped.size());
+    for (std::size_t i = 0; i < flipped.size(); ++i)
+    {
+        const uint64_t id = leaves[i];
+        const slot * n = nullptr;
+        for (const slot & s : frontier)
+        {
+            if (s.id == id)
+            {
+                n = &s;
+                break;
+            }
+        }
+        if (n == nullptr)
+            throw std::logic_error("geneval: query leaf missing from trie");
+        auto share0 = dpf_type::template traverse_exterior<0>(n->s0, result.leaf);
+        auto share1 = dpf_type::template traverse_exterior<0>(n->s1, result.leaf);
+        const auto lane = static_cast<std::size_t>(to_int(flipped[i]));
+        result.party0.push_back(extract_leaf<node, OutputT>(share0, lane));
+        result.party1.push_back(extract_leaf<node, OutputT>(share1, lane));
+    }
+    return result;
+}
+
+template <typename InputT>
+InputT geneval_from_bits(uint64_t bits)
+{
+    using U = std::make_unsigned_t<InputT>;
+    U u = static_cast<U>(bits);
+    InputT out;
+    std::memcpy(&out, &u, sizeof(out));
+    return out;
+}
+
+template <typename InputT>
+bool geneval_out_of_order(InputT from, InputT to)
+{
+    // Numeric order. An unsigned compare of a signed value treats a negative
+    // `from` as larger than a positive `to`, and would reject `[-1, 1]`.
+    if constexpr (std::is_signed_v<InputT>)
+        return from > to;
+    else
+        return utils::to_integral_type<InputT>{}(from)
+            > utils::to_integral_type<InputT>{}(to);
+}
+
+template <typename InputT>
+std::vector<InputT> geneval_full_domain()
+{
+    constexpr std::size_t bitlen = utils::bitlength_of_v<InputT>;
+    if (bitlen > 20)
+        throw std::length_error("geneval_full domain is too large");
+    const uint64_t n = uint64_t{1} << bitlen;
+    std::vector<InputT> qs(static_cast<std::size_t>(n));
+    // Index `i` is the input's bit pattern, including the sign bit. A
+    // narrowing cast of `i` to a signed type is implementation-defined.
+    for (uint64_t i = 0; i < n; ++i)
+        qs[static_cast<std::size_t>(i)] = geneval_from_bits<InputT>(i);
+    return qs;
+}
+
+template <typename InputT>
+std::vector<InputT> geneval_inclusive(InputT from, InputT to)
+{
+    if (geneval_out_of_order(from, to))
+    {
+        throw std::invalid_argument("geneval_interval: from > to");
+    }
+    std::vector<InputT> qs;
+    InputT q = from;
+    const InputT one = utils::make_from_integral_value<InputT>{}(1);
+    for (;;)
+    {
+        qs.push_back(q);
+        if (q == to)
+            break;
+        q = geneval_mod_add(q, one);
+        if (qs.size() > (std::size_t{1} << 22))
+            throw std::length_error("geneval_interval is too large");
+    }
+    return qs;
+}
+
+template <typename InputT, typename TargetSampler>
+InputT geneval_sample_target(TargetSampler & sample)
+{
+    return static_cast<InputT>(sample());
+}
+
+template <typename InputT>
+std::vector<InputT> geneval_shift_all(const std::vector<InputT> & qs, InputT delta)
+{
+    std::vector<InputT> out;
+    out.reserve(qs.size());
+    for (const InputT & q : qs)
+        out.push_back(geneval_mod_add(q, delta));
+    return out;
+}
+
+}  // namespace detail
+
+/// Geneval at one public point. The secret point is `x0 XOR x1`.
+template <typename InteriorPRG = dpf::prg::aes128,
+          typename ExteriorPRG = InteriorPRG,
+          typename InputT,
+          typename OutputT,
+          typename RootSampler,
+          typename PadRng>
+HEDLEY_WARN_UNUSED_RESULT
+auto geneval_point(InputT x0, InputT x1, InputT query,
+    ds_randomness<RootSampler, PadRng> rng, OutputT y)
+{
+    return detail::geneval_run<InteriorPRG, ExteriorPRG>(x0, x1,
+        std::vector<InputT>{query}, rng.root, rng.pad, y);
+}
+
+/// Geneval on the inclusive interval `[from, to]`.
+template <typename InteriorPRG = dpf::prg::aes128,
+          typename ExteriorPRG = InteriorPRG,
+          typename InputT,
+          typename OutputT,
+          typename RootSampler,
+          typename PadRng>
+HEDLEY_WARN_UNUSED_RESULT
+auto geneval_interval(InputT x0, InputT x1, InputT from, InputT to,
+    ds_randomness<RootSampler, PadRng> rng, OutputT y)
+{
+    return detail::geneval_run<InteriorPRG, ExteriorPRG>(x0, x1,
+        detail::geneval_inclusive(from, to), rng.root, rng.pad, y);
+}
+
+/// Geneval on the whole domain. Refuses a domain above 2^20 inputs.
+template <typename InteriorPRG = dpf::prg::aes128,
+          typename ExteriorPRG = InteriorPRG,
+          typename InputT,
+          typename OutputT,
+          typename RootSampler,
+          typename PadRng>
+HEDLEY_WARN_UNUSED_RESULT
+auto geneval_full(InputT x0, InputT x1,
+    ds_randomness<RootSampler, PadRng> rng, OutputT y)
+{
+    return detail::geneval_run<InteriorPRG, ExteriorPRG>(x0, x1,
+        detail::geneval_full_domain<InputT>(), rng.root, rng.pad, y);
+}
+
+/// Geneval on a public sequence, in the order given.
+template <typename InteriorPRG = dpf::prg::aes128,
+          typename ExteriorPRG = InteriorPRG,
+          typename InputT,
+          typename OutputT,
+          typename RootSampler,
+          typename PadRng,
+          typename ForwardIterator>
+HEDLEY_WARN_UNUSED_RESULT
+auto geneval_sequence(InputT x0, InputT x1, ForwardIterator begin,
+    ForwardIterator end, ds_randomness<RootSampler, PadRng> rng, OutputT y)
+{
+    std::vector<InputT> qs(begin, end);
+    return detail::geneval_run<InteriorPRG, ExteriorPRG>(x0, x1,
+        std::move(qs), rng.root, rng.pad, y);
+}
+
+/// Wildcard-input geneval. `x0 + x1` is the real point (additive shares).
+/// `sample_target()` is the random DPF target; the public query is shifted
+/// by `target - (x0 + x1)` before the walk.
+template <typename InteriorPRG = dpf::prg::aes128,
+          typename ExteriorPRG = InteriorPRG,
+          typename InputT,
+          typename OutputT,
+          typename RootSampler,
+          typename PadRng,
+          typename TargetSampler>
+HEDLEY_WARN_UNUSED_RESULT
+auto geneval_point(wildcard_input_t, InputT x0, InputT x1, InputT query,
+    ds_randomness<RootSampler, PadRng> rng, TargetSampler sample_target,
+    OutputT y)
+{
+    const InputT alpha = detail::geneval_sample_target<InputT>(sample_target);
+    const InputT delta = detail::geneval_mod_sub(alpha,
+        detail::geneval_mod_add(x0, x1));
+    const InputT shifted = detail::geneval_mod_add(query, delta);
+    InputT zero{};
+    return detail::geneval_run<InteriorPRG, ExteriorPRG>(zero, alpha,
+        std::vector<InputT>{shifted}, rng.root, rng.pad, y);
+}
+
+template <typename InteriorPRG = dpf::prg::aes128,
+          typename ExteriorPRG = InteriorPRG,
+          typename InputT,
+          typename OutputT,
+          typename RootSampler,
+          typename PadRng>
+HEDLEY_WARN_UNUSED_RESULT
+auto geneval_point(wildcard_input_t, InputT x0, InputT x1, InputT query,
+    ds_randomness<RootSampler, PadRng> rng, OutputT y)
+{
+    return geneval_point<InteriorPRG, ExteriorPRG>(wildcard_input, x0, x1, query,
+        std::move(rng), [] { return dpf::uniform_sample<InputT>(); }, y);
+}
+
+template <typename InteriorPRG = dpf::prg::aes128,
+          typename ExteriorPRG = InteriorPRG,
+          typename InputT,
+          typename OutputT,
+          typename RootSampler,
+          typename PadRng,
+          typename TargetSampler>
+HEDLEY_WARN_UNUSED_RESULT
+auto geneval_interval(wildcard_input_t, InputT x0, InputT x1, InputT from,
+    InputT to, ds_randomness<RootSampler, PadRng> rng, TargetSampler sample_target,
+    OutputT y)
+{
+    const InputT alpha = detail::geneval_sample_target<InputT>(sample_target);
+    const InputT delta = detail::geneval_mod_sub(alpha,
+        detail::geneval_mod_add(x0, x1));
+    auto shifted = detail::geneval_shift_all(
+        detail::geneval_inclusive(from, to), delta);
+    InputT zero{};
+    return detail::geneval_run<InteriorPRG, ExteriorPRG>(zero, alpha,
+        std::move(shifted), rng.root, rng.pad, y);
+}
+
+template <typename InteriorPRG = dpf::prg::aes128,
+          typename ExteriorPRG = InteriorPRG,
+          typename InputT,
+          typename OutputT,
+          typename RootSampler,
+          typename PadRng>
+HEDLEY_WARN_UNUSED_RESULT
+auto geneval_interval(wildcard_input_t, InputT x0, InputT x1, InputT from,
+    InputT to, ds_randomness<RootSampler, PadRng> rng, OutputT y)
+{
+    return geneval_interval<InteriorPRG, ExteriorPRG>(wildcard_input, x0, x1,
+        from, to, std::move(rng), [] { return dpf::uniform_sample<InputT>(); }, y);
+}
+
+template <typename InteriorPRG = dpf::prg::aes128,
+          typename ExteriorPRG = InteriorPRG,
+          typename InputT,
+          typename OutputT,
+          typename RootSampler,
+          typename PadRng,
+          typename TargetSampler>
+HEDLEY_WARN_UNUSED_RESULT
+auto geneval_full(wildcard_input_t, InputT x0, InputT x1,
+    ds_randomness<RootSampler, PadRng> rng, TargetSampler sample_target, OutputT y)
+{
+    const InputT alpha = detail::geneval_sample_target<InputT>(sample_target);
+    const InputT delta = detail::geneval_mod_sub(alpha,
+        detail::geneval_mod_add(x0, x1));
+    InputT zero{};
+    auto full = detail::geneval_run<InteriorPRG, ExteriorPRG>(zero, alpha,
+        detail::geneval_full_domain<InputT>(), rng.root, rng.pad, y);
+    constexpr auto to_int = utils::to_integral_type<InputT>{};
+    const std::size_t n = full.party0.size();
+    std::vector<OutputT> p0(n), p1(n);
+    for (std::size_t i = 0; i < n; ++i)
+    {
+        InputT q = detail::geneval_from_bits<InputT>(i);
+        InputT s = detail::geneval_mod_add(q, delta);
+        const std::size_t si = static_cast<std::size_t>(to_int(s));
+        p0[i] = full.party0[si];
+        p1[i] = full.party1[si];
+    }
+    full.party0 = std::move(p0);
+    full.party1 = std::move(p1);
+    return full;
+}
+
+template <typename InteriorPRG = dpf::prg::aes128,
+          typename ExteriorPRG = InteriorPRG,
+          typename InputT,
+          typename OutputT,
+          typename RootSampler,
+          typename PadRng>
+HEDLEY_WARN_UNUSED_RESULT
+auto geneval_full(wildcard_input_t, InputT x0, InputT x1,
+    ds_randomness<RootSampler, PadRng> rng, OutputT y)
+{
+    return geneval_full<InteriorPRG, ExteriorPRG>(wildcard_input, x0, x1,
+        std::move(rng), [] { return dpf::uniform_sample<InputT>(); }, y);
+}
+
+template <typename InteriorPRG = dpf::prg::aes128,
+          typename ExteriorPRG = InteriorPRG,
+          typename InputT,
+          typename OutputT,
+          typename RootSampler,
+          typename PadRng,
+          typename ForwardIterator,
+          typename TargetSampler>
+HEDLEY_WARN_UNUSED_RESULT
+auto geneval_sequence(wildcard_input_t, InputT x0, InputT x1,
+    ForwardIterator begin, ForwardIterator end,
+    ds_randomness<RootSampler, PadRng> rng, TargetSampler sample_target, OutputT y)
+{
+    const InputT alpha = detail::geneval_sample_target<InputT>(sample_target);
+    const InputT delta = detail::geneval_mod_sub(alpha,
+        detail::geneval_mod_add(x0, x1));
+    std::vector<InputT> qs(begin, end);
+    auto shifted = detail::geneval_shift_all(qs, delta);
+    InputT zero{};
+    return detail::geneval_run<InteriorPRG, ExteriorPRG>(zero, alpha,
+        std::move(shifted), rng.root, rng.pad, y);
+}
+
+template <typename InteriorPRG = dpf::prg::aes128,
+          typename ExteriorPRG = InteriorPRG,
+          typename InputT,
+          typename OutputT,
+          typename RootSampler,
+          typename PadRng,
+          typename ForwardIterator>
+HEDLEY_WARN_UNUSED_RESULT
+auto geneval_sequence(wildcard_input_t, InputT x0, InputT x1,
+    ForwardIterator begin, ForwardIterator end,
+    ds_randomness<RootSampler, PadRng> rng, OutputT y)
+{
+    return geneval_sequence<InteriorPRG, ExteriorPRG>(wildcard_input, x0, x1,
+        begin, end, std::move(rng), [] { return dpf::uniform_sample<InputT>(); }, y);
+}
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_GENEVAL_HPP__

include/dpf/interval_memoizer.hpp

@@ -0,0 +1,542 @@
+/// @file dpf/interval_memoizer.hpp
+/// @brief
+/// @details
+/// @author Ryan Henry <ryan.henry@ucalgary.ca>
+/// @author Christopher Jiang <christopher.jiang@ucalgary.ca>
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_INTERVAL_MEMOIZER_HPP__
+#define LIBDPF_INCLUDE_DPF_INTERVAL_MEMOIZER_HPP__
+
+#include "hedley/hedley.h"
+
+#include <cstddef>
+#include <cstring>
+#include <type_traits>
+#include <functional>
+#include <algorithm>
+#include <new>
+#include <limits>
+#include <stdexcept>
+#include <optional>
+#include <array>
+
+#include "dpf/dpf_key.hpp"
+#include "dpf/secret_share.hpp"
+
+namespace dpf
+{
+
+/// Ping-pong pivot math underflows at 0 leaves. Keep a one-node slab so the
+/// root still has a place to land; callers never walk a 0-leaf interval.
+inline std::size_t interval_memoizer_slots(std::size_t output_len)
+{
+    return output_len == 0 ? std::size_t{1} : output_len;
+}
+
+/// Interval memoizers key on the underlying DPF key type (same rule as path
+/// memoizers): a memoizer built from `party_key<0, Key>` also accepts
+/// `party_key<1, Key>` and bare `Key`.
+template <typename DpfKey>
+using interval_memoizer_key_t = unwrap_party_key_t<DpfKey>;
+
+template <typename DpfKey,
+          typename ReturnT = typename interval_memoizer_key_t<DpfKey>::interior_node *>
+struct interval_memoizer_base
+{
+  public:
+    using dpf_type = interval_memoizer_key_t<DpfKey>;
+    using integral_type = typename dpf_type::integral_type;
+    using return_type = ReturnT;
+    using iterator_type = return_type;
+    using node_type = typename dpf_type::interior_node;
+
+    // level 0 should access the root
+    // level goes up to (and including) depth
+    virtual return_type operator[](std::size_t) const noexcept = 0;
+
+    // iterators should access most recently completed level
+    virtual return_type begin() const noexcept = 0;
+    virtual return_type end() const noexcept = 0;
+
+    virtual std::size_t assign_interval(const dpf_type & dpf, integral_type new_from, integral_type new_to)
+    {
+        static constexpr auto complement_of = std::bit_not{};
+        if (dpf_.has_value() == false
+            || std::memcmp(&dpf_root_, &dpf.root(), sizeof(node_type)) != 0
+            || std::memcmp(&dpf_common_part_hash_, &dpf.common_part_hash(), sizeof(digest_type)) != 0
+            || from_.value_or(complement_of(new_from)) != new_from
+            || to_.value_or(complement_of(new_to)) != new_to)
+        {
+            if (new_to - new_from > output_length)
+            {
+                throw std::length_error("size of new interval is too large for memoizer");
+            }
+
+            this->operator[](0)[0] = dpf.root();
+            dpf_ = std::cref(dpf);
+            dpf_root_ = dpf.root();
+            dpf_common_part_hash_ = dpf.common_part_hash();
+            from_ = new_from;
+            to_ = new_to;
+            level_index = 1;
+        }
+
+        return level_index;
+    }
+
+    std::size_t advance_level()
+    {
+        return ++level_index;
+    }
+
+    std::size_t get_nodes_at_level() const
+    {
+        return get_nodes_at_level(level_index, from_.value_or(0), to_.value_or(0));
+    }
+
+    std::size_t get_nodes_at_level(std::size_t level) const
+    {
+        return get_nodes_at_level(level, from_.value_or(0), to_.value_or(0));
+    }
+
+    static std::size_t get_nodes_at_level(std::size_t level, integral_type from_node, integral_type to_node)
+    {
+        // Algorithm explanation:
+        //   Input:
+        //     * offset - (derived from depth and level, note that level of -1 represents the root of the tree)
+        //     * range of nodes - [from_node, to_node)
+        //
+        //   Observation 1:
+        //     For any level, knowing the range [from, to) allows one to calculate the number of nodes at that level
+        //     as (to - from).
+        //
+        //   Observation 2:
+        //     If the range were stated as [from_0, to_0] for an offset 0, then [from_n, to_n] = [from_0 >> n, to_0 >> n]
+        //     where >> is the bitshift operator. This is because the bits representing a node also represent the path
+        //     taken in a binary tree to get to that node. Since from_0 and to_0 are both inclusive bounds, then their
+        //     parent nodes must also be inclusive bounds for the next level up. These nodes can be found by simply removing
+        //     the LSB from from_0 and to_0. The same can be done for parents further up the tree.
+        //
+        //   Putting it together:
+        //     * to_node-1 converts an excluded node to an included node
+        //     * bit shifting as explained in observation 2
+        //     * add 1 since observation 1 is for an excluded end point whereas now both end points are included
+
+        std::size_t offset = depth - level;
+        return utils::shift_right(to_node - integral_type{1}, offset)
+            - utils::shift_right(from_node, offset) + 1;
+    }
+
+  protected:
+    static constexpr auto depth = dpf_type::depth;
+    std::size_t output_length;
+    std::size_t level_index;  // indicates current level being built
+
+    explicit interval_memoizer_base(std::size_t output_len)
+      : dpf_{std::nullopt},
+        from_{std::nullopt},
+        to_{std::nullopt},
+        output_length{output_len},
+        level_index{0}
+    { }
+
+  private:
+    std::optional<std::reference_wrapper<const dpf_type>> dpf_;
+    node_type dpf_root_;
+    digest_type dpf_common_part_hash_;
+    std::optional<integral_type> from_;
+    std::optional<integral_type> to_;
+};
+
+template <typename DpfKey,
+          typename Allocator = aligned_allocator<
+              typename interval_memoizer_key_t<DpfKey>::interior_node>>
+struct basic_interval_memoizer final : public interval_memoizer_base<DpfKey>
+{
+  private:
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    using parent = interval_memoizer_base<DpfKey>;
+HEDLEY_PRAGMA(GCC diagnostic pop)
+  public:
+    using unique_ptr = typename Allocator::unique_ptr;
+    using return_type = typename interval_memoizer_key_t<DpfKey>::interior_node *;
+    using parent::depth;
+    using parent::level_index;
+    using parent::get_nodes_at_level;
+
+    // See comment for full_tree_interval_memoizer::initialize_endpoints() for
+    //   general explanation of derivation for "nodes at previous level".
+    // When creating the final level of interior nodes from the previous level,
+    //   care must be taken not to overwrite the previous level until the relevant
+    //   nodes have been used to generate the new level. This means the pivot must
+    //   be selected to push the previous level as far to the end of the buffer as
+    //   possible.
+    // For n nodes in the final level:
+    //     n odd => (n+1)/2 nodes on previous level
+    //           => pivot = n-(n+1)/2 = (n-1)/2 = n/2-1/2 = floor(n/2)
+    //     n even => n/2 OR (n+2)/2 nodes on previous level
+    //            => pivot = n-(n+2)/2 = (n-2)/2 = n/2-1
+    //     unified => floor(n/2)-1+(n%2) = (n>>1)+(n&1)-1
+    // In general, each previous level has roughly one half the nodes, but this is
+    //   not true for some small n, which can stay constant up to the root.
+    //   To handle this, take the maximum between the unified calculation shown
+    //   and the number of nodes two levels up from the final level.
+    // For n nodes in the final level:
+    //     at most ((n+2)/2+2)/2 = n+6>>2 nodes two levels up
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    explicit basic_interval_memoizer(std::size_t output_len, Allocator alloc = Allocator{})
+      : parent::interval_memoizer_base(output_len),
+        pivot{std::max((interval_memoizer_slots(output_len)>>1)
+                +(interval_memoizer_slots(output_len)&1)-1,
+            (interval_memoizer_slots(output_len) + 6) >> 2)},
+        buf{alloc.allocate_unique_ptr(
+            pivot+((interval_memoizer_slots(output_len)+2)>>1))}
+    {
+        if (HEDLEY_UNLIKELY(buf == nullptr)) throw std::bad_alloc{};
+    }
+HEDLEY_PRAGMA(GCC diagnostic pop)
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    return_type operator[](std::size_t level) const noexcept override
+    {
+        bool b = (depth ^ level) & 1;
+        return Allocator::assume_aligned(&buf[b*pivot]);
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    return_type begin() const noexcept override
+    {
+        return this->operator[](level_index - 1);
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    return_type end() const noexcept override
+    {
+        return this->operator[](level_index - 1) + get_nodes_at_level(level_index - 1);
+    }
+
+  private:
+    static constexpr auto clz = utils::countl_zero<std::size_t>{};
+    std::size_t pivot;
+    unique_ptr buf;
+};
+
+template <typename DpfKey,
+          typename Allocator = aligned_allocator<
+              typename interval_memoizer_key_t<DpfKey>::interior_node>>
+struct full_tree_interval_memoizer final : public interval_memoizer_base<DpfKey>
+{
+  private:
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    using parent = interval_memoizer_base<DpfKey>;
+HEDLEY_PRAGMA(GCC diagnostic pop)
+  public:
+    using node_type = typename interval_memoizer_key_t<DpfKey>::interior_node;
+    using unique_ptr = typename Allocator::unique_ptr;
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    using return_type = std::add_pointer_t<node_type>;
+HEDLEY_PRAGMA(GCC diagnostic pop)
+    using integral_type = typename interval_memoizer_key_t<DpfKey>::integral_type;
+    using parent::depth;
+    using parent::level_index;
+    using parent::get_nodes_at_level;
+    static constexpr bool retains_all_levels = true;
+
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    explicit full_tree_interval_memoizer(std::size_t output_len,
+        Allocator alloc = Allocator{})
+      : parent::interval_memoizer_base(output_len),
+        level_endpoints{initialize_endpoints(output_len)},
+        buf{alloc.allocate_unique_ptr(level_endpoints[depth] + output_len)}
+    {
+        if (HEDLEY_UNLIKELY(buf == nullptr)) throw std::bad_alloc{};
+    }
+HEDLEY_PRAGMA(GCC diagnostic pop)
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    return_type operator[](std::size_t level) const noexcept override
+    {
+        return Allocator::assume_aligned(&buf[level_endpoints[level]]);
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    return_type begin() const noexcept override
+    {
+        return this->operator[](level_index - 1);
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    return_type end() const noexcept override
+    {
+        return this->operator[](level_index - 1) + get_nodes_at_level(level_index - 1);
+    }
+
+  private:
+    const std::array<std::size_t, depth+1> level_endpoints;
+    unique_ptr buf;
+
+    // For n nodes on a given level, there are the following cases:
+    //     n odd => (n+1)/2 nodes on previous level
+    //         ex. 5 nodes on current level grouped as
+    //             |..|..|.| or |.|..|..|
+    //             where both give 3 nodes on previous level
+    //     n even => n/2 OR (n+2)/2 nodes on previous level
+    //         ex. 6 nodes on current level grouped as
+    //             |..|..|..| or |.|..|..|.|
+    //             gives either 3 or 4 nodes on previous level
+    // Clearly (n+2)/2 is the worst case, so this is used in the derivation
+    //   for the number of nodes on each level.
+    // Also note that at depth (from the root) i, there can't be more than 2^i
+    //   nodes hence the `min()` function call.
+    static constexpr auto initialize_endpoints(integral_type len)
+    {
+        std::array<std::size_t, depth+1> level_endpoints{0};
+        for (std::size_t level=depth; level > 0; --level)
+        {
+            len = std::min(len+2 >> 1, integral_type(1) << level-1);
+            level_endpoints[level] = len;
+        }
+
+        for (std::size_t level = 0; level < depth; ++level)
+        {
+            level_endpoints[level+1] = level_endpoints[level] + level_endpoints[level+1];
+        }
+
+        return level_endpoints;
+    }
+};
+
+/// Interval memoizer whose leaf depth is `StopLevel` (incremental `eval_interval`).
+template <typename DpfKey, std::size_t StopLevel,
+          typename Allocator = aligned_allocator<
+              typename interval_memoizer_key_t<DpfKey>::interior_node>>
+struct basic_interval_memoizer_at
+{
+  public:
+    using dpf_type = interval_memoizer_key_t<DpfKey>;
+    using integral_type = typename dpf_type::integral_type;
+    using node_type = typename dpf_type::interior_node;
+    using return_type = node_type *;
+    using unique_ptr = typename Allocator::unique_ptr;
+    static constexpr std::size_t depth = StopLevel;
+
+    explicit basic_interval_memoizer_at(std::size_t output_len,
+        Allocator alloc = Allocator{})
+      : output_length{output_len},
+        level_index{0},
+        pivot{std::max((interval_memoizer_slots(output_len) >> 1)
+                + (interval_memoizer_slots(output_len) & 1) - 1,
+            (interval_memoizer_slots(output_len) + 6) >> 2)},
+        buf{alloc.allocate_unique_ptr(
+            pivot + ((interval_memoizer_slots(output_len) + 2) >> 1))},
+        from_{std::nullopt},
+        to_{std::nullopt}
+    {
+        if (HEDLEY_UNLIKELY(buf == nullptr)) throw std::bad_alloc{};
+    }
+
+    std::size_t assign_interval(const dpf_type & dpf, integral_type new_from,
+        integral_type new_to)
+    {
+        static constexpr auto complement_of = std::bit_not{};
+        if (from_.has_value() == false
+            || std::memcmp(&dpf_root_, &dpf.root(), sizeof(node_type)) != 0
+            || std::memcmp(&dpf_common_part_hash_, &dpf.common_part_hash(),
+                   sizeof(digest_type)) != 0
+            || from_.value_or(complement_of(new_from)) != new_from
+            || to_.value_or(complement_of(new_to)) != new_to)
+        {
+            if (new_to - new_from > output_length)
+                throw std::length_error("size of new interval is too large for memoizer");
+            (*this)[0][0] = dpf.root();
+            dpf_root_ = dpf.root();
+            dpf_common_part_hash_ = dpf.common_part_hash();
+            from_ = new_from;
+            to_ = new_to;
+            level_index = 1;
+        }
+        return level_index;
+    }
+
+    std::size_t advance_level() { return ++level_index; }
+
+    std::size_t get_nodes_at_level() const
+    {
+        return get_nodes_at_level(level_index, from_.value_or(0), to_.value_or(0));
+    }
+
+    std::size_t get_nodes_at_level(std::size_t level) const
+    {
+        return get_nodes_at_level(level, from_.value_or(0), to_.value_or(0));
+    }
+
+    static std::size_t get_nodes_at_level(std::size_t level, integral_type from_node,
+        integral_type to_node)
+    {
+        std::size_t offset = depth - level;
+        return utils::shift_right(to_node - integral_type{1}, offset)
+            - utils::shift_right(from_node, offset) + 1;
+    }
+
+    return_type operator[](std::size_t level) const noexcept
+    {
+        bool b = (depth ^ level) & 1;
+        return Allocator::assume_aligned(&buf[b * pivot]);
+    }
+
+  private:
+    std::size_t output_length;
+    std::size_t level_index;
+    std::size_t pivot;
+    unique_ptr buf;
+    node_type dpf_root_;
+    digest_type dpf_common_part_hash_;
+    std::optional<integral_type> from_;
+    std::optional<integral_type> to_;
+};
+
+namespace detail
+{
+
+template <typename DpfKey,
+          typename MemoizerT,
+          typename InputT>
+HEDLEY_ALWAYS_INLINE
+auto make_interval_memoizer(InputT from, InputT to)
+{
+    using dpf_type = DpfKey;
+
+    utils::flip_msb_if_signed_integral(from);
+    utils::flip_msb_if_signed_integral(to);
+
+    std::size_t nodes_in_interval = utils::get_leafnodes_in_output_interval<dpf_type>(from, to);
+
+    return MemoizerT(nodes_in_interval);
+}
+
+}  // namespace detail
+
+template <typename DpfKey,
+          typename InputT>
+inline auto make_basic_interval_memoizer(InputT from, InputT to)
+{
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    using key_t = interval_memoizer_key_t<DpfKey>;
+    return detail::make_interval_memoizer<key_t, basic_interval_memoizer<key_t>, InputT>(from, to);
+HEDLEY_PRAGMA(GCC diagnostic pop)
+}
+
+template <typename DpfKey,
+          typename InputT>
+inline auto make_basic_interval_memoizer(const DpfKey &, InputT from, InputT to)
+{
+    return make_basic_interval_memoizer<DpfKey>(from, to);
+}
+
+template <typename DpfKey>
+inline auto make_basic_full_memoizer()
+{
+    using input_type = typename DpfKey::input_type;
+
+    return make_basic_interval_memoizer<DpfKey>(
+        std::numeric_limits<input_type>::min(),
+        std::numeric_limits<input_type>::max());
+}
+
+template <typename DpfKey>
+inline auto make_basic_full_memoizer(const DpfKey &)
+{
+    return make_basic_full_memoizer<DpfKey>();
+}
+
+template <typename DpfKey,
+          typename InputT>
+inline auto make_full_tree_interval_memoizer(InputT from, InputT to)
+{
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    using key_t = interval_memoizer_key_t<DpfKey>;
+    return detail::make_interval_memoizer<key_t, full_tree_interval_memoizer<key_t>, InputT>(from, to);
+HEDLEY_PRAGMA(GCC diagnostic pop)
+}
+
+template <typename DpfKey,
+          typename InputT>
+inline auto make_full_tree_interval_memoizer(const DpfKey &, InputT from, InputT to)
+{
+    return make_full_tree_interval_memoizer<DpfKey>(from, to);
+}
+
+template <typename DpfKey>
+inline auto make_full_tree_full_memoizer()
+{
+    using input_type = typename DpfKey::input_type;
+
+    return make_full_tree_interval_memoizer<DpfKey>(
+        std::numeric_limits<input_type>::min(),
+        std::numeric_limits<input_type>::max());
+}
+
+template <typename DpfKey>
+inline auto make_full_tree_full_memoizer(const DpfKey &)
+{
+    return make_full_tree_full_memoizer<DpfKey>();
+}
+
+template <typename DpfKey, std::size_t StopLevel>
+inline auto make_basic_interval_memoizer_at(std::size_t leaf_nodes)
+{
+    return basic_interval_memoizer_at<DpfKey, StopLevel>(leaf_nodes);
+}
+
+/// Stop-level interval memoizer for output slot `I` of a multi-level key.
+/// Sizes the ping-pong buffer for the lane-domain interval `[from, to]`
+/// expanded to `meta[I].tree_level` (the leaf level of slot `I`). This is the
+/// default memoizer for a multi-level `eval_interval(out<I>, ...)`.
+template <typename DpfKey, std::size_t I,
+          typename InputT,
+          std::enable_if_t<DpfKey::is_multilevel, bool> = true>
+inline auto make_basic_interval_memoizer(InputT from, InputT to)
+{
+    constexpr std::size_t stop = DpfKey::meta[I].tree_level;
+    constexpr auto lg = DpfKey::template lg_outputs_per_leaf_of<I>;
+    using integral_type = typename DpfKey::integral_type;
+    constexpr auto to_int = utils::to_integral_type<InputT>{};
+
+    utils::flip_msb_if_signed_integral(from);
+    utils::flip_msb_if_signed_integral(to);
+
+    const integral_type from_node = utils::leaf_node_floor(
+        static_cast<integral_type>(to_int(from)), lg);
+    const integral_type to_node = utils::leaf_node_ceil_exclusive(
+        static_cast<integral_type>(to_int(to)), lg);
+    const auto segs = utils::split_leaf_nodes(from_node, to_node, stop);
+    return basic_interval_memoizer_at<DpfKey, stop>(segs.total);
+}
+
+template <typename DpfKey, std::size_t I,
+          typename InputT,
+          std::enable_if_t<DpfKey::is_multilevel, bool> = true>
+inline auto make_basic_interval_memoizer(const DpfKey &, InputT from, InputT to)
+{
+    return make_basic_interval_memoizer<DpfKey, I>(from, to);
+}
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_INTERVAL_MEMOIZER_HPP__

include/dpf/json.hpp

@@ -0,0 +1,253 @@
+/// @file dpf/json.hpp
+/// @brief
+/// @details
+/// @author Ryan Henry <ryan.henry@ucalgary.ca>
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_JSON_HPP__
+#define LIBDPF_INCLUDE_DPF_JSON_HPP__
+
+#include <cstddef>
+#include <cstdint>
+#include <tuple>
+#include <array>
+#include <string>
+#include <bitset>
+#include <type_traits>
+#include <utility>
+
+#include "json/include/nlohmann/json.hpp"
+#include "portable-snippets/exact-int/exact-int.h"
+
+#include "dpf/dpf_key.hpp"
+
+namespace nlohmann
+{
+
+template <typename NodeT,
+          typename OutputT>
+struct adl_serializer<dpf::beaver<true, NodeT, OutputT>>
+{
+    static void from_json(const nlohmann::json & j, dpf::beaver<true, NodeT, OutputT> & beaver)  // NOLINT(runtime/references)
+    {
+        j.get_to(beaver.output_blind);
+        j.get_to(beaver.vector_blind);
+        j.get_to(beaver.blinded_vector);
+    }
+
+    static void to_json(nlohmann::json & j, const dpf::beaver<true, NodeT, OutputT> & beaver)  // NOLINT(runtime/references)
+    {
+        j = nlohmann::json{
+            {"output_blind", beaver.output_blind},
+            {"vector_blind", beaver.vector_blind},
+            {"blinded_vector", beaver.blinded_vector}
+        };
+    }
+};
+
+template <>
+struct adl_serializer<simde__m128i>
+{
+    static void from_json(const nlohmann::json & j, simde__m128i & a)  // NOLINT(runtime/references)
+    {
+        std::array<psnip_uint64_t, 2> A;
+        j.get_to(A);
+        a = simde_mm_set_epi64x(A[1], A[0]);
+    }
+
+    static void to_json(nlohmann::json & j, const simde__m128i & a)  // NOLINT(runtime/references)
+    {
+        j = nlohmann::json{a[0], a[1]};
+    }
+};
+
+template <>
+struct adl_serializer<simde__m256i>
+{
+    static void from_json(const nlohmann::json & j, simde__m256i & a)  // NOLINT(runtime/references)
+    {
+        std::array<psnip_uint64_t, 4> A;
+        j.get_to(A);
+        a = simde_mm256_set_epi64x(A[3], A[2], A[1], A[0]);
+    }
+
+    static void to_json(nlohmann::json & j, const simde__m256i & a)  // NOLINT(runtime/references)
+    {
+        j = nlohmann::json{a[0], a[1], a[2], a[3]};
+    }
+};
+
+template <>
+struct adl_serializer<dpf::detail::cmp_meta>
+{
+    static void from_json(const nlohmann::json & j, dpf::detail::cmp_meta & c)  // NOLINT(runtime/references)
+    {
+        j.at("nbits").get_to(c.nbits);
+        j.at("mask").get_to(c.mask);
+        c.kind = static_cast<dpf::cmp_kind>(j.at("kind").get<psnip_uint8_t>());
+        c.trivial =
+            static_cast<dpf::cmp_trivial>(j.at("trivial").get<psnip_uint8_t>());
+        j.at("eval_as_ge").get_to(c.eval_as_ge);
+        j.at("include_eq").get_to(c.include_eq);
+        j.at("active").get_to(c.active);
+    }
+
+    static void to_json(nlohmann::json & j, const dpf::detail::cmp_meta & c)  // NOLINT(runtime/references)
+    {
+        j = nlohmann::json{
+            {"nbits", c.nbits},
+            {"mask", c.mask},
+            {"kind", static_cast<psnip_uint8_t>(c.kind)},
+            {"trivial", static_cast<psnip_uint8_t>(c.trivial)},
+            {"eval_as_ge", c.eval_as_ge},
+            {"include_eq", c.include_eq},
+            {"active", c.active}
+        };
+    }
+};
+
+// Classic single-level key (no `at<>` / no comparison channel).
+template <typename InteriorPRG,
+          typename ExteriorPRG,
+          typename InputT,
+          typename OutputT,
+          typename ...OutputTs>
+struct adl_serializer<dpf::dpf_key<InteriorPRG, ExteriorPRG, InputT, OutputT, OutputTs...>,
+    std::enable_if_t<!dpf::dpf_key<InteriorPRG, ExteriorPRG, InputT, OutputT,
+        OutputTs...>::is_multilevel>>
+{
+    using dpf_type = dpf::dpf_key<InteriorPRG, ExteriorPRG, InputT, OutputT, OutputTs...>;
+    using interior_node = typename dpf_type::interior_node;
+    using leaf_tuple = typename dpf_type::leaf_tuple;
+    using beaver_tuple = typename dpf_type::beaver_tuple;
+
+    static dpf_type from_json(const nlohmann::json & j)
+    {
+        interior_node root;
+        j.at("root").get_to(root);
+        std::array<interior_node, dpf_type::depth> correction_words;
+        j.at("correction_words").get_to(correction_words);
+        std::array<psnip_uint8_t, dpf_type::depth> correction_advice;
+        j.at("correction_advice").get_to(correction_advice);
+        leaf_tuple leaves;
+        j.at("leaves").get_to(leaves);
+        std::string wildcard_mask_str;
+        j.at("wildcards").get_to(wildcard_mask_str);
+        beaver_tuple beavers;
+        j.at("beavers").get_to(beavers);
+
+        return dpf_type{
+            root,
+            correction_words,
+            correction_advice,
+            leaves,
+            std::bitset<std::tuple_size_v<leaf_tuple>>(wildcard_mask_str),
+            beavers
+        };
+    }
+
+    static void to_json(nlohmann::json & j, const dpf_type & dpf)  // NOLINT(runtime/references)
+    {
+        j = nlohmann::json{
+            {"root", dpf.root()},
+            {"correction_words", dpf.correction_words()},
+            {"correction_advice", dpf.correction_advice()},
+            {"leaves", dpf.mutable_leaf_tuple()},
+            {"wildcards", dpf.mutable_wildcard_mask()},
+            {"beavers", dpf.mutable_beaver_tuple()}
+        };
+    }
+};
+
+// Multi-level / comparison key (`at<>` and/or a `cmp` channel). Round-trips
+// the public tree (root, CWs, advice) and the comparison channel (cmp meta,
+// value CWs, `cw_last`, and this party's `cmp_addend` share). Leaf outputs are
+// not yet serialized here, so this path currently supports comparison-only
+// keys (`num_outputs == 0`, e.g. `make_dpf(x, dpf::lt(...))`).
+template <typename InteriorPRG,
+          typename ExteriorPRG,
+          typename InputT,
+          typename OutputT,
+          typename ...OutputTs>
+struct adl_serializer<dpf::dpf_key<InteriorPRG, ExteriorPRG, InputT, OutputT, OutputTs...>,
+    std::enable_if_t<dpf::dpf_key<InteriorPRG, ExteriorPRG, InputT, OutputT,
+        OutputTs...>::is_multilevel>>
+{
+    using dpf_type = dpf::dpf_key<InteriorPRG, ExteriorPRG, InputT, OutputT, OutputTs...>;
+    using interior_node = typename dpf_type::interior_node;
+    using input_type = typename dpf_type::input_type;
+
+    static dpf_type from_json(const nlohmann::json & j)
+    {
+        static_assert(dpf_type::num_outputs == 0,
+            "dpf::json round-trip currently supports comparison-only "
+            "multi-level keys (no leaf outputs)");
+        interior_node root;
+        j.at("root").get_to(root);
+        typename dpf_type::correction_words_array correction_words;
+        j.at("correction_words").get_to(correction_words);
+        typename dpf_type::correction_advice_array correction_advice;
+        j.at("correction_advice").get_to(correction_advice);
+
+        dpf::detail::cmp_meta cmp;
+        j.at("cmp").get_to(cmp);
+        typename dpf_type::value_cw_array value_cws;
+        j.at("value_cw").get_to(value_cws);
+        uint64_t cw_last = j.at("cw_last").template get<uint64_t>();
+        uint64_t cmp_addend = j.at("cmp_addend").template get<uint64_t>();
+
+        typename dpf_type::leaf_wrapper_tuple leaves{};
+        input_type offset_share{};
+        typename dpf_type::addend_tuple addends{};
+
+        return dpf_type{root, correction_words, correction_advice,
+            std::move(leaves), offset_share, cmp, value_cws, cw_last,
+            cmp_addend, addends};
+    }
+
+    static void to_json(nlohmann::json & j, const dpf_type & dpf)  // NOLINT(runtime/references)
+    {
+        static_assert(dpf_type::num_outputs == 0,
+            "dpf::json round-trip currently supports comparison-only "
+            "multi-level keys (no leaf outputs)");
+        j = nlohmann::json{
+            {"root", dpf.root()},
+            {"correction_words", dpf.correction_words()},
+            {"correction_advice", dpf.correction_advice()},
+            {"cmp", dpf.cmp()},
+            {"value_cw", dpf.value_cw()},
+            {"cw_last", static_cast<uint64_t>(dpf.cw_last())},
+            {"cmp_addend", static_cast<uint64_t>(dpf.cmp_addend())}
+        };
+    }
+};
+
+}  // namespace nlohmann
+
+namespace dpf
+{
+
+namespace json
+{
+
+template <typename DpfKey>
+static std::string to_json(const DpfKey & dpf)
+{
+    nlohmann::json json = dpf;
+    return json.dump();
+}
+
+template <typename DpfType>
+static auto from_json(const std::string & json_string)
+{
+    nlohmann::json json = nlohmann::json::parse(json_string);
+    return static_cast<DpfType>(json);
+}
+
+}  // namespace json
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_JSON_HPP__

include/dpf/keyword.hpp

@@ -0,0 +1,546 @@
+/// @file dpf/keyword.hpp
+/// @brief defines `dpf::keyword` and associated helpers
+/// @details A `dpf::keyword` is an integer representation of a fixed-length
+///          string over a given alphabet. The integer representation uses
+///          the fewest bits possible for the given string length and alphabet
+///          size and uses an encoding that preserves the lexicographic
+///          ordering of the underlying strings. This type is inteded to be
+///          used as an input type for a DPF and, as such, specializes
+///          `dpf::utils::bitlength_of`, `dpf::utils::msb_of`, and
+///          `dpf::utils::countl_zero_symmetric_difference`. When used as a
+///          DPF input type, the aforementioned properties of the encoding
+///          equate to minimizing the DPF tree depth and maximizing the
+///          potential for effective memoization in the context of
+///          `eval_point`- and `eval_sequence`-based evaluation. As a discreet
+///          (non-numeric) type, `dpf::keyword`s are not optimized for use in
+///          `eval_interval`-based evaluation.
+///
+///          This file also defines the `dpf::alphabets` namespace, which
+///          defines various alphabets of interest, including the printable
+///          ASCII characters (`dpf::alphabets::printable_ascii`), lowercase
+///          Roman letters (`dpf::alphabets::lowercase_alpha`), lowercase
+///          hexademical digits (`dpf::alpbabets::lowercase_hex`), among
+///          others.
+/// @author Ryan Henry <ryan.henry@ucalgary.ca>
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_KEYWORD_HPP__
+#define LIBDPF_INCLUDE_DPF_KEYWORD_HPP__
+
+#include <cstddef>
+#include <cmath>
+#include <type_traits>
+#include <limits>
+#include <string>
+#include <string_view>
+#include <memory>
+#include <iterator>
+#include <algorithm>
+#include <array>
+#include <istream>
+#include <ostream>
+#include <stdexcept>
+
+#include "hedley/hedley.h"
+
+#include "dpf/utils.hpp"
+#include "dpf/modint.hpp"
+
+namespace dpf
+{
+
+/// @brief defines common alphabets for convenient use with `dpf::keyword`
+/// N.B.: The first char in an alphabet has value `0`. All strings will be
+///       implicitly padded to the max length by prepending this char. For
+///       strings, it should typically be `\0`; for numbers, the zero digit.
+namespace alphabets
+{
+    /// @brief the printable ASCII chars
+    inline constexpr char printable_ascii[] = "\0 !\"#$%&'()*+,-./0123456789:"
+                                               ";<=>?@ABCDEFGHIJKLMNOPQRSTUV"
+                                               "WXYZ[\\]^_`abcdefghijklmnopq"
+                                               "rstuvwxyz{|}~";
+    /// @brief the extended ASCII characters (0-255) using hexadecimal escape sequences with lowercase letters
+    inline constexpr char extended_ascii[] = 
+        "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+        "\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+        " !\"#$%&'()*+,-./0123456789:;<=>?"
+        "@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_"
+        "`abcdefghijklmnopqrstuvwxyz{|}~\x7f"
+        "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+        "\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+        "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+        "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+        "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+        "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+        "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+        "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff";
+    /// @brief the lowercase Roman alphabet
+    inline constexpr char lowercase_alpha[] = "\0abcdefghijklmnopqrstuvwxyz";
+    /// @brief the lowercase and uppercase Roman alphabet
+    inline constexpr char alpha[] = "\0abcdefghijklmnopqrstuvwxyzABCDEFGHIJKL"
+                                     "MNOPQRSTUVWXYZ";
+    /// @brief the lowercase and uppercase Roman alphabet plus digits 0-9
+    inline constexpr char alphanumeric[] = "\0abcdefghijklmnopqrstuvwxyzABCDE"
+                                            "FGHIJKLMNOPQRSTUVWXYZ0123456789";
+    /// @brief the lowercase Roman alphabet plus digits 0-9
+    inline constexpr char lowercase_alphanumeric[] = "\0abcdefghijklmnopqrstu"
+                                                      "vwxyz0123456789";
+    /// @brief hashtags
+    inline constexpr char hashtag[] = "\0abcdefghijklmnopqrstuvwxyz#-";
+    /// @brief binary
+    inline constexpr char binary[] = "01";
+    /// @brief octal
+    inline constexpr char octal[] = "01234567";
+    /// @brief decimal
+    inline constexpr char decimal[] = "0123456789";
+    /// @brief hex w/ lowercase letters
+    inline constexpr char hex[] = "0123456789abcdef";
+    /// @brief hex w/ uppercase letters
+    inline constexpr char uppercase_hex[] = "0123456789ABCDEF";
+    /// @brief base64 digits
+    inline constexpr char base64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijk"
+                                      "lmnopqrstuvwxyz0123456789+/=";
+    /// @brief URL-safe base64 digits
+    inline constexpr char url64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijk"
+                                     "lmnopqrstuvwxyz0123456789-_";
+    /// @brief URI alphabet
+    inline constexpr char uri[] = "\0:/?#[]@"       // gen-delims
+                                   "!$&'()*+,;="    // sub-delims
+                                   "abcdefghijklmnopqrstuvwxyz"
+                                   "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+                                   "0123456789-._~%";
+    /// @brief lowercase email address
+    inline constexpr char email[] = "\0abcdefghijklmnopqrstuvwxyz"
+                                     "0123456789.-_@";
+    /// @brief Distinct UTF-8 bytes of a small emoji sample.
+    ///        A `char` alphabet cannot store one symbol per emoji: those
+    ///        code points are several bytes long, and repeated bytes would
+    ///        make `find` non-injective. This list keeps first-seen bytes only.
+    inline constexpr char emoji[] =
+        "\x00\xf0\x9f\x98\x80\x81\x82\xa4\xa3\x83\x84\x85\x86\x89\x8a\x8b"
+        "\x8e\x8d\xa5\xb0\x8f";
+}  // namespace alphabets
+
+template <std::size_t MaxLen,
+          // std::size_t MinLen = 1,
+          typename CharT = char,
+          const CharT * Alphabet = alphabets::printable_ascii,
+          typename Traits = std::char_traits<CharT>,
+          typename Allocator = std::allocator<CharT>>
+class basic_fixed_length_string : public dpf::modint<static_cast<std::size_t>(std::ceil(MaxLen*std::log2(std::basic_string_view<CharT, Traits>(&Alphabet[1]).size() + 1)))>
+{
+  public:
+    using string_view = std::basic_string_view<CharT, Traits>;
+
+    /// @brief radix used by the integer representation of the string
+    static constexpr std::size_t radix = string_view(&Alphabet[1]).size() + 1;
+
+    /// @brief the alphabet over which the string is constructed
+    static constexpr string_view alphabet = string_view(Alphabet, radix);
+
+    // /// @brief the minimum explicit length of a string
+    // /// @details strings of length at least `min_length` are padded to
+    // ///          `max_length` with the 0th character in the alphabet; strings
+    // ///          of length less than `min_length` are out of range
+    // static constexpr std::size_t min_length = MinLen;
+
+    /// @brief the (maximum) length of a string
+    static constexpr std::size_t max_length = MaxLen;
+
+    /// @brief the number of bits needed to uniquely represent any string
+    ///        of length at least `min_length` and at most `max_length` over
+    ///        `alphabet`
+    static constexpr std::size_t bits
+        = std::ceil((max_length)*std::log2(radix));
+
+    static_assert(!alphabet.empty(), "alphabet must be non-empty");
+    // static_assert(MinLen != 0, "minimum string length must be positive");
+    static_assert(MaxLen != 0, "maximum string length must be positive");
+    // static_assert(MinLen <= MaxLen, "minimum string length must be less than or equal to maximum");
+
+  private:
+    using parent = dpf::modint<bits>;
+
+  public:
+    /// @brief the primitive integral type used to represent the string
+    using integral_type = dpf::utils::nonvoid_integral_type_from_bitlength_t<bits>;
+
+    /// @brief construct the `basic_fixed_length_string`
+    /// @{
+
+    /// @brief default constructor
+    /// @details Constructs the `basic_fixed_length_string` with a value
+    ///          corresponding to the empty string.
+    constexpr basic_fixed_length_string() noexcept = default;
+
+    /// @brief copy constructor
+    /// @details Constructs the `basic_fixed_length_string` with a value
+    ///          copied from another `basic_fixed_length_string`.
+    constexpr
+    basic_fixed_length_string(const basic_fixed_length_string &)
+    noexcept = default;
+
+    /// @brief move constructor
+    /// @details Constructs the `basic_fixed_length_string` from another
+    ///          `basic_fixed_length_string` using move semantics.
+    constexpr
+    basic_fixed_length_string(basic_fixed_length_string &&)
+    noexcept = default;
+
+    /// @brief value constructor
+    /// @details Constructs a `basic_fixed_length_string` whose value is
+    ///          initialized to the integer representation of `str`.
+    /// @param str the string to initialize with
+    constexpr
+    // cppcheck-suppress noExplicitConstructor
+    basic_fixed_length_string(string_view str)  // NOLINT(runtime/explicit)
+      : parent::modint(encode_(str)) { }
+
+    /// @brief value constructor
+    /// @details Constructs a `basic_fixed_length_string` whose value is
+    ///          initialized to the integer representation of `str`.
+    /// @param str the string to initialize with
+    constexpr
+    // cppcheck-suppress noExplicitConstructor
+    basic_fixed_length_string(const CharT * str)  // NOLINT(runtime/explicit)
+      : parent::modint(encode_(str)) { }
+
+    /// @}
+
+    /// @brief assign the `basic_fixed_length_string`
+    /// @{
+
+    /// @brief value assignment
+    /// @details Sets the value of this `basic_fixed_length_string` to
+    ///          the integer representation of `str`.
+    /// @param str the string to assign with
+    constexpr basic_fixed_length_string & operator=(string_view str)
+    {
+        parent::operator=(encode_(str));
+        return *this;
+    }
+
+    /// @brief copy assignment
+    /// @details Assigns the `basic_fixed_length_string` with a value
+    ///          copied from another `basic_fixed_length_string`.
+    constexpr basic_fixed_length_string &
+    operator=(const basic_fixed_length_string &) = default;
+
+    /// @brief move assignment
+    /// @details Assigns the `basic_fixed_length_string` from another
+    ///          `basic_fixed_length_string` using move semantics.
+    constexpr basic_fixed_length_string &
+    operator=(basic_fixed_length_string &&) noexcept = default;
+
+    /// @}
+
+    ~basic_fixed_length_string() = default;
+
+    HEDLEY_PURE
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr basic_fixed_length_string operator~() const noexcept
+    {
+        return basic_fixed_length_string{parent::operator~()};
+    }
+
+    /// @brief recreates the string representation of this
+    ///        `basic_fixed_length_string`
+    /// @complexity `O(MaxLen)` where `MaxLen` is the maximum string length
+    constexpr
+    operator std::basic_string<CharT, Traits, Allocator>() const
+    {
+        auto tmp = parent::reduced_value();
+        std::basic_string<CharT, Traits, Allocator> rev;
+        rev.reserve(max_length);
+        while (tmp != 0)
+        {
+            rev.push_back(alphabet[static_cast<std::size_t>(tmp % radix)]);
+            tmp /= radix;
+        }
+        std::reverse(std::begin(rev), std::end(rev));
+        return rev;
+    }
+
+  private:
+    constexpr
+    // cppcheck-suppress noExplicitConstructor
+    basic_fixed_length_string(integral_type val)  // NOLINT(runtime/explicit)
+    noexcept
+      : parent::modint(val) { }
+
+    constexpr basic_fixed_length_string(parent val) noexcept
+      : parent::modint(val) { }
+
+    /// @brief converts a string of length at-most `max_length` over
+    ///        `alphabet` into an integer
+    /// @throws `std::length_error` if `str` exceeds `max_length`
+    /// @throws `std::domain_error` if `str` contains a char not in `alphabet`
+    HEDLEY_ALWAYS_INLINE
+    static constexpr integral_type encode_(string_view str)
+    {
+        constexpr auto npos = string_view::npos;
+        using std::string_literals::operator""s;
+
+        utils::constexpr_maybe_throw<std::length_error>(
+            str.size() > max_length,
+            "str.size() cannot exceed max_length");
+
+        integral_type val{0};
+        for (CharT c : str)
+        {
+            auto next_digit = digit_of_(c);
+            utils::constexpr_maybe_throw<std::domain_error>(
+                next_digit == npos,
+                "str contains a disallowed char");
+            val = val * radix + next_digit;
+        }
+        return val;
+    }
+
+    /// @brief Index of `c` in `alphabet`, or `npos` when `c` is absent.
+    ///        Byte alphabets use a 256-entry table; wider character types scan.
+    static constexpr std::size_t digit_of_(CharT c)
+    {
+        constexpr auto missing = string_view::npos;
+        if constexpr (sizeof(CharT) == 1)
+        {
+            constexpr auto table = []()
+            {
+                std::array<std::size_t, 256> digits{};
+                for (auto & slot : digits)
+                    slot = missing;
+                for (std::size_t i = 0; i < radix; ++i)
+                {
+                    auto uc = static_cast<unsigned char>(alphabet[i]);
+                    if (digits[uc] == missing)
+                        digits[uc] = i;
+                }
+                return digits;
+            }();
+            return table[static_cast<unsigned char>(c)];
+        }
+        else
+        {
+            return alphabet.find(c);
+        }
+    }
+
+    /// @brief performs stream input and output on
+    ///        `dpf::basic_fixed_length_string`s
+    /// @{
+
+    /// @brief Writes the decoded string, not the packed integer.
+    friend std::basic_ostream<CharT, Traits> &
+    operator<<(std::basic_ostream<CharT, Traits> & os,
+        const basic_fixed_length_string & k)
+    {
+        return os << static_cast<std::basic_string<CharT, Traits, Allocator>>(k);
+    }
+
+    /// @brief Reads a whitespace-delimited token and encodes it.
+    friend std::basic_istream<CharT, Traits> &
+    operator>>(std::basic_istream<CharT, Traits> & is,
+        basic_fixed_length_string & k)
+    {
+        std::basic_string<CharT, Traits, Allocator> tmp;
+        if (!(is >> tmp))
+            return is;
+        try
+        {
+            k = basic_fixed_length_string(string_view(tmp));
+        }
+        catch (const std::exception &)
+        {
+            is.setstate(std::ios::failbit);
+        }
+        return is;
+    }
+
+    /// @}
+
+    friend struct utils::countl_zero_symmetric_difference<basic_fixed_length_string>;
+    friend struct utils::msb_of<basic_fixed_length_string>;
+    friend struct utils::mod_pow_2<basic_fixed_length_string>;
+    friend struct utils::make_from_integral_value<basic_fixed_length_string>;
+};  // class dpf::basic_fixed_length_string
+
+/// @brief instantiation of the `dpf::basic_fixed_length_string` class that
+///        uses `char` (i.e., bytes) as its **character type**, with its
+///        default `char_traits` and `allocator` types (see
+///        `dpf::basic_fixed_length_string` for more info on the template).
+template <std::size_t MaxLen,
+          const char * Alphabet = alphabets::lowercase_alpha>
+using keyword = basic_fixed_length_string<MaxLen, char, Alphabet>;
+
+/// @brief convert a `dpf::basic_fixed_length_string` to a `std::basic_string`
+/// @details Uses a `static_cast` to convert `str` to recreate the string
+///          representation of a `basic_fixed_length_string`
+/// @complexity `O(MaxLen)` where `MaxLen` is the maximum string length
+template <std::size_t MaxLen,
+          typename CharT,
+          const CharT * Alphabet,
+          typename Traits = std::char_traits<CharT>,
+          typename Allocator = std::allocator<CharT>>
+static constexpr std::basic_string<CharT, Traits, Allocator>
+to_string(basic_fixed_length_string<MaxLen, CharT, Alphabet, Traits, Allocator>
+    str)
+{
+    return static_cast<std::basic_string<CharT, Traits, Allocator>>(str);
+}
+
+namespace utils
+{
+
+/// @brief specializes `dpf::bitlength_of` for `dpf::basic_fixed_length_string`
+template <std::size_t MaxLen,
+          typename CharT,
+          const CharT * Alpha,
+          typename Traits,
+          typename Alloc>
+struct bitlength_of<
+    dpf::basic_fixed_length_string<MaxLen, CharT, Alpha, Traits, Alloc>>
+  : public std::integral_constant<std::size_t,
+    dpf::basic_fixed_length_string<MaxLen, CharT, Alpha, Traits, Alloc>::bits> { };
+
+/// @brief specializes `dpf::msb_of` for `dpf::basic_fixed_length_string`
+template <std::size_t MaxLen,
+          typename CharT,
+          const CharT * Alpha,
+          typename Traits,
+          typename Alloc>
+struct msb_of<dpf::basic_fixed_length_string<MaxLen, CharT, Alpha, Traits, Alloc>>
+{
+    using T = dpf::basic_fixed_length_string<MaxLen, CharT, Alpha, Traits, Alloc>;
+    using U = typename T::integral_type;
+    static constexpr T value = U{1} << bitlength_of_v<T> - 1ul;
+};
+
+/// @brief specializes `dpf::countl_zero_symmetric_difference` for
+///        `dpf::basic_fixed_length_string`
+template <std::size_t MaxLen,
+          typename CharT,
+          const CharT * Alpha,
+          typename Traits,
+          typename Alloc>
+struct countl_zero_symmetric_difference<dpf::basic_fixed_length_string<MaxLen, CharT, Alpha, Traits, Alloc>>
+  : countl_zero_symmetric_difference<typename dpf::basic_fixed_length_string<MaxLen, CharT, Alpha, Traits, Alloc>::parent>
+{ };
+
+template <std::size_t MaxLen,
+          typename CharT,
+          const CharT * Alpha,
+          typename Traits,
+          typename Alloc>
+struct mod_pow_2<dpf::basic_fixed_length_string<MaxLen, CharT, Alpha, Traits, Alloc>>
+  : mod_pow_2<typename dpf::basic_fixed_length_string<MaxLen, CharT, Alpha, Traits, Alloc>::parent>
+{ };
+
+template <std::size_t MaxLen,
+          typename CharT,
+          const CharT * Alpha,
+          typename Traits,
+          typename Alloc>
+struct make_from_integral_value<dpf::basic_fixed_length_string<MaxLen, CharT, Alpha, Traits, Alloc>>
+{
+    using T = dpf::basic_fixed_length_string<MaxLen, CharT, Alpha, Traits, Alloc>;
+    using integral_type = integral_type_from_bitlength_t<bitlength_of_v<T>>;
+    constexpr T operator()(integral_type val) const noexcept
+    {
+        return T{val};
+    }
+};
+
+}  // namespace utils
+
+}  // namespace dpf
+
+namespace std
+{
+
+/// @brief specializes `std::numeric_limits` for CV-qualified `dpf::keyword`s
+/// @{
+
+/// @details specializes `std::numeric_limits` for `dpf::basic_fixed_length_string<MaxLen, CharT, Alpha, Traits, Alloc>`
+template <std::size_t MaxLen,
+          typename CharT,
+          const CharT * Alpha,
+          typename Traits,
+          typename Alloc>
+class numeric_limits<dpf::basic_fixed_length_string<MaxLen, CharT, Alpha, Traits, Alloc>>
+{
+  public:
+    using keyword_type = dpf::basic_fixed_length_string<MaxLen, CharT, Alpha, Traits, Alloc>;
+    static constexpr bool is_specialized = true;
+    static constexpr bool is_signed = false;
+    static constexpr bool is_integer = true;
+    static constexpr bool is_exact = true;
+    static constexpr bool has_infinity = false;
+    static constexpr bool has_quiet_NaN = false;
+    static constexpr bool has_signaling_NaN = false;
+    static constexpr std::float_denorm_style has_denorm = std::denorm_absent;
+    static constexpr bool has_denorm_loss = false;
+    static constexpr std::float_round_style round_style = std::round_toward_zero;
+    static constexpr bool is_iec559 = false;
+    static constexpr bool is_bounded = true;
+    static constexpr bool is_modulo = true;
+    static constexpr int digits = keyword_type::bits;
+    static constexpr int digits10 = static_cast<int>((static_cast<unsigned long long>(keyword_type::bits) * 30103ull) / 100000ull);
+    static constexpr int max_digits10 = 0;
+    static constexpr int radix = 2;
+    static constexpr int min_exponent = 0;
+    static constexpr int max_exponent = 0;
+    static constexpr int min_exponent10 = 0;
+    static constexpr int max_exponent10 = 0;
+    static constexpr bool traps
+        = std::numeric_limits<typename keyword_type::integral_type>::traps;
+    static constexpr bool tinyness_before = false;
+
+    static constexpr keyword_type min() noexcept { return keyword_type{""}; }
+    static constexpr keyword_type lowest() noexcept { return keyword_type{""}; }
+    static constexpr keyword_type max() noexcept { return ~keyword_type{""}; }
+    static constexpr keyword_type epsilon() noexcept { return 0; }
+    static constexpr keyword_type round_error() noexcept { return 0; }
+    static constexpr keyword_type infinity() noexcept { return 0; }
+    static constexpr keyword_type quiet_NaN() noexcept { return 0; }
+    static constexpr keyword_type signaling_NaN() noexcept { return 0; }
+    static constexpr keyword_type denorm_min() noexcept { return 0; }
+};
+
+/// @details specializes `std::numeric_limits` for `dpf::basic_fixed_length_string<MaxLen, CharT, Alpha, Traits, Alloc> const`
+template <std::size_t MaxLen,
+          typename CharT,
+          const CharT * Alpha,
+          typename Traits,
+          typename Alloc>
+class numeric_limits<dpf::basic_fixed_length_string<MaxLen, CharT, Alpha, Traits, Alloc> const>
+  : public numeric_limits<dpf::basic_fixed_length_string<MaxLen, CharT, Alpha, Traits, Alloc>> {};
+
+/// @details specializes `std::numeric_limits` for
+///          `dpf::basic_fixed_length_string<MaxLen, CharT, Alpha, Traits, Alloc> volatile`
+template <std::size_t MaxLen,
+          typename CharT,
+          const CharT * Alpha,
+          typename Traits,
+          typename Alloc>
+class numeric_limits<dpf::basic_fixed_length_string<MaxLen, CharT, Alpha, Traits, Alloc> volatile>
+  : public numeric_limits<dpf::basic_fixed_length_string<MaxLen, CharT, Alpha, Traits, Alloc>> {};
+
+/// @details specializes `std::numeric_limits` for
+///          `dpf::basic_fixed_length_string<MaxLen, CharT, Alpha, Traits, Alloc> const volatile`
+template <std::size_t MaxLen,
+          typename CharT,
+          const CharT * Alpha,
+          typename Traits,
+          typename Alloc>
+class numeric_limits<dpf::basic_fixed_length_string<MaxLen, CharT, Alpha, Traits, Alloc> const volatile>
+  : public numeric_limits<dpf::basic_fixed_length_string<MaxLen, CharT, Alpha, Traits, Alloc>> {};
+
+/// @}
+
+}  // namespace std
+
+#endif  // LIBDPF_INCLUDE_DPF_KEYWORD_HPP__

include/dpf/leaf_node.hpp

@@ -0,0 +1,485 @@
+/// @file dpf/leaf_node.hpp
+/// @brief
+/// @details
+/// @author Ryan Henry <ryan.henry@ucalgary.ca>
+/// @author Christopher Jiang <christopher.jiang@ucalgary.ca>
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_LEAF_NODE_HPP__
+#define LIBDPF_INCLUDE_DPF_LEAF_NODE_HPP__
+
+#include "hedley/hedley.h"
+
+#include <cstddef>
+#include <cmath>
+#include <cstring>
+#include <type_traits>
+#include <utility>
+#include <memory>
+#include <functional>
+#include <tuple>
+#include <atomic>
+#include <array>
+
+#include "simde/simde/x86/avx2.h"
+
+#include "dpf/bit.hpp"
+#include "dpf/packed_lane.hpp"
+#include "dpf/xor_wrapper.hpp"
+#include "dpf/wildcard.hpp"
+#include "dpf/leaf_arithmetic.hpp"
+#include "dpf/utils.hpp"
+#include "dpf/random.hpp"
+
+namespace dpf
+{
+
+/// @brief `value` is `true` if multiple leaves are packed into each leaf node
+template <typename OutputT,
+          typename NodeT>
+using is_packable = std::bool_constant<
+    std::less<>{}(utils::bitlength_of_output_v<OutputT, NodeT>, utils::bitlength_of_output_v<NodeT, NodeT>) &&
+    std::equal_to<>{}(utils::bitlength_of_output_v<NodeT, NodeT> % utils::bitlength_of_output_v<OutputT, NodeT>, 0)>;
+
+template <typename OutputT,
+          typename NodeT>
+static constexpr bool is_packable_v = is_packable<OutputT, NodeT>::value;
+
+template <typename OutputT,
+          typename NodeT>
+struct outputs_per_leaf
+    : public std::integral_constant<std::size_t,
+        !is_packable_v<OutputT, NodeT> ? 1 :
+            utils::bitlength_of_output_v<NodeT, NodeT> / utils::bitlength_of_output_v<OutputT, NodeT>> { };
+
+template <typename OutputT,
+          typename NodeT>
+static constexpr std::size_t outputs_per_leaf_v
+    = outputs_per_leaf<OutputT, NodeT>::value;
+
+template <typename OutputT,
+          typename NodeT>
+static constexpr std::size_t lg_outputs_per_leaf_v
+    = std::log2(outputs_per_leaf<OutputT, NodeT>::value);
+
+template <typename OutputT,
+          typename NodeT>
+struct block_length_of_leaf
+    : std::integral_constant<std::size_t, is_packable_v<OutputT, NodeT> ? 1 :
+                                    utils::quotient_ceiling(
+                                        utils::bitlength_of_output_v<OutputT, NodeT>,
+                                        utils::bitlength_of_output_v<NodeT, NodeT>)
+                                    >{ };
+
+template <typename OutputT,
+          typename NodeT>
+static constexpr std::size_t block_length_of_leaf_v
+    = block_length_of_leaf<OutputT, NodeT>::value;
+
+template <typename OutputT,
+          typename NodeT,
+          typename InputT>
+constexpr std::size_t offset_within_block(InputT x) noexcept
+{
+    constexpr auto mod = utils::mod_pow_2<InputT>{};
+    return mod(x, dpf::lg_outputs_per_leaf_v<OutputT, NodeT>);
+}
+
+template <std::size_t I,
+          typename N,
+          std::size_t I_,
+          typename OutputsT>
+struct block_offset_of_leaf
+{
+    static constexpr std::size_t value = dpf::block_length_of_leaf_v<std::tuple_element_t<I_, OutputsT>, N>
+        + block_offset_of_leaf<I, N, I_+1, OutputsT>::value;
+};
+
+template <std::size_t I,
+          typename N,
+          typename OutputsT>
+struct block_offset_of_leaf<I, N, I, OutputsT>
+{
+    static constexpr std::size_t value = 0;
+};
+
+template <std::size_t I, typename N, typename OutputsT>
+inline constexpr std::size_t block_offset_of_leaf_v
+    = block_offset_of_leaf<I, N, 0, OutputsT>::value;
+
+template <std::size_t First, std::size_t ...Rest>
+struct const_min_size
+{
+    static constexpr std::size_t value
+        = (First < const_min_size<Rest...>::value)
+            ? First : const_min_size<Rest...>::value;
+};
+template <std::size_t Only>
+struct const_min_size<Only>
+{
+    static constexpr std::size_t value = Only;
+};
+
+template <std::size_t First, std::size_t ...Rest>
+struct const_max_size
+{
+    static constexpr std::size_t value
+        = (First > const_max_size<Rest...>::value)
+            ? First : const_max_size<Rest...>::value;
+};
+template <std::size_t Only>
+struct const_max_size<Only>
+{
+    static constexpr std::size_t value = Only;
+};
+
+/// PRG position span covering output indices `Is...` of `OutputsTuple`.
+/// `is_contiguous` is true when the selected outputs occupy a hole-free
+/// range, so one `ExteriorPRG::eval(..., count, pos_min)` produces every
+/// leaf mask.
+template <typename NodeT,
+          typename OutputsTuple,
+          std::size_t ...Is>
+struct leaf_prg_range
+{
+    static constexpr std::size_t pos_min
+        = const_min_size<block_offset_of_leaf_v<Is, NodeT, OutputsTuple>...>::value;
+    static constexpr std::size_t pos_end
+        = const_max_size<(block_offset_of_leaf_v<Is, NodeT, OutputsTuple>
+            + block_length_of_leaf_v<std::tuple_element_t<Is, OutputsTuple>, NodeT>)...>::value;
+    static constexpr std::size_t count = pos_end - pos_min;
+    static constexpr std::size_t needed
+        = (block_length_of_leaf_v<std::tuple_element_t<Is, OutputsTuple>, NodeT> + ...);
+    static constexpr bool is_contiguous = (count == needed);
+};
+
+template <typename NodeT,
+          typename OutputT,
+          std::size_t block_len = block_length_of_leaf_v<OutputT, NodeT>>
+struct leaf_node
+{
+    static_assert(block_len == block_length_of_leaf_v<OutputT, NodeT>);
+    using type = std::array<NodeT, block_len>;
+};
+
+template <typename NodeT,
+          typename OutputT>
+struct leaf_node<NodeT, OutputT, 1>
+{
+    static_assert(1 == block_length_of_leaf_v<OutputT, NodeT>);
+    using type = NodeT;
+};
+
+template <typename NodeT,
+          typename OutputT>
+using leaf_node_t = typename leaf_node<NodeT, OutputT>::type;
+
+template <typename NodeT,
+          typename OutputT,
+          typename ...OutputTs>
+struct leaf_tuple
+{
+    using type = std::tuple<leaf_node_t<NodeT, OutputT>,
+                            leaf_node_t<NodeT, OutputTs>...>;
+};
+template <typename NodeT,
+          typename OutputT,
+          typename ...OutputTs>
+using leaf_tuple_t = typename leaf_tuple<NodeT, OutputT, OutputTs...>::type;
+
+template <bool isWildcard,
+          typename NodeT,
+          typename OutputT>
+struct beaver final { char c = '\0'; };
+
+template <typename NodeT,
+          typename OutputT>
+struct beaver<true, NodeT, OutputT> final
+{
+    using LeafT = dpf::leaf_node_t<NodeT, OutputT>;
+    OutputT output_blind;
+    LeafT vector_blind;
+    LeafT blinded_vector;
+};
+
+template <typename NodeT,
+          typename OutputT,
+          typename ...OutputTs>
+struct beaver_tuple
+{
+    using type = std::tuple<beaver<is_wildcard_v<OutputT>, NodeT, concrete_type_t<OutputT>>,
+                              beaver<is_wildcard_v<OutputTs>, NodeT, concrete_type_t<OutputTs>>...>;
+};
+template <typename NodeT,
+          typename OutputT,
+          typename ...OutputTs>
+using beaver_tuple_t = typename beaver_tuple<NodeT, OutputT, OutputTs...>::type;
+
+template <typename NodeT,
+          typename OutputT>
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+HEDLEY_PURE
+static OutputT extract_leaf(const leaf_node_t<NodeT, OutputT> & leaf, std::size_t x) noexcept
+{
+    auto off = offset_within_block<OutputT, NodeT>(x);
+
+    OutputT y;
+    if constexpr (utils::is_packed_subbyte_v<OutputT>)
+    {
+        y = packed::extract_lane<OutputT>(leaf, off);
+    }
+    else
+    {
+        std::memcpy(&y,
+            reinterpret_cast<const unsigned char *>(std::addressof(leaf))
+                + off * sizeof(OutputT),
+            sizeof(y));
+    }
+
+    return y;
+}
+
+// Inserts y at correct place (based on x) within a (otherwise 0) NodeT
+template <typename NodeT,
+          typename InputT,
+          typename OutputT>
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+auto make_naked_leaf(InputT x, OutputT y) noexcept
+{
+    using leaf_type = dpf::leaf_node_t<NodeT, OutputT>;
+    auto off = offset_within_block<OutputT, NodeT>(x);
+    leaf_type Y{};
+    if constexpr (utils::is_packed_subbyte_v<OutputT>)
+    {
+        packed::deposit_lane(Y, off, y);
+    }
+    else if constexpr (!dpf::is_wildcard_v<OutputT>)
+    {
+        std::memcpy(reinterpret_cast<unsigned char *>(std::addressof(Y))
+                + off * sizeof(OutputT),
+            std::addressof(y), sizeof(OutputT));
+    }
+
+    return Y;
+}
+
+/// Address of the first `NodeT` block inside a leaf.
+/// A one-block leaf *is* a `NodeT`; a longer leaf is `std::array<NodeT, N>`.
+template <typename NodeT, typename LeafT>
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+constexpr auto * leaf_blocks(LeafT & leaf) noexcept
+{
+    if constexpr (std::is_same_v<std::remove_cv_t<LeafT>, NodeT>)
+        return std::addressof(leaf);
+    else
+        return leaf.data();
+}
+
+template <typename ExteriorPRG,
+          std::size_t I,
+          typename OutputsTuple,
+          typename InteriorBlock>
+auto make_leaf_mask_inner(const InteriorBlock & seed, std::size_t pos_base = 0)
+{
+    using node_type = typename ExteriorPRG::block_type;
+    using output_type = std::tuple_element_t<I, OutputsTuple>;
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    using leaf_type = dpf::leaf_node_t<node_type, output_type>;
+
+    auto count = dpf::block_length_of_leaf_v<output_type, node_type>;
+    auto pos = pos_base + dpf::block_offset_of_leaf_v<I, node_type, OutputsTuple>;
+    leaf_type output;
+    auto seed_ = utils::to_exterior_node<node_type>(seed);
+    ExteriorPRG::eval(seed_, leaf_blocks<node_type>(output), count,
+        static_cast<psnip_uint32_t>(pos));
+
+    return output;
+HEDLEY_PRAGMA(GCC diagnostic pop)
+}
+
+template <typename ExteriorPRG,
+          std::size_t I,
+          typename OutputsTuple,
+          typename InteriorBlock>
+auto make_leaf_mask(const InteriorBlock & seed0, const InteriorBlock & seed1,
+    std::size_t pos_base = 0)
+{
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    using output_type = concrete_type_t<std::tuple_element_t<I, OutputsTuple>>;
+
+    auto mask0 = make_leaf_mask_inner<ExteriorPRG, I, OutputsTuple, InteriorBlock>(
+        seed0, pos_base);
+    auto mask1 = make_leaf_mask_inner<ExteriorPRG, I, OutputsTuple, InteriorBlock>(
+        seed1, pos_base);
+
+    return dpf::subtract_leaf<output_type>(mask1, mask0);
+HEDLEY_PRAGMA(GCC diagnostic pop)
+}
+
+template <typename ExteriorPRG,
+          std::size_t I,
+          typename InputT,
+          typename ExteriorBlock,
+          typename ...OutputTs>
+auto make_leaf(InputT x, const ExteriorBlock & seed0, const ExteriorBlock & seed1, bool sign,
+    std::size_t pos_base, OutputTs ...ys)
+{
+    using output_tuple_type = std::tuple<OutputTs...>;
+    output_tuple_type output_tuple = std::make_tuple(ys...);
+    using output_type = std::tuple_element_t<I, output_tuple_type>;
+    output_type Y = std::get<I>(output_tuple);
+
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    using node_type = typename ExteriorPRG::block_type;
+
+    return sign ? dpf::subtract_leaf<output_type>(
+                    make_naked_leaf<node_type>(x, Y),
+                    make_leaf_mask<ExteriorPRG, I, output_tuple_type, ExteriorBlock>(
+                        seed0, seed1, pos_base))
+                : dpf::subtract_leaf<output_type>(
+                    make_leaf_mask<ExteriorPRG, I, output_tuple_type, ExteriorBlock>(
+                        seed0, seed1, pos_base),
+                    make_naked_leaf<node_type>(x, Y));
+HEDLEY_PRAGMA(GCC diagnostic pop)
+}
+
+template <typename ExteriorPRG,
+          typename InputT,
+          typename ExteriorBlock,
+          typename ...OutputTs,
+          std::size_t ...Is>
+auto make_leaves_impl(InputT x, const ExteriorBlock & seed0, const ExteriorBlock & seed1,
+    bool sign, std::size_t pos_base, std::index_sequence<Is...>, OutputTs ...ys)
+{
+    return std::make_tuple(
+        make_leaf<ExteriorPRG, Is>(x, seed0, seed1, sign, pos_base, ys...)...);
+}
+
+template <typename ExteriorPRG,
+          typename InputT,
+          typename ExteriorBlock,
+          typename OutputT,
+          typename ...OutputTs,
+          typename Indices = std::make_index_sequence<1+sizeof...(OutputTs)>>
+auto make_leaves(InputT x, const ExteriorBlock & seed0, const ExteriorBlock & seed1,
+    bool sign, std::size_t pos_base, OutputT y, OutputTs ...ys)
+{
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    using node_type = typename ExteriorPRG::block_type;
+    using leaf_type = dpf::leaf_tuple_t<node_type, OutputT, OutputTs...>;
+    using beaver_type = dpf::beaver_tuple_t<node_type, OutputT, OutputTs...>;
+HEDLEY_PRAGMA(GCC diagnostic pop)
+
+    leaf_type leaves = make_leaves_impl<ExteriorPRG>(x, seed0, seed1, sign,
+        pos_base, Indices{}, y, ys...);
+
+    // post-processing to secret-share any wildcard leaves
+    // that is, after the call to `make_leaves_impl`, any values that were
+    // should be `wildcards` will currently have a correction_word for `0` in
+    // `leaves`. Below is a glorified loop that creates two tuples from `leaves`
+    // (stored in the pair `return_tuple`). For concrete output_types, it simply copies the
+    // corresponding correction_words from `leaves`; for the `wildcard`s, it
+    // additively shares them.
+
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    std::pair<
+        std::pair<leaf_type, beaver_type>,
+        std::pair<leaf_type, beaver_type> > return_tuple;
+
+    // N.B.: Despite the nesting, the loops below advance in lockstep, making
+    // only a single pass over each of the tuples being looped over
+
+    // loop over the original inputs (to interrogate their output_types)
+    std::apply([x, &sign, &return_tuple, &leaves](auto && ...y)
+    {
+        // loop over the elements of `leaves`, our "template" for a leaf tuple
+        std::apply([x, &sign, &return_tuple, &y...](auto && ...leaf)
+        {
+            // and also over the elements of `return_tuple.first.first`, the first leaf tuple
+            std::apply([x, &sign, &return_tuple, &y..., &leaf...](auto && ...leaf0)
+            {
+                // and also `return_tuple.second.first`, the secound leaf tuple
+                std::apply([x, &sign, &return_tuple, &y..., &leaf..., &leaf0...](auto && ...leaf1)
+                {
+                    // plus `return_tuple.first.second`, the first beaver tuple
+                    std::apply([x, &sign, &return_tuple, &y..., &leaf..., &leaf0..., &leaf1...](auto && ...beaver0)
+                    {
+                        // and `return_tuple.second.second`, the secound beaver tuple
+                        std::apply([x, &sign, &y..., &leaf..., &leaf0..., &leaf1..., &beaver0...](auto && ...beaver1)
+                        {
+                            // lambda to decide whether to copy the leaf (for concrete output_types)
+                            // or whether to secret share it (for wildcard output_types)
+                            ([](auto & x, auto & y, auto & leaf, auto & leaf0, auto & leaf1, auto & beaver0, auto & beaver1, bool sign)
+                            {
+                                using output_type = typename std::decay_t<decltype(y)>;
+                                if constexpr(dpf::is_wildcard_v<output_type>)
+                                {
+                                    using concrete_type = dpf::concrete_type_t<output_type>;
+                                    // secret share the value
+                                    dpf::uniform_fill(leaf0);
+                                    leaf1 = dpf::subtract_leaf<concrete_type>(leaf, leaf0);
+                                    // also initialize the beavers
+                                    if constexpr(!dpf::utils::has_characteristic_two_v<concrete_type>
+                                        || dpf::outputs_per_leaf_v<concrete_type, node_type> > 1)
+                                    {
+                                        dpf::leaf_node_t<node_type, concrete_type> vector;
+                                        // XOR-group multiply is AND, whose unit is ~0, not ±1.
+                                        // Check the OUTPUT type: input may be modint while the
+                                        // leaf is xor_wrapper (wildcard XOR payload).
+                                        if constexpr(utils::is_xor_wrapper_v<std::decay_t<decltype(x)>> == true
+                                                  || utils::is_xor_wrapper_v<concrete_type> == true)
+                                        {
+                                            vector = make_naked_leaf<node_type>(x, concrete_type(~0));
+                                        }
+                                        else
+                                        {
+                                            vector = make_naked_leaf<node_type>(x, concrete_type(2*sign-1));
+                                        }
+
+                                        uniform_fill(beaver0.output_blind);
+                                        uniform_fill(beaver0.vector_blind);
+
+                                        uniform_fill(beaver1.output_blind);
+                                        uniform_fill(beaver1.vector_blind);
+
+                                        beaver0.blinded_vector = dpf::add_leaf<concrete_type>(vector, beaver1.vector_blind);
+                                        beaver1.blinded_vector = dpf::add_leaf<concrete_type>(vector, beaver0.vector_blind);
+
+                                        leaf0 = dpf::add_leaf<concrete_type>(leaf0,
+                                            dpf::multiply_leaf(beaver0.vector_blind, beaver1.output_blind));
+                                        leaf1 = dpf::add_leaf<concrete_type>(leaf1,
+                                            dpf::multiply_leaf(beaver1.vector_blind, beaver0.output_blind));
+                                    }
+                                }
+                                else
+                                {
+                                    // copy concrete value; beaver is a trivial type
+                                    leaf0 = leaf;
+                                    leaf1 = leaf;
+                                }
+                            }(x, y, leaf, leaf0, leaf1, beaver0, beaver1, sign), ...);
+                        }, return_tuple.second.second);
+                    }, return_tuple.first.second);
+                }, return_tuple.second.first);
+            }, return_tuple.first.first);
+        }, leaves);
+    }, std::make_tuple(y, ys...));
+
+HEDLEY_PRAGMA(GCC diagnostic pop)
+
+    return return_tuple;
+}
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_LEAF_NODE_HPP__

include/dpf/leaf_wrapper.hpp

@@ -0,0 +1,235 @@
+/// @file dpf/leaf_wrapper.hpp
+/// @brief
+/// @details
+/// @author Ryan Henry <ryan.henry@ucalgary.ca>
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_LEAF_WRAPPER_HPP__
+#define LIBDPF_INCLUDE_DPF_LEAF_WRAPPER_HPP__
+
+#include "hedley/hedley.h"
+
+#include "dpf/secret_share.hpp"
+
+namespace dpf
+{
+
+template <typename OutputT,
+          typename NodeT>
+struct leaf_wrapper
+{
+  public:
+    using leaf_type = dpf::leaf_node_t<NodeT, OutputT>;
+    using output_type = OutputT;
+
+    leaf_wrapper() = delete;
+    leaf_wrapper(leaf_type leaf, dpf::beaver<false, NodeT, OutputT> = dpf::beaver<false, NodeT, OutputT>{})
+        : leaf_{std::forward<leaf_type>(leaf)} { }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    constexpr const leaf_type & get() const noexcept { return leaf_; }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    constexpr const leaf_type & raw_leaf() const noexcept { return leaf_; }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    const dpf::beaver<false, NodeT, OutputT> & beaver() const noexcept
+    {
+        static const dpf::beaver<false, NodeT, OutputT> dummy{};
+        return dummy;
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_PURE
+    HEDLEY_NO_THROW
+    constexpr bool is_ready() const noexcept { return true; }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_PURE
+    HEDLEY_NO_THROW
+    static constexpr bool is_wildcard() noexcept { return false; }
+
+  private:
+    leaf_type leaf_;
+};
+
+// // unpacked wildcard reconstruction
+// template <typename ConcreteOutputT,
+//           typename NodeT>
+// struct leaf_wrapper<wildcard_value<ConcreteOutputT>, NodeT, false>
+// {
+//   public:
+//     using leaf_type = dpf::leaf_node_t<NodeT, ConcreteOutputT>;
+//     using output_type = ConcreteOutputT;
+
+//     leaf_wrapper() = delete;
+//     leaf_wrapper(leaf_type leaf_share, dpf::beaver<NodeT, output_type> = dpf::beaver<NodeT, output_type>{})
+//         : leaf_{leaf_share},
+//           leaf_state_(std::make_unique<std::atomic<leaf_status>>(leaf_status::notset)),
+//           ready_{false} { }
+
+//     HEDLEY_ALWAYS_INLINE
+//     const leaf_type & get() const
+//     {
+//         if (HEDLEY_UNLIKELY(!ready_))
+//         {
+//             throw std::runtime_error("offset not set");
+//         }
+//         return leaf_;
+//     }
+
+//     const leaf_type compute_and_get_leaf_share(output_type output_share)
+//     {
+//         leaf_status notset = leaf_status::notset;
+//         if (HEDLEY_UNLIKELY(!leaf_state_->compare_exchange_strong(notset,
+//             leaf_status::computing,
+//             std::memory_order_seq_cst, std::memory_order_relaxed)))
+//         {
+//             throw std::runtime_error("invalid state transition");
+//         }
+//         leaf_type tmp;
+//         std::memcpy(&tmp, &output_share, sizeof(output_type));
+//         leaf_ = add_leaf<output_type>(leaf_, tmp);
+//         leaf_state_->store(leaf_status::waiting, std::memory_order_release);
+//         return leaf_;
+//     }
+
+//     const leaf_type reconstruct_correction_word(leaf_type other_share)
+//     {
+//         leaf_status waiting = leaf_status::waiting;
+//         if (HEDLEY_UNLIKELY(!leaf_state_->compare_exchange_strong(waiting,
+//             leaf_status::computing,
+//             std::memory_order_acquire, std::memory_order_relaxed)))
+//         {
+//             throw std::runtime_error("invalid state transition");
+//         }
+//         leaf_ = add_leaf<output_type>(leaf_, other_share);
+//         ready_ = true;
+//         leaf_state_->store(leaf_status::ready, std::memory_order_relaxed);
+//         return leaf_;
+//     }
+
+//     HEDLEY_ALWAYS_INLINE
+//     HEDLEY_NO_THROW
+//     bool is_ready() const noexcept { return ready_; }
+
+//     HEDLEY_ALWAYS_INLINE
+//     HEDLEY_PURE
+//     HEDLEY_NO_THROW
+//     static constexpr bool is_wildcard() noexcept { return true; }
+
+// //   private:
+//     enum class leaf_status : psnip_uint8_t { ready = 0, waiting = 1, computing = 2, notset = 3 };
+//     leaf_type leaf_;
+//     std::unique_ptr<std::atomic<leaf_status>> leaf_state_;
+//     bool ready_;
+// };
+
+template <typename ConcreteOutputT,
+          typename NodeT>
+struct leaf_wrapper<wildcard_value<ConcreteOutputT>, NodeT>
+{
+  public:
+    using node_type = NodeT;
+    using output_type = ConcreteOutputT;
+    using leaf_type = dpf::leaf_node_t<node_type, output_type>;
+    using beaver_type = dpf::beaver<true, node_type, output_type>;
+
+    leaf_wrapper() = delete;
+    leaf_wrapper(leaf_type leaf_share, beaver_type beaver)
+        : leaf_{std::forward<leaf_type>(leaf_share)},
+          beaver_{beaver},
+          output_share_{},
+          leaf_state_{leaf_status::notset}
+    { }
+
+    HEDLEY_ALWAYS_INLINE
+    const leaf_type & get() const
+    {
+        if (HEDLEY_UNLIKELY(leaf_state_ != leaf_status::ready))
+        {
+            throw std::runtime_error("offset not set");
+        }
+        return leaf_;
+    }
+
+    const output_type compute_and_get_blinded_output_share(output_type output_share)
+    {
+        begin_transition(leaf_status::notset);
+        output_share_ = output_share;
+        auto blinded_output_share = output_share_ + beaver_.output_blind;
+        leaf_state_ = leaf_status::blinded;
+        return blinded_output_share;
+    }
+
+    /// Accept a party-tagged share; convert to additive before Beaver math.
+    template <std::size_t Party, sharing Scheme>
+    const output_type compute_and_get_blinded_output_share(
+        const secret_share<output_type, Party, Scheme> & output_share)
+    {
+        return compute_and_get_blinded_output_share(
+            output_share.as_additive().raw());
+    }
+
+    const leaf_type compute_and_get_leaf_share(output_type other_output_share)
+    {
+        begin_transition(leaf_status::blinded);
+        leaf_ = add_leaf<output_type>(leaf_, subtract_leaf<output_type>(
+            multiply_leaf(beaver_.blinded_vector, output_share_),
+            multiply_leaf(beaver_.vector_blind, other_output_share)));
+        leaf_state_ = leaf_status::waiting;
+        return leaf_;
+    }
+
+    const leaf_type reconstruct_correction_word(leaf_type other_share)
+    {
+        begin_transition(leaf_status::waiting);
+        leaf_ = add_leaf<output_type>(leaf_, other_share);
+        leaf_state_ = leaf_status::ready;
+        return leaf_;
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    bool is_ready() const noexcept { return leaf_state_ == leaf_status::ready; }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_PURE
+    HEDLEY_NO_THROW
+    static constexpr bool is_wildcard() noexcept { return true; }
+
+    // Unassigned leaf / Beaver (before online payload). get() throws until ready.
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    const leaf_type & raw_leaf() const noexcept { return leaf_; }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    const beaver_type & beaver() const noexcept { return beaver_; }
+
+  private:
+    enum class leaf_status : psnip_uint8_t { ready = 0, waiting = 1, computing = 2, blinded = 3, notset = 4 };
+
+    void begin_transition(leaf_status expected)
+    {
+        if (HEDLEY_UNLIKELY(leaf_state_ != expected))
+        {
+            throw std::runtime_error("invalid state transition");
+        }
+        leaf_state_ = leaf_status::computing;
+    }
+
+    leaf_type leaf_;
+    beaver_type beaver_;
+    output_type output_share_;
+    leaf_status leaf_state_;
+};
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_LEAF_WRAPPER_HPP__

include/dpf/literals.hpp

@@ -0,0 +1,23 @@
+#ifndef LIBDPF_INCLUDE_DPF_LITERALS_HPP__
+#define LIBDPF_INCLUDE_DPF_LITERALS_HPP__
+
+#include "dpf/twobit.hpp"
+#include "dpf/nyble.hpp"
+
+namespace dpf
+{
+
+namespace literals
+{
+
+namespace modints{} using namespace dpf::literals::modints;
+namespace xints{} using namespace dpf::literals::xints;
+namespace bitstrings{} using namespace dpf::literals::bitstrings;
+namespace twobit{} using namespace dpf::literals::twobit;
+namespace nyble{} using namespace dpf::literals::nyble;
+
+}  // namespace literals
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_LITERALS_HPP__

include/dpf/nyble.hpp

@@ -0,0 +1,209 @@
+/// @file dpf/nyble.hpp
+/// @brief `dpf::nyble`, a 4-bit output lane in the ring Z/16Z.
+/// @details Values are `0..15`. Scalar `+` and `-` wrap mod 16. A leaf node
+///          packs one lane every four bits, low nibble first. Leaf addition
+///          is not XOR and is not `add_epi8`: a carry must not cross into
+///          the neighbouring nibble. See `packed_lane_arithmetic.hpp`.
+
+#ifndef LIBDPF_INCLUDE_DPF_NYBLE_HPP__
+#define LIBDPF_INCLUDE_DPF_NYBLE_HPP__
+
+#include <cstddef>
+#include <cstdint>
+#include <istream>
+#include <limits>
+#include <ostream>
+#include <stdexcept>
+#include <string>
+#include <type_traits>
+
+#include "hedley/hedley.h"
+
+#include "dpf/utils.hpp"
+
+namespace dpf
+{
+
+/// @brief 4-bit unsigned ring element, packed one nibble per lane
+enum class nyble : std::uint8_t
+{
+    zero = 0
+};
+
+HEDLEY_CONST
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+static constexpr dpf::nyble to_nyble(unsigned value) noexcept
+{
+    return static_cast<dpf::nyble>(value & 0x0fu);
+}
+
+HEDLEY_CONST
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+static constexpr dpf::nyble to_nyble(unsigned long long value) noexcept
+{
+    return static_cast<dpf::nyble>(value & 0x0full);
+}
+
+/// @brief parse one hex digit as a nibble
+/// @throws std::domain_error if `value` is not `0-9`, `a-f`, or `A-F`
+template <typename CharT>
+static constexpr dpf::nyble to_nyble(CharT value)
+{
+    auto u = static_cast<unsigned>(value);
+    if (u >= static_cast<unsigned>(CharT('0')) && u <= static_cast<unsigned>(CharT('9')))
+    {
+        return static_cast<dpf::nyble>(u - static_cast<unsigned>(CharT('0')));
+    }
+    if (u >= static_cast<unsigned>(CharT('a')) && u <= static_cast<unsigned>(CharT('f')))
+    {
+        return static_cast<dpf::nyble>(10u + u - static_cast<unsigned>(CharT('a')));
+    }
+    if (u >= static_cast<unsigned>(CharT('A')) && u <= static_cast<unsigned>(CharT('F')))
+    {
+        return static_cast<dpf::nyble>(10u + u - static_cast<unsigned>(CharT('A')));
+    }
+    throw std::domain_error("Unrecognized nyble character");
+}
+
+inline std::string to_string(dpf::nyble value)
+{
+    constexpr char digits[] = "0123456789abcdef";
+    return std::string(1, digits[static_cast<unsigned>(value) & 0x0fu]);
+}
+
+template <typename CharT, typename Traits>
+std::basic_ostream<CharT, Traits> &
+operator<<(std::basic_ostream<CharT, Traits> & os, dpf::nyble value)
+{
+    constexpr char digits[] = "0123456789abcdef";
+    return os << digits[static_cast<unsigned>(value) & 0x0fu];
+}
+
+template <typename CharT, typename Traits>
+std::basic_istream<CharT, Traits> &
+operator>>(std::basic_istream<CharT, Traits> & is, dpf::nyble & value)
+{
+    try
+    {
+        value = to_nyble<CharT>(is.get());
+    }
+    catch (const std::exception &)
+    {
+        is.setstate(std::ios::failbit);
+    }
+    return is;
+}
+
+/// @brief addition in Z/16Z
+HEDLEY_CONST
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+constexpr dpf::nyble operator+(dpf::nyble lhs, dpf::nyble rhs) noexcept
+{
+    return static_cast<dpf::nyble>(
+        (static_cast<unsigned>(lhs) + static_cast<unsigned>(rhs)) & 0x0fu);
+}
+
+/// @brief subtraction in Z/16Z
+HEDLEY_CONST
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+constexpr dpf::nyble operator-(dpf::nyble lhs, dpf::nyble rhs) noexcept
+{
+    return static_cast<dpf::nyble>(
+        (static_cast<unsigned>(lhs) - static_cast<unsigned>(rhs)) & 0x0fu);
+}
+
+/// @brief additive inverse in Z/16Z
+HEDLEY_CONST
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+constexpr dpf::nyble operator-(dpf::nyble value) noexcept
+{
+    return static_cast<dpf::nyble>((0u - static_cast<unsigned>(value)) & 0x0fu);
+}
+
+/// @brief multiplication in Z/16Z
+HEDLEY_CONST
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+constexpr dpf::nyble operator*(dpf::nyble lhs, dpf::nyble rhs) noexcept
+{
+    return static_cast<dpf::nyble>(
+        (static_cast<unsigned>(lhs) * static_cast<unsigned>(rhs)) & 0x0fu);
+}
+
+namespace utils
+{
+
+template <>
+struct bitlength_of<dpf::nyble>
+  : public std::integral_constant<std::size_t, 4> {};
+
+template <typename NodeT>
+struct bitlength_of_output<dpf::nyble, NodeT>
+  : public std::integral_constant<std::size_t, 4> {};
+
+template <>
+struct is_packed_subbyte<dpf::nyble> : std::true_type {};
+
+template <>
+struct packed_lane_bits<dpf::nyble>
+  : public std::integral_constant<std::size_t, 4> {};
+
+template <>
+struct make_from_integral_value<dpf::nyble>
+{
+    using integral_type = std::uint8_t;
+    constexpr dpf::nyble operator()(integral_type val) const noexcept
+    {
+        return dpf::to_nyble(val);
+    }
+};
+
+}  // namespace utils
+
+namespace literals
+{
+namespace nyble
+{
+
+constexpr dpf::nyble operator""_nyble(unsigned long long x)
+{
+    return dpf::to_nyble(x);
+}
+
+}  // namespace nyble
+}  // namespace literals
+
+}  // namespace dpf
+
+namespace std
+{
+
+template <>
+class numeric_limits<dpf::nyble> : public numeric_limits<std::uint8_t>
+{
+  public:
+    static constexpr int digits = 4;
+    static constexpr int digits10 = 1;
+    static constexpr dpf::nyble min() noexcept { return dpf::nyble::zero; }
+    static constexpr dpf::nyble max() noexcept { return dpf::nyble{15}; }
+    static constexpr dpf::nyble lowest() noexcept { return min(); }
+};
+
+template <>
+class numeric_limits<dpf::nyble const> : public numeric_limits<dpf::nyble> {};
+
+template <>
+class numeric_limits<dpf::nyble volatile> : public numeric_limits<dpf::nyble> {};
+
+template <>
+class numeric_limits<dpf::nyble const volatile>
+  : public numeric_limits<dpf::nyble> {};
+
+}  // namespace std
+
+#endif  // LIBDPF_INCLUDE_DPF_NYBLE_HPP__

include/dpf/offset_wrapper.hpp

@@ -0,0 +1,129 @@
+/// @file dpf/offset_wrapper.hpp
+/// @brief
+/// @details
+/// @author Ryan Henry <ryan.henry@ucalgary.ca>
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_OFFSET_WRAPPER_HPP__
+#define LIBDPF_INCLUDE_DPF_OFFSET_WRAPPER_HPP__
+
+#include "hedley/hedley.h"
+
+namespace dpf
+{
+
+template <typename InputT>
+struct offset_wrapper final
+{
+  public:
+    using input_type = dpf::concrete_type_t<InputT>;
+
+    offset_wrapper(input_type = input_type{})
+        : offset_{} { }
+
+    template <typename InputType>
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_PURE
+    HEDLEY_NO_THROW
+    constexpr auto operator()(InputType && x) const noexcept
+    {
+        static_assert(std::is_convertible_v<InputType, input_type>);
+        return input_type(std::forward<InputType>(x));
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_PURE
+    HEDLEY_NO_THROW
+    constexpr bool is_ready() const noexcept { return true; }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_PURE
+    HEDLEY_NO_THROW
+    static constexpr bool is_wildcard() noexcept { return false; }
+
+  private:
+    input_type offset_;  // waste an `input_type` to make `sizeof` match up
+};
+
+template <typename ConcreteInputT>
+struct offset_wrapper<dpf::wildcard_value<ConcreteInputT>>
+{
+  public:
+    using input_type = ConcreteInputT;
+
+    offset_wrapper(input_type x)
+        : offset_{x},
+          offset_state_{offset_status::notset}
+    { }
+
+    template <typename InputType>
+    HEDLEY_INLINE
+    input_type operator()(InputType && x) const
+    {
+        static_assert(std::is_convertible_v<InputType, input_type>);
+        if (HEDLEY_UNLIKELY(offset_state_ != offset_status::ready))
+        {
+            throw std::runtime_error("offset not set");
+        }
+        return input_type(x) + offset_;
+    }
+
+    template <typename InputType>
+    const input_type & compute_and_get_share(InputType && input_share)
+    {
+        static_assert(std::is_convertible_v<InputType, input_type>);
+        begin_transition(offset_status::notset);
+        offset_ -= input_share;
+        offset_state_ = offset_status::waiting;
+        return offset_; 
+    }
+
+    template <typename InputType>
+    const input_type & reconstruct(InputType && other_share)
+    {
+        static_assert(std::is_convertible_v<InputType, input_type>);
+        begin_transition(offset_status::waiting);
+        offset_ += other_share;
+        offset_state_ = offset_status::ready;
+        return offset_;
+    }
+
+    template <typename InputType>
+    const input_type & set(InputType && offset)
+    {
+        static_assert(std::is_convertible_v<InputType, input_type>);
+        begin_transition(offset_status::notset);
+        offset_ += offset;
+        offset_state_ = offset_status::ready;
+        return offset_;
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    bool is_ready() const noexcept { return offset_state_ == offset_status::ready; }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_PURE
+    HEDLEY_NO_THROW
+    static constexpr bool is_wildcard() noexcept { return true; }
+  private:
+    enum class offset_status : psnip_uint8_t { ready = 0, waiting = 1, computing = 2, notset = 3 };
+
+    void begin_transition(offset_status expected)
+    {
+        if (HEDLEY_UNLIKELY(offset_state_ != expected))
+        {
+            throw std::runtime_error("invalid state transition");
+        }
+        offset_state_ = offset_status::computing;
+    }
+
+    input_type offset_;
+    offset_status offset_state_;
+};
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_OFFSET_WRAPPER_HPP__

include/dpf/output_buffer.hpp

@@ -0,0 +1,465 @@
+/// @file dpf/output_buffer.hpp
+/// @brief
+/// @details
+/// @author Ryan Henry <ryan.henry@ucalgary.ca>
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_OUTPUT_BUFFER_HPP__
+#define LIBDPF_INCLUDE_DPF_OUTPUT_BUFFER_HPP__
+
+#include <cstddef>
+#include <algorithm>
+#include <tuple>
+#include <limits>
+#include <iterator>
+#include <new>
+#include <type_traits>
+#include <vector>
+
+#include "dpf/aligned_allocator.hpp"
+#include "dpf/leaf_node.hpp"
+#include "dpf/utils.hpp"
+#include "dpf/bit.hpp"
+#include "dpf/bit_array.hpp"
+#include "dpf/packed_array.hpp"
+#include "dpf/secret_share.hpp"
+#include "dpf/sequence_recipe.hpp"
+#include "dpf/sequence_utils.hpp"
+
+namespace dpf
+{
+
+/// Buffer element type for leaf eval of `KeyT`: party-tagged subtractive
+/// share when `KeyT` is a `party_key`, otherwise the concrete output.
+template <typename KeyT, typename OutputT, bool = is_party_key_v<KeyT>>
+struct leaf_buffer_elem
+{
+    using type = OutputT;
+};
+template <typename KeyT, typename OutputT>
+struct leaf_buffer_elem<KeyT, OutputT, true>
+{
+    using type = subtractive_share<OutputT, party_of_v<KeyT>>;
+};
+template <typename KeyT, typename OutputT>
+using leaf_buffer_elem_t = typename leaf_buffer_elem<KeyT, OutputT>::type;
+
+/// Buffer element type for comparison eval of `KeyT`.
+template <typename KeyT, typename Beta, bool = is_party_key_v<KeyT>>
+struct cmp_buffer_elem
+{
+    using type = Beta;
+};
+template <typename KeyT, typename Beta>
+struct cmp_buffer_elem<KeyT, Beta, true>
+{
+    using type = additive_share<Beta, party_of_v<KeyT>>;
+};
+template <typename KeyT, typename Beta>
+using cmp_buffer_elem_t = typename cmp_buffer_elem<KeyT, Beta>::type;
+
+/// `std::vector(n)` value-initializes every slot. Interval / full eval
+/// overwrites the whole buffer, so skip default-construction for trivial
+/// `T`. Non-trivial outputs still run their default constructor.
+template <typename T,
+          std::size_t Alignment>
+class output_buffer_allocator : public aligned_allocator<T, Alignment>
+{
+  public:
+    using is_always_equal = std::true_type;
+    using propagate_on_container_move_assignment = std::true_type;
+
+    template <typename U>
+    struct rebind
+    {
+        using other = output_buffer_allocator<U, Alignment>;
+    };
+
+    output_buffer_allocator() noexcept = default;
+    output_buffer_allocator(const output_buffer_allocator &) noexcept = default;
+    template <typename U>
+    output_buffer_allocator(const output_buffer_allocator<U, Alignment> &) noexcept {}
+
+    template <typename U>
+    void construct(U * p)
+        noexcept(std::is_nothrow_default_constructible_v<U>)
+    {
+        if constexpr (!std::is_trivially_default_constructible_v<U>)
+        {
+            ::new (static_cast<void *>(p)) U();
+        }
+    }
+
+    template <typename U, typename A0, typename ...Args>
+    void construct(U * p, A0 && a0, Args && ...args)
+    {
+        ::new (static_cast<void *>(p)) U(std::forward<A0>(a0),
+            std::forward<Args>(args)...);
+    }
+
+    template <typename U>
+    void destroy(U * p) noexcept
+    {
+        if constexpr (!std::is_trivially_destructible_v<U>)
+        {
+            p->~U();
+        }
+    }
+};
+
+template <typename T, std::size_t A, typename U, std::size_t B>
+constexpr bool operator==(const output_buffer_allocator<T, A> &,
+    const output_buffer_allocator<U, B> &) noexcept
+{
+    return A == B;
+}
+
+template <typename T, std::size_t A, typename U, std::size_t B>
+constexpr bool operator!=(const output_buffer_allocator<T, A> & lhs,
+    const output_buffer_allocator<U, B> & rhs) noexcept
+{
+    return !(lhs == rhs);
+}
+
+template <typename T,
+          std::size_t Alignment = utils::max_align_v>
+class output_buffer final
+  : private std::vector<T, dpf::output_buffer_allocator<T, Alignment>>
+{
+  private:
+    using vector = std::vector<T, dpf::output_buffer_allocator<T, Alignment>>;
+  public:
+    using value_type = typename vector::value_type;
+    using iterator = typename vector::iterator;
+    using const_iterator = typename vector::const_iterator;
+    using size_type = typename vector::size_type;
+    output_buffer() noexcept = default;
+    explicit output_buffer(size_type size) : vector(size) { }
+    output_buffer(output_buffer &&) noexcept = default;
+    output_buffer(const output_buffer &) = delete;
+    output_buffer & operator=(output_buffer &&) noexcept = default;
+    output_buffer & operator=(const output_buffer &) = delete;
+    ~output_buffer() = default;
+
+    // "selectively public" inheritance
+    using vector::at;
+    using vector::operator[];
+    using vector::data;
+    using vector::begin;
+    using vector::cbegin;
+    using vector::end;
+    using vector::cend;
+    using vector::size;
+};
+
+template <>
+class output_buffer<dpf::bit> : public dpf::dynamic_bit_array<>
+{
+  private:
+    using size_type = typename dpf::dynamic_bit_array<>::size_type;
+  public:
+    explicit output_buffer(size_type size) : dynamic_bit_array(size) { }
+    output_buffer(output_buffer &&) noexcept = default;
+    output_buffer(const output_buffer &) = delete;
+    output_buffer & operator=(output_buffer &&) noexcept = default;
+    output_buffer & operator=(const output_buffer &) = delete;
+    ~output_buffer() = default;
+};
+
+template <>
+class output_buffer<dpf::twobit> : public dpf::dynamic_packed_array<dpf::twobit>
+{
+    using base = dpf::dynamic_packed_array<dpf::twobit>;
+  public:
+    using size_type = typename base::size_type;
+    explicit output_buffer(size_type size) : base(size) { }
+    output_buffer(output_buffer &&) noexcept = default;
+    output_buffer(const output_buffer &) = delete;
+    output_buffer & operator=(output_buffer &&) noexcept = default;
+    output_buffer & operator=(const output_buffer &) = delete;
+    ~output_buffer() = default;
+};
+
+template <>
+class output_buffer<dpf::nyble> : public dpf::dynamic_packed_array<dpf::nyble>
+{
+    using base = dpf::dynamic_packed_array<dpf::nyble>;
+  public:
+    using size_type = typename base::size_type;
+    explicit output_buffer(size_type size) : base(size) { }
+    output_buffer(output_buffer &&) noexcept = default;
+    output_buffer(const output_buffer &) = delete;
+    output_buffer & operator=(output_buffer &&) noexcept = default;
+    output_buffer & operator=(const output_buffer &) = delete;
+    ~output_buffer() = default;
+};
+
+#define LIBDPF_PACKED_SHARE_BUFFER(LANE, PARTY)                                   \
+template <>                                                                       \
+class output_buffer<subtractive_share<LANE, PARTY>>                              \
+  : public packed_share_output<LANE, PARTY>                                      \
+{                                                                                \
+    using base = packed_share_output<LANE, PARTY>;                               \
+  public:                                                                        \
+    using size_type = typename base::size_type;                                  \
+    explicit output_buffer(size_type size) : base(size) {}                       \
+    output_buffer(output_buffer &&) noexcept = default;                          \
+    output_buffer(const output_buffer &) = delete;                               \
+    output_buffer & operator=(output_buffer &&) noexcept = default;              \
+    output_buffer & operator=(const output_buffer &) = delete;                   \
+    ~output_buffer() = default;                                                  \
+};
+
+LIBDPF_PACKED_SHARE_BUFFER(dpf::twobit, 0);
+LIBDPF_PACKED_SHARE_BUFFER(dpf::twobit, 1);
+LIBDPF_PACKED_SHARE_BUFFER(dpf::nyble, 0);
+LIBDPF_PACKED_SHARE_BUFFER(dpf::nyble, 1);
+#undef LIBDPF_PACKED_SHARE_BUFFER
+
+/// Packed bit share buffers reuse the bit-array image; iterators yield shares.
+#define LIBDPF_BIT_SHARE_BUFFER(PARTY)                                            \
+template <>                                                                       \
+class output_buffer<subtractive_share<dpf::bit, PARTY>>                          \
+  : public dpf::dynamic_bit_array<>                                              \
+{                                                                                \
+  private:                                                                        \
+    using size_type = typename dpf::dynamic_bit_array<>::size_type;              \
+  public:                                                                         \
+    explicit output_buffer(size_type size) : dynamic_bit_array(size) {}          \
+    output_buffer(output_buffer &&) noexcept = default;                          \
+    output_buffer(const output_buffer &) = delete;                               \
+    output_buffer & operator=(output_buffer &&) noexcept = default;              \
+    output_buffer & operator=(const output_buffer &) = delete;                   \
+    ~output_buffer() = default;                                                  \
+};
+LIBDPF_BIT_SHARE_BUFFER(0);
+LIBDPF_BIT_SHARE_BUFFER(1);
+#undef LIBDPF_BIT_SHARE_BUFFER
+
+template <typename DpfKey,
+          std::size_t I = 0,
+          typename InputT>
+auto make_output_buffer_for_interval(InputT from, InputT to)
+{
+    using dpf_type = DpfKey;
+    using output_type = typename DpfKey::concrete_output_type<I>;
+    using buffer_elem = leaf_buffer_elem_t<DpfKey, output_type>;
+
+    utils::flip_msb_if_signed_integral(from);
+    utils::flip_msb_if_signed_integral(to);
+
+    std::size_t nodes_in_interval = utils::get_leafnodes_in_output_interval<dpf_type>(from, to);
+    return dpf::output_buffer<buffer_elem>(nodes_in_interval*dpf_type::outputs_per_leaf);
+}
+
+template <typename DpfKey,
+          std::size_t I0,
+          std::size_t I1,
+          std::size_t ...Is,
+          typename InputT>
+auto make_output_buffer_for_interval(InputT from, InputT to)
+{
+    return std::make_tuple(
+        make_output_buffer_for_interval<DpfKey, I0>(from, to),
+        make_output_buffer_for_interval<DpfKey, I1>(from, to),
+        make_output_buffer_for_interval<DpfKey, Is>(from, to)...);
+}
+
+template <std::size_t I = 0,
+          typename DpfKey,
+          typename InputT>
+inline auto make_output_buffer_for_interval(const DpfKey &, InputT from, InputT to)
+{
+    return make_output_buffer_for_interval<DpfKey, I>(from, to);
+}
+
+template <std::size_t I0,
+          std::size_t I1,
+          std::size_t ...Is,
+          typename DpfKey,
+          typename InputT>
+inline auto make_output_buffer_for_interval(const DpfKey &, InputT from, InputT to)
+{
+    return make_output_buffer_for_interval<DpfKey, I0, I1, Is...>(from, to);
+}
+
+template <typename DpfKey,
+          std::size_t I = 0>
+auto make_output_buffer_for_full()
+{
+    using dpf_type = DpfKey;
+    using input_type = typename dpf_type::input_type;
+
+    return make_output_buffer_for_interval<dpf_type, I>(
+        std::numeric_limits<input_type>::min(),
+        std::numeric_limits<input_type>::max());
+}
+
+template <typename DpfKey,
+          std::size_t I0,
+          std::size_t I1,
+          std::size_t ...Is>
+auto make_output_buffer_for_full()
+{
+    return std::make_tuple(
+        make_output_buffer_for_full<DpfKey, I0>(),
+        make_output_buffer_for_full<DpfKey, I1>(),
+        make_output_buffer_for_full<DpfKey, Is>()...);
+}
+
+template <std::size_t I = 0,
+          typename DpfKey>
+inline auto make_output_buffer_for_full(const DpfKey &)
+{
+    return make_output_buffer_for_full<DpfKey, I>();
+}
+
+template <std::size_t I0,
+          std::size_t I1,
+          std::size_t ...Is,
+          typename DpfKey>
+inline auto make_output_buffer_for_full(const DpfKey &)
+{
+    return make_output_buffer_for_full<DpfKey, I0, I1, Is...>();
+}
+
+template <typename DpfKey,
+          std::size_t I = 0,
+          typename ForwardIterator,
+          typename ReturnType = return_entire_node_tag_>
+auto make_output_buffer_for_subsequence(ForwardIterator begin, ForwardIterator end, ReturnType return_type = ReturnType{})
+{
+    using dpf_type = DpfKey;
+    using output_type = typename DpfKey::concrete_output_type<I>;
+    using buffer_elem = leaf_buffer_elem_t<DpfKey, output_type>;
+    std::size_t points_in_sequence = std::distance(begin, end);
+
+    static_assert(std::is_same_v<ReturnType, return_entire_node_tag_> ||
+                    std::is_same_v<ReturnType, return_output_only_tag_>);
+    if constexpr(std::is_same_v<ReturnType, return_entire_node_tag_>)
+    {
+        return dpf::output_buffer<buffer_elem>(points_in_sequence*dpf_type::outputs_per_leaf);
+    }
+    else
+    {
+        if constexpr(std::is_same_v<typename DpfKey::concrete_output_type<0>, dpf::bit>)
+        {
+            auto tmp = dpf::output_buffer<buffer_elem>(points_in_sequence);
+            tmp.unset();
+            return std::move(tmp);
+        }
+        else
+        {
+            return dpf::output_buffer<buffer_elem>(points_in_sequence);
+        }
+    }
+}
+
+template <typename DpfKey,
+          std::size_t I0,
+          std::size_t I1,
+          std::size_t ...Is,
+          typename ForwardIterator,
+          typename ReturnType = return_entire_node_tag_>
+auto make_output_buffer_for_subsequence(ForwardIterator begin, ForwardIterator end, ReturnType return_type = ReturnType{})
+{
+    return std::make_tuple(
+        make_output_buffer_for_subsequence<DpfKey, I0>(begin, end, return_type),
+        make_output_buffer_for_subsequence<DpfKey, I1>(begin, end, return_type),
+        make_output_buffer_for_subsequence<DpfKey, Is>(begin, end, return_type)...);
+}
+
+template <std::size_t I = 0,
+          typename DpfKey,
+          typename ForwardIterator,
+          typename ReturnType = return_entire_node_tag_>
+inline auto make_output_buffer_for_subsequence(const DpfKey &, ForwardIterator begin, ForwardIterator end, ReturnType return_type = ReturnType{})
+{
+    return make_output_buffer_for_subsequence<DpfKey, I>(begin, end, return_type);
+}
+
+template <std::size_t I0,
+          std::size_t I1,
+          std::size_t ...Is,
+          typename DpfKey,
+          typename ForwardIterator,
+          typename ReturnType = return_entire_node_tag_>
+inline auto make_output_buffer_for_subsequence(const DpfKey &, ForwardIterator begin, ForwardIterator end, ReturnType return_type = ReturnType{})
+{
+    return make_output_buffer_for_subsequence<DpfKey, I0, I1, Is...>(begin, end, return_type);
+}
+
+template <typename DpfKey,
+          std::size_t I = 0,
+          typename ReturnType = return_entire_node_tag_>
+auto make_output_buffer_for_recipe_subsequence(const sequence_recipe & recipe, ReturnType return_type = ReturnType{})
+{
+    using dpf_type = DpfKey;
+    using output_type = typename DpfKey::concrete_output_type<I>;
+    using buffer_elem = leaf_buffer_elem_t<DpfKey, output_type>;
+
+    static_assert(std::is_same_v<ReturnType, return_entire_node_tag_> ||
+                    std::is_same_v<ReturnType, return_output_only_tag_>);
+    if constexpr(std::is_same_v<ReturnType, return_entire_node_tag_>)
+    {
+        return dpf::output_buffer<buffer_elem>(recipe.num_leaf_nodes()*dpf_type::outputs_per_leaf);
+    }
+    else
+    {
+        if constexpr(std::is_same_v<typename DpfKey::concrete_output_type<0>, dpf::bit>)
+        {
+            auto tmp = dpf::output_buffer<buffer_elem>(recipe.output_indices().size());
+            tmp.unset();
+            return std::move(tmp);
+        }
+        else
+        {
+            return dpf::output_buffer<buffer_elem>(recipe.output_indices().size());
+        }
+    }
+}
+
+template <typename DpfKey,
+          std::size_t I0,
+          std::size_t I1,
+          std::size_t ...Is,
+          typename ReturnType = return_entire_node_tag_>
+inline auto make_output_buffer_for_recipe_subsequence(const sequence_recipe & recipe, ReturnType return_type = ReturnType{})
+{
+    return std::make_tuple(
+        make_output_buffer_for_recipe_subsequence<DpfKey, I0>(recipe, return_type),
+        make_output_buffer_for_recipe_subsequence<DpfKey, I1>(recipe, return_type),
+        make_output_buffer_for_recipe_subsequence<DpfKey, Is>(recipe, return_type)...);
+}
+
+template <std::size_t I = 0,
+          typename DpfKey,
+          typename ReturnType = return_entire_node_tag_>
+inline auto make_output_buffer_for_recipe_subsequence(const DpfKey &, const sequence_recipe & recipe, ReturnType return_type = ReturnType{})
+{
+    return make_output_buffer_for_recipe_subsequence<DpfKey, I>(recipe, return_type);
+}
+
+template <std::size_t I0,
+          std::size_t I1,
+          std::size_t ...Is,
+          typename DpfKey,
+          typename ReturnType = return_entire_node_tag_>
+inline auto make_output_buffer_for_recipe_subsequence(const DpfKey &, const sequence_recipe & recipe, ReturnType return_type = ReturnType{})
+{
+    return make_output_buffer_for_recipe_subsequence<DpfKey, I0, I1, Is...>(recipe, return_type);
+}
+
+namespace utils
+{
+
+template <>
+struct is_bit_array<output_buffer<bit>> : std::true_type {};
+
+}  // namespace utils
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_OUTPUT_BUFFER_HPP__

include/dpf/packed_array.hpp

@@ -0,0 +1,411 @@
+/// @file dpf/packed_array.hpp
+/// @brief Dynamic array of `dpf::twobit` or `dpf::nyble` lanes.
+/// @details Storage is the same bit layout as a packed leaf, so
+///          `store_leaf_bytes` can memcpy a node at lane `index`.
+///          Iterators hold a pointer to that storage and stay valid when
+///          the owning buffer is moved.
+
+#ifndef LIBDPF_INCLUDE_DPF_PACKED_ARRAY_HPP__
+#define LIBDPF_INCLUDE_DPF_PACKED_ARRAY_HPP__
+
+#include <algorithm>
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+#include <iterator>
+#include <memory>
+#include <new>
+#include <utility>
+
+#include "hedley/hedley.h"
+
+#include "dpf/aligned_allocator.hpp"
+#include "dpf/nyble.hpp"
+#include "dpf/secret_share.hpp"
+#include "dpf/twobit.hpp"
+#include "dpf/utils.hpp"
+
+namespace dpf
+{
+
+template <typename LaneT>
+class dynamic_packed_array
+{
+    static constexpr std::size_t lane_bits = utils::packed_lane_bits_v<LaneT>;
+    static_assert(lane_bits == 2 || lane_bits == 4,
+        "dynamic_packed_array lanes are 2 or 4 bits");
+    static constexpr std::size_t lanes_per_word = 64u / lane_bits;
+    static constexpr unsigned lane_mask = (1u << lane_bits) - 1u;
+
+  public:
+    using value_type = LaneT;
+    using size_type = std::size_t;
+    using difference_type = std::ptrdiff_t;
+    using word_type = std::uint64_t;
+
+  private:
+    using allocator = aligned_allocator<word_type, utils::max_align_v>;
+    using unique_ptr = typename allocator::unique_ptr;
+
+    class lane_ref
+    {
+      public:
+        lane_ref(word_type * word, unsigned shift) noexcept
+          : word_{word}, shift_{shift} {}
+
+        HEDLEY_ALWAYS_INLINE
+        operator LaneT() const noexcept
+        {
+            return static_cast<LaneT>((*word_ >> shift_) & lane_mask);
+        }
+
+        HEDLEY_ALWAYS_INLINE
+        lane_ref & operator=(LaneT value) noexcept
+        {
+            const auto val = static_cast<word_type>(
+                static_cast<unsigned>(value) & lane_mask);
+            *word_ = (*word_ & ~(static_cast<word_type>(lane_mask) << shift_))
+                | (val << shift_);
+            return *this;
+        }
+
+        lane_ref & operator=(const lane_ref & other) noexcept
+        {
+            return (*this = static_cast<LaneT>(other));
+        }
+
+        friend bool operator==(lane_ref lhs, LaneT rhs) noexcept
+        {
+            return static_cast<LaneT>(lhs) == rhs;
+        }
+        friend bool operator==(LaneT lhs, lane_ref rhs) noexcept
+        {
+            return rhs == lhs;
+        }
+        friend bool operator!=(lane_ref lhs, LaneT rhs) noexcept
+        {
+            return !(lhs == rhs);
+        }
+        friend bool operator!=(LaneT lhs, lane_ref rhs) noexcept
+        {
+            return !(rhs == lhs);
+        }
+
+      private:
+        word_type * word_;
+        unsigned shift_;
+    };
+
+  public:
+    using reference = lane_ref;
+
+    class iterator
+    {
+      public:
+        using iterator_category = std::random_access_iterator_tag;
+        using value_type = LaneT;
+        using difference_type = std::ptrdiff_t;
+        using pointer = void;
+        using reference = lane_ref;
+
+        iterator() noexcept = default;
+        iterator(word_type * data, size_type index) noexcept
+          : data_{data}, index_{index} {}
+
+        lane_ref operator*() const noexcept { return ref_at(index_); }
+        lane_ref operator[](difference_type n) const noexcept
+        {
+            return ref_at(static_cast<size_type>(
+                static_cast<difference_type>(index_) + n));
+        }
+
+        iterator & operator++() noexcept { ++index_; return *this; }
+        iterator operator++(int) noexcept
+        {
+            iterator prev = *this;
+            ++*this;
+            return prev;
+        }
+        iterator & operator--() noexcept { --index_; return *this; }
+        iterator operator--(int) noexcept
+        {
+            iterator prev = *this;
+            --*this;
+            return prev;
+        }
+
+        iterator & operator+=(difference_type n) noexcept
+        {
+            index_ = static_cast<size_type>(
+                static_cast<difference_type>(index_) + n);
+            return *this;
+        }
+        iterator & operator-=(difference_type n) noexcept
+        {
+            return *this += -n;
+        }
+        friend iterator operator+(iterator it, difference_type n) noexcept
+        {
+            it += n;
+            return it;
+        }
+        friend iterator operator+(difference_type n, iterator it) noexcept
+        {
+            return it + n;
+        }
+        friend iterator operator-(iterator it, difference_type n) noexcept
+        {
+            it -= n;
+            return it;
+        }
+        friend difference_type operator-(iterator a, iterator b) noexcept
+        {
+            return static_cast<difference_type>(a.index_)
+                - static_cast<difference_type>(b.index_);
+        }
+
+        friend bool operator==(iterator a, iterator b) noexcept
+        {
+            return a.index_ == b.index_;
+        }
+        friend bool operator!=(iterator a, iterator b) noexcept
+        {
+            return !(a == b);
+        }
+        friend bool operator<(iterator a, iterator b) noexcept
+        {
+            return a.index_ < b.index_;
+        }
+        friend bool operator>(iterator a, iterator b) noexcept { return b < a; }
+        friend bool operator<=(iterator a, iterator b) noexcept { return !(b < a); }
+        friend bool operator>=(iterator a, iterator b) noexcept { return !(a < b); }
+
+      private:
+        lane_ref ref_at(size_type index) const noexcept
+        {
+            const size_type bit = index * lane_bits;
+            return lane_ref(data_ + (bit / 64u),
+                static_cast<unsigned>(bit % 64u));
+        }
+
+        word_type * data_ = nullptr;
+        size_type index_ = 0;
+    };
+
+    using const_iterator = iterator;
+
+    explicit dynamic_packed_array(size_type nlanes)
+      : nlanes_{nlanes},
+        nwords_{utils::quotient_ceiling(nlanes, lanes_per_word)}
+    {
+        if (nwords_ == 0)
+        {
+            return;
+        }
+        data_ = allocator{}.allocate_unique_ptr(nwords_);
+        if (HEDLEY_UNLIKELY(data_ == nullptr))
+        {
+            throw std::bad_alloc{};
+        }
+        std::fill_n(data_.get(), nwords_, word_type{0});
+    }
+
+    dynamic_packed_array(const dynamic_packed_array &) = delete;
+    dynamic_packed_array & operator=(const dynamic_packed_array &) = delete;
+
+    dynamic_packed_array(dynamic_packed_array && other) noexcept
+      : nlanes_{std::exchange(other.nlanes_, 0)},
+        nwords_{std::exchange(other.nwords_, 0)},
+        data_{std::move(other.data_)}
+    {}
+
+    dynamic_packed_array & operator=(dynamic_packed_array && other) noexcept
+    {
+        if (this != &other)
+        {
+            nlanes_ = std::exchange(other.nlanes_, 0);
+            nwords_ = std::exchange(other.nwords_, 0);
+            data_ = std::move(other.data_);
+        }
+        return *this;
+    }
+
+    ~dynamic_packed_array() = default;
+
+    size_type size() const noexcept { return nlanes_; }
+    bool empty() const noexcept { return nlanes_ == 0; }
+    size_type data_length() const noexcept { return nwords_; }
+
+    word_type * data() noexcept { return data_.get(); }
+    const word_type * data() const noexcept { return data_.get(); }
+
+    LaneT operator[](size_type i) const noexcept
+    {
+        assert(i < nlanes_);
+        const size_type bit = i * lane_bits;
+        const unsigned shift = static_cast<unsigned>(bit % 64u);
+        return static_cast<LaneT>((data_[bit / 64u] >> shift) & lane_mask);
+    }
+
+    lane_ref operator[](size_type i) noexcept
+    {
+        assert(i < nlanes_);
+        const size_type bit = i * lane_bits;
+        return lane_ref(data_.get() + (bit / 64u),
+            static_cast<unsigned>(bit % 64u));
+    }
+
+    iterator begin() noexcept { return iterator{data(), 0}; }
+    iterator end() noexcept { return iterator{data(), nlanes_}; }
+    iterator begin() const noexcept { return iterator{data_.get(), 0}; }
+    iterator end() const noexcept { return iterator{data_.get(), nlanes_}; }
+    iterator cbegin() const noexcept { return begin(); }
+    iterator cend() const noexcept { return end(); }
+
+  private:
+    size_type nlanes_ = 0;
+    size_type nwords_ = 0;
+    unique_ptr data_{};
+};
+
+/// Packed lane storage whose iterators yield `subtractive_share<LaneT, Party>`.
+/// The bytes are the leaf image (`store_leaf_bytes`); each lane is one share.
+template <typename LaneT, std::size_t Party>
+class packed_share_output : public dynamic_packed_array<LaneT>
+{
+    using lanes = dynamic_packed_array<LaneT>;
+    using share_type = subtractive_share<LaneT, Party>;
+
+  public:
+    using value_type = share_type;
+    using size_type = typename lanes::size_type;
+    using difference_type = typename lanes::difference_type;
+
+    class reference
+    {
+      public:
+        explicit reference(typename lanes::reference lane) noexcept : lane_{lane} {}
+
+        operator share_type() const noexcept
+        {
+            return share_type::from_raw(static_cast<LaneT>(lane_));
+        }
+
+        reference & operator=(const share_type & share) noexcept
+        {
+            lane_ = share.raw();
+            return *this;
+        }
+
+        reference & operator=(LaneT value) noexcept
+        {
+            lane_ = value;
+            return *this;
+        }
+
+        reference & operator=(const reference & other) noexcept
+        {
+            return (*this = static_cast<share_type>(other));
+        }
+
+      private:
+        typename lanes::reference lane_;
+    };
+
+    class iterator
+    {
+      public:
+        using iterator_category = std::random_access_iterator_tag;
+        using value_type = share_type;
+        using difference_type = std::ptrdiff_t;
+        using pointer = void;
+        using reference = share_type;
+
+        iterator() noexcept = default;
+        explicit iterator(typename lanes::iterator it) noexcept : it_{it} {}
+
+        share_type operator*() const noexcept
+        {
+            return share_type::from_raw(static_cast<LaneT>(*it_));
+        }
+        share_type operator[](difference_type n) const noexcept
+        {
+            return share_type::from_raw(static_cast<LaneT>(it_[n]));
+        }
+
+        iterator & operator++() noexcept { ++it_; return *this; }
+        iterator operator++(int) noexcept { iterator p = *this; ++*this; return p; }
+        iterator & operator--() noexcept { --it_; return *this; }
+        iterator operator--(int) noexcept { iterator p = *this; --*this; return p; }
+        iterator & operator+=(difference_type n) noexcept { it_ += n; return *this; }
+        iterator & operator-=(difference_type n) noexcept { it_ -= n; return *this; }
+
+        friend iterator operator+(iterator it, difference_type n) noexcept
+        {
+            it += n;
+            return it;
+        }
+        friend iterator operator+(difference_type n, iterator it) noexcept
+        {
+            return it + n;
+        }
+        friend iterator operator-(iterator it, difference_type n) noexcept
+        {
+            it -= n;
+            return it;
+        }
+        friend difference_type operator-(iterator a, iterator b) noexcept
+        {
+            return a.it_ - b.it_;
+        }
+        friend bool operator==(iterator a, iterator b) noexcept { return a.it_ == b.it_; }
+        friend bool operator!=(iterator a, iterator b) noexcept { return !(a == b); }
+        friend bool operator<(iterator a, iterator b) noexcept { return a.it_ < b.it_; }
+        friend bool operator>(iterator a, iterator b) noexcept { return b < a; }
+        friend bool operator<=(iterator a, iterator b) noexcept { return !(b < a); }
+        friend bool operator>=(iterator a, iterator b) noexcept { return !(a < b); }
+
+      private:
+        typename lanes::iterator it_{};
+    };
+
+    using const_iterator = iterator;
+
+    explicit packed_share_output(size_type nlanes) : lanes(nlanes) {}
+
+    packed_share_output(const packed_share_output &) = delete;
+    packed_share_output & operator=(const packed_share_output &) = delete;
+    packed_share_output(packed_share_output &&) noexcept = default;
+    packed_share_output & operator=(packed_share_output &&) noexcept = default;
+    ~packed_share_output() = default;
+
+    using lanes::data;
+    using lanes::empty;
+    using lanes::size;
+
+    reference operator[](size_type i) noexcept
+    {
+        return reference{lanes::operator[](i)};
+    }
+    share_type operator[](size_type i) const noexcept
+    {
+        return share_type::from_raw(lanes::operator[](i));
+    }
+
+    iterator begin() noexcept { return iterator{lanes::begin()}; }
+    iterator end() noexcept { return iterator{lanes::end()}; }
+    iterator begin() const noexcept
+    {
+        return iterator{typename lanes::iterator{this->data(), 0}};
+    }
+    iterator end() const noexcept
+    {
+        return iterator{typename lanes::iterator{this->data(), this->size()}};
+    }
+    iterator cbegin() const noexcept { return begin(); }
+    iterator cend() const noexcept { return end(); }
+};
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_PACKED_ARRAY_HPP__

include/dpf/packed_lane.hpp

@@ -0,0 +1,75 @@
+/// @file dpf/packed_lane.hpp
+/// @brief Extract and deposit one `dpf::bit`, `dpf::twobit`, or `dpf::nyble`
+///        lane in a leaf node. Lane 0 is the low bits of the first byte.
+
+#ifndef LIBDPF_INCLUDE_DPF_PACKED_LANE_HPP__
+#define LIBDPF_INCLUDE_DPF_PACKED_LANE_HPP__
+
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+
+#include "hedley/hedley.h"
+
+#include "dpf/bit.hpp"
+#include "dpf/nyble.hpp"
+#include "dpf/twobit.hpp"
+
+namespace dpf
+{
+namespace packed
+{
+
+template <typename LaneT, typename LeafT>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+HEDLEY_PURE
+LaneT extract_lane(const LeafT & leaf, std::size_t lane) noexcept
+{
+    constexpr unsigned bits = utils::packed_lane_bits_v<LaneT>;
+    static_assert(bits == 1 || bits == 2 || bits == 4,
+        "packed lane width must be 1, 2, or 4");
+    constexpr unsigned mask = (1u << bits) - 1u;
+    const auto * bytes = reinterpret_cast<const unsigned char *>(
+        std::addressof(leaf));
+    const std::size_t bit = lane * bits;
+    assert(bit / 8u < sizeof(LeafT));
+    const unsigned shift = static_cast<unsigned>(bit % 8u);
+    const unsigned val = (bytes[bit / 8u] >> shift) & mask;
+    return static_cast<LaneT>(val);
+}
+
+/// @brief zero `out` is the caller's job; this writes one lane and leaves
+///        every other lane untouched.
+template <typename LaneT, typename LeafT>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+void deposit_lane(LeafT & leaf, std::size_t lane, LaneT value) noexcept
+{
+    constexpr unsigned bits = utils::packed_lane_bits_v<LaneT>;
+    static_assert(bits == 1 || bits == 2 || bits == 4,
+        "packed lane width must be 1, 2, or 4");
+    constexpr unsigned mask = (1u << bits) - 1u;
+    auto * bytes = reinterpret_cast<unsigned char *>(std::addressof(leaf));
+    const std::size_t bit = lane * bits;
+    assert(bit / 8u < sizeof(LeafT));
+    const unsigned shift = static_cast<unsigned>(bit % 8u);
+    const unsigned val = static_cast<unsigned>(value) & mask;
+    unsigned char & cell = bytes[bit / 8u];
+    cell = static_cast<unsigned char>((cell & ~(mask << shift)) | (val << shift));
+}
+
+template <typename NodeT, typename LaneT>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+NodeT make_lane_node(std::size_t lane, LaneT value) noexcept
+{
+    NodeT node{};
+    deposit_lane<LaneT>(node, lane, value);
+    return node;
+}
+
+}  // namespace packed
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_PACKED_LANE_HPP__

include/dpf/packed_lane_arithmetic.hpp

@@ -0,0 +1,470 @@
+/// @file dpf/packed_lane_arithmetic.hpp
+/// @brief AVX2 lane arithmetic for 2-bit (Z/4Z) and 4-bit (Z/16Z) leaves.
+///
+/// Vector-vector add of 2-bit lanes is the bitsliced form measured fastest
+/// on AVX2 (carry stays in bit 0 of each pair). 4-bit add isolates nibbles
+/// inside `add_epi8` so a sum cannot spill into the next nibble; that is
+/// shorter than an unpack/pack into 16-bit lanes.
+///
+/// Scalar-vector multiply builds a 16-entry `pshufb` table, which won the
+/// scalar-vector timings. Vector-vector multiply is bitsliced mod 4, and
+/// `mullo_epi16` on nibbles split into even and odd bytes mod 16.
+
+#ifndef LIBDPF_INCLUDE_DPF_PACKED_LANE_ARITHMETIC_HPP__
+#define LIBDPF_INCLUDE_DPF_PACKED_LANE_ARITHMETIC_HPP__
+
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <type_traits>
+
+#include "hedley/hedley.h"
+#include "simde/simde/x86/avx2.h"
+
+#include "dpf/nyble.hpp"
+#include "dpf/twobit.hpp"
+
+namespace dpf
+{
+namespace lane_arith
+{
+
+namespace detail
+{
+
+template <unsigned Bits>
+HEDLEY_ALWAYS_INLINE
+unsigned lane_at(unsigned byte, unsigned shift) noexcept
+{
+    constexpr unsigned mask = (1u << Bits) - 1u;
+    return (byte >> shift) & mask;
+}
+
+template <unsigned Bits, typename NodeT, typename Op>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+NodeT apply_bytes(const NodeT & a, const NodeT & b, Op op) noexcept
+{
+    static_assert(Bits == 2 || Bits == 4);
+    constexpr unsigned mask = (1u << Bits) - 1u;
+    constexpr unsigned per_byte = 8u / Bits;
+    NodeT out{};
+    const auto * ab = reinterpret_cast<const unsigned char *>(std::addressof(a));
+    const auto * bb = reinterpret_cast<const unsigned char *>(std::addressof(b));
+    auto * cb = reinterpret_cast<unsigned char *>(std::addressof(out));
+    for (std::size_t i = 0; i < sizeof(NodeT); ++i)
+    {
+        unsigned packed = 0;
+        for (unsigned s = 0; s < per_byte; ++s)
+        {
+            const unsigned shift = s * Bits;
+            const unsigned av = lane_at<Bits>(ab[i], shift);
+            const unsigned bv = lane_at<Bits>(bb[i], shift);
+            packed |= (op(av, bv) & mask) << shift;
+        }
+        cb[i] = static_cast<unsigned char>(packed);
+    }
+    return out;
+}
+
+inline simde__m128i nibble_lut(const unsigned char lut[16]) noexcept
+{
+    simde__m128i table;
+    std::memcpy(&table, lut, sizeof(table));
+    return table;
+}
+
+inline simde__m256i nibble_lut256(const unsigned char lut[16]) noexcept
+{
+    alignas(32) unsigned char both[32];
+    std::memcpy(both, lut, 16);
+    std::memcpy(both + 16, lut, 16);
+    simde__m256i table;
+    std::memcpy(&table, both, sizeof(table));
+    return table;
+}
+
+inline void fill_epi2_mul_lut(unsigned k, unsigned char lut[16]) noexcept
+{
+    k &= 3u;
+    for (unsigned n = 0; n < 16u; ++n)
+    {
+        const unsigned l0 = ((n & 3u) * k) & 3u;
+        const unsigned l1 = (((n >> 2) & 3u) * k) & 3u;
+        lut[n] = static_cast<unsigned char>(l0 | (l1 << 2));
+    }
+}
+
+inline void fill_epi4_mul_lut(unsigned k, unsigned char lut[16]) noexcept
+{
+    k &= 0x0fu;
+    for (unsigned n = 0; n < 16u; ++n)
+    {
+        lut[n] = static_cast<unsigned char>((n * k) & 0x0fu);
+    }
+}
+
+inline simde__m128i shuffle_nibbles(simde__m128i table, simde__m128i a) noexcept
+{
+    const auto m = simde_mm_set1_epi8(0x0f);
+    const auto lo = simde_mm_shuffle_epi8(table, simde_mm_and_si128(a, m));
+    const auto hi_idx = simde_mm_and_si128(simde_mm_srli_epi16(a, 4), m);
+    const auto hi = simde_mm_shuffle_epi8(table, hi_idx);
+    return simde_mm_or_si128(simde_mm_and_si128(lo, m),
+        simde_mm_slli_epi16(simde_mm_and_si128(hi, m), 4));
+}
+
+inline simde__m256i shuffle_nibbles(simde__m256i table, simde__m256i a) noexcept
+{
+    const auto m = simde_mm256_set1_epi8(0x0f);
+    const auto lo = simde_mm256_shuffle_epi8(table, simde_mm256_and_si256(a, m));
+    const auto hi_idx = simde_mm256_and_si256(simde_mm256_srli_epi16(a, 4), m);
+    const auto hi = simde_mm256_shuffle_epi8(table, hi_idx);
+    return simde_mm256_or_si256(simde_mm256_and_si256(lo, m),
+        simde_mm256_slli_epi16(simde_mm256_and_si256(hi, m), 4));
+}
+
+/// Low nibble of every byte, product mod 16. Even and odd bytes are split
+/// so a product in one byte cannot land in the next.
+inline simde__m128i mul_low_nibbles(simde__m128i a, simde__m128i b) noexcept
+{
+    const auto lane = simde_mm_set1_epi16(0x000f);
+    const auto ae = simde_mm_and_si128(a, lane);
+    const auto be = simde_mm_and_si128(b, lane);
+    const auto pe = simde_mm_and_si128(simde_mm_mullo_epi16(ae, be), lane);
+    const auto ao = simde_mm_and_si128(simde_mm_srli_epi16(a, 8), lane);
+    const auto bo = simde_mm_and_si128(simde_mm_srli_epi16(b, 8), lane);
+    const auto po = simde_mm_and_si128(simde_mm_mullo_epi16(ao, bo), lane);
+    return simde_mm_or_si128(pe, simde_mm_slli_epi16(po, 8));
+}
+
+inline simde__m256i mul_low_nibbles(simde__m256i a, simde__m256i b) noexcept
+{
+    const auto lane = simde_mm256_set1_epi16(0x000f);
+    const auto ae = simde_mm256_and_si256(a, lane);
+    const auto be = simde_mm256_and_si256(b, lane);
+    const auto pe = simde_mm256_and_si256(simde_mm256_mullo_epi16(ae, be), lane);
+    const auto ao = simde_mm256_and_si256(simde_mm256_srli_epi16(a, 8), lane);
+    const auto bo = simde_mm256_and_si256(simde_mm256_srli_epi16(b, 8), lane);
+    const auto po = simde_mm256_and_si256(simde_mm256_mullo_epi16(ao, bo), lane);
+    return simde_mm256_or_si256(pe, simde_mm256_slli_epi16(po, 8));
+}
+
+}  // namespace detail
+
+template <typename NodeT>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+HEDLEY_PURE
+NodeT add_mod2(const NodeT & a, const NodeT & b) noexcept
+{
+    return detail::apply_bytes<2>(a, b, [](unsigned x, unsigned y) {
+        return (x + y) & 3u;
+    });
+}
+
+template <typename NodeT>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+HEDLEY_PURE
+NodeT sub_mod2(const NodeT & a, const NodeT & b) noexcept
+{
+    return detail::apply_bytes<2>(a, b, [](unsigned x, unsigned y) {
+        return (x - y) & 3u;
+    });
+}
+
+template <typename NodeT>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+HEDLEY_PURE
+NodeT mul_mod2(const NodeT & a, dpf::twobit k) noexcept
+{
+    const unsigned s = static_cast<unsigned>(k);
+    return detail::apply_bytes<2>(a, a, [s](unsigned x, unsigned) {
+        return (x * s) & 3u;
+    });
+}
+
+template <typename NodeT>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+HEDLEY_PURE
+NodeT mullo_mod2(const NodeT & a, const NodeT & b) noexcept
+{
+    return detail::apply_bytes<2>(a, b, [](unsigned x, unsigned y) {
+        return (x * y) & 3u;
+    });
+}
+
+template <typename NodeT>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+HEDLEY_PURE
+NodeT add_mod4(const NodeT & a, const NodeT & b) noexcept
+{
+    return detail::apply_bytes<4>(a, b, [](unsigned x, unsigned y) {
+        return (x + y) & 0x0fu;
+    });
+}
+
+template <typename NodeT>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+HEDLEY_PURE
+NodeT sub_mod4(const NodeT & a, const NodeT & b) noexcept
+{
+    return detail::apply_bytes<4>(a, b, [](unsigned x, unsigned y) {
+        return (x - y) & 0x0fu;
+    });
+}
+
+template <typename NodeT>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+HEDLEY_PURE
+NodeT mul_mod4(const NodeT & a, dpf::nyble k) noexcept
+{
+    const unsigned s = static_cast<unsigned>(k);
+    return detail::apply_bytes<4>(a, a, [s](unsigned x, unsigned) {
+        return (x * s) & 0x0fu;
+    });
+}
+
+template <typename NodeT>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+HEDLEY_PURE
+NodeT mullo_mod4(const NodeT & a, const NodeT & b) noexcept
+{
+    return detail::apply_bytes<4>(a, b, [](unsigned x, unsigned y) {
+        return (x * y) & 0x0fu;
+    });
+}
+
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+HEDLEY_PURE
+simde__m128i add_epi2(simde__m128i a, simde__m128i b) noexcept
+{
+    const auto m = simde_mm_set1_epi8(0x55);
+    const auto a0 = simde_mm_and_si128(a, m);
+    const auto a1 = simde_mm_and_si128(simde_mm_srli_epi16(a, 1), m);
+    const auto b0 = simde_mm_and_si128(b, m);
+    const auto b1 = simde_mm_and_si128(simde_mm_srli_epi16(b, 1), m);
+    const auto bit0 = simde_mm_xor_si128(a0, b0);
+    const auto carry = simde_mm_and_si128(a0, b0);
+    const auto bit1 = simde_mm_xor_si128(simde_mm_xor_si128(a1, b1), carry);
+    return simde_mm_or_si128(bit0, simde_mm_slli_epi16(bit1, 1));
+}
+
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+HEDLEY_PURE
+simde__m256i add_epi2(simde__m256i a, simde__m256i b) noexcept
+{
+    const auto m = simde_mm256_set1_epi8(0x55);
+    const auto a0 = simde_mm256_and_si256(a, m);
+    const auto a1 = simde_mm256_and_si256(simde_mm256_srli_epi16(a, 1), m);
+    const auto b0 = simde_mm256_and_si256(b, m);
+    const auto b1 = simde_mm256_and_si256(simde_mm256_srli_epi16(b, 1), m);
+    const auto bit0 = simde_mm256_xor_si256(a0, b0);
+    const auto carry = simde_mm256_and_si256(a0, b0);
+    const auto bit1 = simde_mm256_xor_si256(simde_mm256_xor_si256(a1, b1), carry);
+    return simde_mm256_or_si256(bit0, simde_mm256_slli_epi16(bit1, 1));
+}
+
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+HEDLEY_PURE
+simde__m128i sub_epi2(simde__m128i a, simde__m128i b) noexcept
+{
+    const auto m = simde_mm_set1_epi8(0x55);
+    const auto a0 = simde_mm_and_si128(a, m);
+    const auto a1 = simde_mm_and_si128(simde_mm_srli_epi16(a, 1), m);
+    const auto b0 = simde_mm_and_si128(b, m);
+    const auto b1 = simde_mm_and_si128(simde_mm_srli_epi16(b, 1), m);
+    const auto bit0 = simde_mm_xor_si128(a0, b0);
+    const auto borrow = simde_mm_andnot_si128(a0, b0);
+    const auto bit1 = simde_mm_xor_si128(simde_mm_xor_si128(a1, b1), borrow);
+    return simde_mm_or_si128(bit0, simde_mm_slli_epi16(bit1, 1));
+}
+
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+HEDLEY_PURE
+simde__m256i sub_epi2(simde__m256i a, simde__m256i b) noexcept
+{
+    const auto m = simde_mm256_set1_epi8(0x55);
+    const auto a0 = simde_mm256_and_si256(a, m);
+    const auto a1 = simde_mm256_and_si256(simde_mm256_srli_epi16(a, 1), m);
+    const auto b0 = simde_mm256_and_si256(b, m);
+    const auto b1 = simde_mm256_and_si256(simde_mm256_srli_epi16(b, 1), m);
+    const auto bit0 = simde_mm256_xor_si256(a0, b0);
+    const auto borrow = simde_mm256_andnot_si256(a0, b0);
+    const auto bit1 = simde_mm256_xor_si256(simde_mm256_xor_si256(a1, b1), borrow);
+    return simde_mm256_or_si256(bit0, simde_mm256_slli_epi16(bit1, 1));
+}
+
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+HEDLEY_PURE
+simde__m128i mullo_epi2(simde__m128i a, simde__m128i b) noexcept
+{
+    const auto m = simde_mm_set1_epi8(0x55);
+    const auto a0 = simde_mm_and_si128(a, m);
+    const auto a1 = simde_mm_and_si128(simde_mm_srli_epi16(a, 1), m);
+    const auto b0 = simde_mm_and_si128(b, m);
+    const auto b1 = simde_mm_and_si128(simde_mm_srli_epi16(b, 1), m);
+    const auto bit0 = simde_mm_and_si128(a0, b0);
+    const auto bit1 = simde_mm_xor_si128(
+        simde_mm_and_si128(a0, b1), simde_mm_and_si128(a1, b0));
+    return simde_mm_or_si128(bit0, simde_mm_slli_epi16(bit1, 1));
+}
+
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+HEDLEY_PURE
+simde__m256i mullo_epi2(simde__m256i a, simde__m256i b) noexcept
+{
+    const auto m = simde_mm256_set1_epi8(0x55);
+    const auto a0 = simde_mm256_and_si256(a, m);
+    const auto a1 = simde_mm256_and_si256(simde_mm256_srli_epi16(a, 1), m);
+    const auto b0 = simde_mm256_and_si256(b, m);
+    const auto b1 = simde_mm256_and_si256(simde_mm256_srli_epi16(b, 1), m);
+    const auto bit0 = simde_mm256_and_si256(a0, b0);
+    const auto bit1 = simde_mm256_xor_si256(
+        simde_mm256_and_si256(a0, b1), simde_mm256_and_si256(a1, b0));
+    return simde_mm256_or_si256(bit0, simde_mm256_slli_epi16(bit1, 1));
+}
+
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+HEDLEY_PURE
+simde__m128i mul_epi2(simde__m128i a, dpf::twobit k) noexcept
+{
+    unsigned char lut[16];
+    detail::fill_epi2_mul_lut(static_cast<unsigned>(k), lut);
+    return detail::shuffle_nibbles(detail::nibble_lut(lut), a);
+}
+
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+HEDLEY_PURE
+simde__m256i mul_epi2(simde__m256i a, dpf::twobit k) noexcept
+{
+    unsigned char lut[16];
+    detail::fill_epi2_mul_lut(static_cast<unsigned>(k), lut);
+    return detail::shuffle_nibbles(detail::nibble_lut256(lut), a);
+}
+
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+HEDLEY_PURE
+simde__m128i add_epi4(simde__m128i a, simde__m128i b) noexcept
+{
+    const auto lo_m = simde_mm_set1_epi8(0x0f);
+    const auto hi_m = simde_mm_set1_epi8(static_cast<int8_t>(0xf0));
+    const auto lo = simde_mm_add_epi8(
+        simde_mm_and_si128(a, lo_m), simde_mm_and_si128(b, lo_m));
+    const auto hi = simde_mm_add_epi8(
+        simde_mm_and_si128(a, hi_m), simde_mm_and_si128(b, hi_m));
+    return simde_mm_or_si128(
+        simde_mm_and_si128(lo, lo_m), simde_mm_and_si128(hi, hi_m));
+}
+
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+HEDLEY_PURE
+simde__m256i add_epi4(simde__m256i a, simde__m256i b) noexcept
+{
+    const auto lo_m = simde_mm256_set1_epi8(0x0f);
+    const auto hi_m = simde_mm256_set1_epi8(static_cast<int8_t>(0xf0));
+    const auto lo = simde_mm256_add_epi8(
+        simde_mm256_and_si256(a, lo_m), simde_mm256_and_si256(b, lo_m));
+    const auto hi = simde_mm256_add_epi8(
+        simde_mm256_and_si256(a, hi_m), simde_mm256_and_si256(b, hi_m));
+    return simde_mm256_or_si256(
+        simde_mm256_and_si256(lo, lo_m), simde_mm256_and_si256(hi, hi_m));
+}
+
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+HEDLEY_PURE
+simde__m128i sub_epi4(simde__m128i a, simde__m128i b) noexcept
+{
+    const auto lo_m = simde_mm_set1_epi8(0x0f);
+    const auto hi_m = simde_mm_set1_epi8(static_cast<int8_t>(0xf0));
+    const auto lo = simde_mm_sub_epi8(
+        simde_mm_and_si128(a, lo_m), simde_mm_and_si128(b, lo_m));
+    const auto hi = simde_mm_sub_epi8(
+        simde_mm_and_si128(a, hi_m), simde_mm_and_si128(b, hi_m));
+    return simde_mm_or_si128(
+        simde_mm_and_si128(lo, lo_m), simde_mm_and_si128(hi, hi_m));
+}
+
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+HEDLEY_PURE
+simde__m256i sub_epi4(simde__m256i a, simde__m256i b) noexcept
+{
+    const auto lo_m = simde_mm256_set1_epi8(0x0f);
+    const auto hi_m = simde_mm256_set1_epi8(static_cast<int8_t>(0xf0));
+    const auto lo = simde_mm256_sub_epi8(
+        simde_mm256_and_si256(a, lo_m), simde_mm256_and_si256(b, lo_m));
+    const auto hi = simde_mm256_sub_epi8(
+        simde_mm256_and_si256(a, hi_m), simde_mm256_and_si256(b, hi_m));
+    return simde_mm256_or_si256(
+        simde_mm256_and_si256(lo, lo_m), simde_mm256_and_si256(hi, hi_m));
+}
+
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+HEDLEY_PURE
+simde__m128i mullo_epi4(simde__m128i a, simde__m128i b) noexcept
+{
+    const auto m = simde_mm_set1_epi8(0x0f);
+    const auto lo = detail::mul_low_nibbles(a, b);
+    const auto ah = simde_mm_and_si128(simde_mm_srli_epi16(a, 4), m);
+    const auto bh = simde_mm_and_si128(simde_mm_srli_epi16(b, 4), m);
+    const auto hi = detail::mul_low_nibbles(ah, bh);
+    return simde_mm_or_si128(lo, simde_mm_slli_epi16(hi, 4));
+}
+
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+HEDLEY_PURE
+simde__m256i mullo_epi4(simde__m256i a, simde__m256i b) noexcept
+{
+    const auto m = simde_mm256_set1_epi8(0x0f);
+    const auto lo = detail::mul_low_nibbles(a, b);
+    const auto ah = simde_mm256_and_si256(simde_mm256_srli_epi16(a, 4), m);
+    const auto bh = simde_mm256_and_si256(simde_mm256_srli_epi16(b, 4), m);
+    const auto hi = detail::mul_low_nibbles(ah, bh);
+    return simde_mm256_or_si256(lo, simde_mm256_slli_epi16(hi, 4));
+}
+
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+HEDLEY_PURE
+simde__m128i mul_epi4(simde__m128i a, dpf::nyble k) noexcept
+{
+    unsigned char lut[16];
+    detail::fill_epi4_mul_lut(static_cast<unsigned>(k), lut);
+    return detail::shuffle_nibbles(detail::nibble_lut(lut), a);
+}
+
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+HEDLEY_PURE
+simde__m256i mul_epi4(simde__m256i a, dpf::nyble k) noexcept
+{
+    unsigned char lut[16];
+    detail::fill_epi4_mul_lut(static_cast<unsigned>(k), lut);
+    return detail::shuffle_nibbles(detail::nibble_lut256(lut), a);
+}
+
+}  // namespace lane_arith
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_PACKED_LANE_ARITHMETIC_HPP__

include/dpf/parallel_bit_iterable.hpp

@@ -0,0 +1,357 @@
+/// @file dpf/parallel_bit_iterable.hpp
+/// @author Christopher Jiang <christopher.jiang@ucalgary.ca>
+/// @brief
+/// @details
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others]{@ref authors}
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_PARALLEL_BIT_ITERABLE_HPP__
+#define LIBDPF_INCLUDE_DPF_PARALLEL_BIT_ITERABLE_HPP__
+
+#include "hedley/hedley.h"
+
+#include <cstddef>
+#include <cmath>
+#include <cstring>
+#include <type_traits>
+#include <iterator>
+#include <array>
+#include <utility>
+#include <algorithm>
+#include <functional>
+
+#include "dpf/bit_array.hpp"
+#include "dpf/parallel_bit_iterable_helpers.hpp"
+
+namespace dpf
+{
+
+template <std::size_t BatchSize,
+          typename ChildT>
+class parallel_const_bit_iterator;  // forward declaration
+
+template <std::size_t BatchSize,
+          typename ChildT>
+class parallel_bit_iterable
+{
+  public:
+    using word_pointer = typename bit_array_base<ChildT>::const_word_pointer;
+    static constexpr auto batch_size = BatchSize;
+    using const_iterator = parallel_const_bit_iterator<batch_size, ChildT>;
+
+    template <typename Iter>
+    explicit parallel_bit_iterable(Iter it)
+      : begin_{init_array(it,
+            [](Iter it){ return it->data(); })},
+        end_{init_array(it,
+            [](Iter it){ return it->data() + it->data_length(); })}
+    { }
+
+    template <typename T,
+              typename ...Ts>
+    explicit parallel_bit_iterable(const T & t, const Ts & ...ts)
+      : begin_{t.data(), ts.data()...},
+        end_{t.data()+t.data_length(), ts.data()+ts.data_length()...}
+    { }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    const_iterator begin() const noexcept
+    {
+        return const_iterator{begin_};
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    const_iterator cbegin() const noexcept
+    {
+        return begin();
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    const_iterator end() const noexcept
+    {
+        return const_iterator{end_};
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    const_iterator cend() const noexcept
+    {
+        return end();
+    }
+
+  private:
+    using array_type = std::array<word_pointer, batch_size>;
+
+    template <typename F,
+              typename Iter,
+              std::size_t ...Is>
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    static constexpr array_type init_array_impl(Iter it,
+        std::index_sequence<Is...>, const F & lambda) noexcept
+    {
+        return {{ ((void)Is, lambda(it++))... }};
+    }
+
+    template <typename F,
+              typename Iter,
+              typename Indices = std::make_index_sequence<batch_size>>
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    static constexpr array_type init_array(Iter it,
+        const F & lambda) noexcept
+    {
+        return init_array_impl(it, Indices{}, lambda);
+    }
+
+    const array_type begin_, end_;
+};  // class dpf::parallel_bit_iterable
+
+template <std::size_t N,
+          typename ChildT>
+class parallel_const_bit_iterator
+{
+  private:
+    using word_type = typename bit_array_base<ChildT>::word_type;
+    using word_array = std::array<word_type, N>;
+    using word_pointer = typename bit_array_base<ChildT>::const_word_pointer;
+    using word_pointer_array = std::array<word_pointer, N>;
+    static constexpr std::size_t lg_batch_size = (N <= 2)
+                                     ? 2 : std::ceil(std::log2(N));
+    using helper = dpf::parallel_bit_iterable_helper<lg_batch_size, ChildT>;
+    using element_type = typename helper::element_type;
+    using simde_type = typename helper::simde_type;
+    static constexpr auto bits_per_word = bit_array_base<ChildT>::bits_per_word;
+    static constexpr auto bits_per_element = helper::bits_per_element;
+    static constexpr auto bytes_per_batch = N * (bits_per_element/CHAR_BIT);
+    static_assert(CHAR_BIT == 8, "CHAR_BIT not equal to 8");
+    static constexpr auto elements_per_word = helper::elements_per_word;
+    using simde_array = typename helper::simde_array;
+
+  public:
+    static constexpr auto batch_size = N;
+
+    using iterator_category = std::bidirectional_iterator_tag;
+    using difference_type = std::array<std::pair<std::ptrdiff_t, std::size_t>,
+                                       batch_size>;
+    using value_type = std::array<element_type, batch_size>;
+    using reference = value_type &;
+    using const_reference = const value_type &;
+    using pointer = std::add_pointer_t<value_type>;
+
+    HEDLEY_ALWAYS_INLINE
+    constexpr
+    parallel_const_bit_iterator(parallel_const_bit_iterator &&) noexcept = default;
+    HEDLEY_ALWAYS_INLINE
+    constexpr
+    parallel_const_bit_iterator(const parallel_const_bit_iterator &) noexcept = default;
+
+    HEDLEY_PURE
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    value_type operator*() const noexcept
+    {
+        value_type ret;
+        simde_type temp = helper::bit_and(all_vecs_[element_cnt_], vec_mask_);
+        std::memcpy(ret.data(), &temp, bytes_per_batch);
+        return ret;
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    parallel_const_bit_iterator & operator++() noexcept
+    {
+        if (HEDLEY_UNLIKELY(!(word_mask_ <<= 1)))
+        {
+            word_mask_ = word_lsb;
+            element_mask_ = element_lsb;
+            vec_mask_ = helper::right_shift(vec_mask_, bits_per_element - 1);
+            element_cnt_ = 0;
+
+            std::transform(iter_.begin(), iter_.end(), cur_word_.begin(),
+                [](auto & it)
+                {
+                    return *(++it);
+                });
+            all_vecs_ = helper::build_vecs(cur_word_.data(), batch_size);
+        }
+        else if (HEDLEY_UNLIKELY(!(element_mask_ <<= 1)))
+        {
+            element_mask_ = element_lsb;
+            vec_mask_ = helper::right_shift(vec_mask_, bits_per_element - 1);
+            ++element_cnt_;
+        }
+        else
+        {
+            vec_mask_ = helper::left_shift(vec_mask_, 1);
+        }
+        return *this;
+    }
+
+    HEDLEY_NO_THROW
+    parallel_const_bit_iterator operator++(int) noexcept
+    {
+        auto tmp = *this;
+        parallel_const_bit_iterator::operator++();
+        return tmp;
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    parallel_const_bit_iterator & operator--() noexcept
+    {
+        if (HEDLEY_UNLIKELY(!(word_mask_ >>= 1)))
+        {
+            word_mask_ = word_msb;
+            element_mask_ = element_msb;
+            vec_mask_ = helper::left_shift(vec_mask_, bits_per_element - 1);
+            element_cnt_ = elements_per_word - 1;
+
+            std::transform(iter_.begin(), iter_.end(), cur_word_.begin(),
+                [](auto & it)
+                {
+                    return *(--it);
+                });
+            all_vecs_ = helper::build_vecs(cur_word_.data(), batch_size);
+        }
+        else if (HEDLEY_UNLIKELY(!(element_mask_ >>= 1)))
+        {
+            element_mask_ = element_msb;
+            vec_mask_ = helper::left_shift(vec_mask_, bits_per_element - 1);
+            --element_cnt_;
+        }
+        else
+        {
+            vec_mask_ = helper::right_shift(vec_mask_, 1);
+        }
+        return *this;
+    }
+
+    HEDLEY_NO_THROW
+    parallel_const_bit_iterator operator--(int) noexcept
+    {
+        auto tmp = *this;
+        parallel_const_bit_iterator::operator--();
+        return tmp;
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr bool operator==(const parallel_const_bit_iterator & rhs) const noexcept
+    {
+        return (word_mask_ == rhs.word_mask_)
+            && (std::equal(iter_.begin(), iter_.end(), rhs.iter_.begin()));
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr bool operator!=(const parallel_const_bit_iterator & rhs) const noexcept
+    {
+        return !(*this == rhs);
+    }
+
+  private:
+    static constexpr word_type word_lsb = word_type(1);
+    static constexpr word_type word_msb = word_lsb << (bits_per_word-1);
+    static constexpr element_type element_lsb = element_type(1);
+    static constexpr element_type element_msb = element_lsb << (bits_per_element-1);
+
+    template<std::size_t ...Is>
+    word_array dereferencing_initializer_impl(
+        const word_pointer_array & arr, std::index_sequence<Is...>)
+    {
+        return {{ *(arr[Is])... }};
+    }
+
+    word_array dereferencing_initializer(const word_pointer_array & arr)
+    {
+        return dereferencing_initializer_impl(arr,
+            std::make_index_sequence<batch_size>());
+    }
+
+    explicit constexpr parallel_const_bit_iterator(
+        const word_pointer_array & arr) noexcept
+      : iter_{arr},
+        word_mask_{word_lsb},
+        element_mask_{element_lsb},
+        element_cnt_{0},
+        cur_word_{dereferencing_initializer(arr)},
+        vec_mask_{helper::get_mask()},
+        all_vecs_{helper::build_vecs(cur_word_.data(), batch_size)}
+    { }
+
+    word_pointer_array iter_;
+    word_type word_mask_;
+    element_type element_mask_;
+    std::size_t element_cnt_;
+
+    word_array cur_word_;
+    simde_type vec_mask_;
+    simde_array all_vecs_;
+
+    friend parallel_const_bit_iterator parallel_bit_iterable<batch_size, ChildT>::begin() const noexcept;
+    friend parallel_const_bit_iterator parallel_bit_iterable<batch_size, ChildT>::end() const noexcept;
+};  // class dpf::parallel_const_bit_iterator
+
+template <std::size_t N,
+          typename ChildT,
+          typename Iter>
+HEDLEY_PURE
+HEDLEY_ALWAYS_INLINE
+auto batch_of(Iter it) noexcept
+{
+    return dpf::parallel_bit_iterable<N, ChildT>{it};
+}
+
+template <typename ChildT,
+          typename ...Ts>
+HEDLEY_PURE
+HEDLEY_ALWAYS_INLINE
+auto batch_of(const dpf::bit_array_base<ChildT> & t, const Ts & ...ts) noexcept
+{
+    return dpf::parallel_bit_iterable<1+sizeof...(Ts), ChildT>{t, ts...};
+}
+
+template <std::size_t N, typename Iter,
+          typename UnaryFunction>
+HEDLEY_ALWAYS_INLINE
+void for_each_bit_parallel(Iter it, UnaryFunction f)
+{
+    for (auto i : batch_of<N>(it)) f(i);
+}
+
+template <typename ChildT,
+          typename UnaryFunction,
+          typename ...Ts>
+HEDLEY_ALWAYS_INLINE
+void for_each_bit_parallel(const dpf::bit_array_base<ChildT> & t, const Ts & ...ts, UnaryFunction f)
+{
+    for (auto i : batch_of<1+sizeof...(Ts), ChildT>(t, ts...)) f(i);
+}
+
+}  // namespace dpf
+
+namespace std
+{
+
+template <std::size_t N, typename ChildT>
+struct iterator_traits<typename dpf::parallel_const_bit_iterator<N, ChildT>>
+{
+  private:
+    using type = dpf::parallel_const_bit_iterator<N, ChildT>;
+  public:
+    using iterator_category = typename type::iterator_category;
+    using difference_type = typename type::difference_type;
+    using value_type = typename type::value_type;
+    using reference = typename type::reference;
+    using const_reference = typename type::const_reference;
+    using pointer = typename type::pointer;
+};
+
+}  // namespace std
+
+#endif  // LIBDPF_INCLUDE_DPF_PARALLEL_BIT_ITERABLE_HPP__

include/dpf/parallel_bit_iterable_helpers.hpp

@@ -0,0 +1,298 @@
+/// @file dpf/parallel_bit_iterable_helpers.hpp
+/// @author Christopher Jiang <christopher.jiang@ucalgary.ca>
+/// @brief
+/// @details
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_PARALLEL_BIT_ITERABLE_HELPERS_HPP__
+#define LIBDPF_INCLUDE_DPF_PARALLEL_BIT_ITERABLE_HELPERS_HPP__
+
+#include "hedley/hedley.h"
+
+#include <cstddef>
+#include <cstring>
+#include <limits>
+#include <array>
+#include <type_traits>
+
+#include "simde/simde/x86/avx2.h"
+#include "portable-snippets/exact-int/exact-int.h"
+
+#include "dpf/bit_array.hpp"
+
+namespace dpf
+{
+
+namespace
+{
+
+/// Unaligned 256-bit load of `words_per_vec` words starting at `offset`.
+/// Words past `nwords` are zero so a short batch does not read off the end.
+template <typename Word>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+simde__m256i loadu_word_vec(const Word * words, std::size_t nwords,
+    std::size_t offset) noexcept
+{
+    constexpr std::size_t words_per_vec = sizeof(simde__m256i) / sizeof(Word);
+    static_assert(sizeof(simde__m256i) % sizeof(Word) == 0,
+        "word size must divide a 256-bit vector");
+    if (offset >= nwords)
+    {
+        return simde_mm256_setzero_si256();
+    }
+    if (offset + words_per_vec <= nwords)
+    {
+        return simde_mm256_loadu_si256(static_cast<const void *>(words + offset));
+    }
+    alignas(simde__m256i) Word tmp[words_per_vec]{};
+    std::memcpy(tmp, words + offset, (nwords - offset) * sizeof(Word));
+    return simde_mm256_loadu_si256(static_cast<const void *>(tmp));
+}
+
+template <std::size_t batch_size_log_2, typename ChildT>
+struct parallel_bit_iterable_helper;
+
+/// @brief for batch_size in 1..4
+template <typename ChildT>
+struct parallel_bit_iterable_helper<2, ChildT>
+{
+  public:
+    using word_type = typename dpf::bit_array_base<ChildT>::word_type;
+    using word_pointer = typename dpf::bit_array_base<ChildT>::word_pointer;
+    using simde_type = simde__m256i;
+    using simde_ptr = simde_type *;
+    using element_type = psnip_uint64_t;
+    static constexpr auto bits_per_word = dpf::bit_array_base<ChildT>::bits_per_word;
+    static constexpr auto bits_per_element = std::numeric_limits<element_type>::digits;
+    static constexpr auto elements_per_word = bits_per_word / bits_per_element;
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    using simde_array = std::array<simde_type, elements_per_word>;
+HEDLEY_PRAGMA(GCC diagnostic pop)
+    static constexpr auto left_shift = simde_mm256_slli_epi64;
+    static constexpr auto right_shift = simde_mm256_srli_epi64;
+    static constexpr auto bit_and = simde_mm256_and_si256;
+    static auto get_mask() noexcept
+    {
+        return simde_mm256_set1_epi64x(1);
+    }
+    static simde_array build_vecs(const word_type * cur_word, std::size_t nwords) noexcept
+    {
+        return { loadu_word_vec(cur_word, nwords, 0) };
+    }
+};  // struct parallel_bit_iterable_helper<2>
+
+/// @brief for batch_size in 5..8
+template <typename ChildT>
+struct parallel_bit_iterable_helper<3, ChildT>
+{
+  public:
+    using word_pointer = typename dpf::bit_array_base<ChildT>::word_pointer;
+    using simde_type = simde__m256i;
+    using simde_ptr = simde_type *;
+    using element_type = psnip_uint32_t;
+    static constexpr auto bits_per_word = dpf::bit_array_base<ChildT>::bits_per_word;
+    static constexpr auto bits_per_element = std::numeric_limits<element_type>::digits;
+    static constexpr auto elements_per_word = bits_per_word / bits_per_element;
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    using simde_array = std::array<simde_type, elements_per_word>;
+HEDLEY_PRAGMA(GCC diagnostic pop)
+    static constexpr auto left_shift = simde_mm256_slli_epi64;
+    static constexpr auto right_shift = simde_mm256_srli_epi64;
+    static constexpr auto bit_and = simde_mm256_and_si256;
+    static auto get_mask() noexcept
+    {
+        return simde_mm256_set1_epi32(1);
+    }
+    static simde_array build_vecs(const typename dpf::bit_array_base<ChildT>::word_type * cur_word,
+        std::size_t nwords) noexcept
+    {
+        using word_type_ = typename dpf::bit_array_base<ChildT>::word_type;
+        simde_type vec1 = loadu_word_vec<word_type_>(cur_word, nwords, 0);
+        simde_type vec2 = loadu_word_vec<word_type_>(cur_word, nwords, 4);
+
+        vec1 = simde_mm256_permutevar8x32_epi32(vec1, mask1);
+        vec2 = simde_mm256_permutevar8x32_epi32(vec2, mask1);
+
+        return {
+            simde_mm256_permute2x128_si256(vec1, vec2, 0b00100000),
+            simde_mm256_permute2x128_si256(vec1, vec2, 0b00110001) };
+    }
+  private:
+    static constexpr simde_type mask1 = {
+        0b0000000000000000000000000000001000000000000000000000000000000000,
+        0b0000000000000000000000000000011000000000000000000000000000000100,
+        0b0000000000000000000000000000001100000000000000000000000000000001,
+        0b0000000000000000000000000000011100000000000000000000000000000101};
+};  // struct parallel_bit_iterable_helper<3>
+
+/// @brief for batch_size in 9..16
+template <typename ChildT>
+struct parallel_bit_iterable_helper<4, ChildT>
+{
+  public:
+    using word_pointer = typename dpf::bit_array_base<ChildT>::word_pointer;
+    using simde_type = simde__m256i;
+    using simde_ptr = simde_type *;
+    using element_type = psnip_uint16_t;
+    static constexpr auto bits_per_word = dpf::bit_array_base<ChildT>::bits_per_word;
+    static constexpr auto bits_per_element = std::numeric_limits<element_type>::digits;
+    static constexpr auto elements_per_word = bits_per_word / bits_per_element;
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    using simde_array = std::array<simde_type, elements_per_word>;
+HEDLEY_PRAGMA(GCC diagnostic pop)
+    static constexpr auto left_shift = simde_mm256_slli_epi64;
+    static constexpr auto right_shift = simde_mm256_srli_epi64;
+    static constexpr auto bit_and = simde_mm256_and_si256;
+    static auto get_mask() noexcept
+    {
+        return simde_mm256_set1_epi16(1);
+    }
+    static simde_array build_vecs(const typename dpf::bit_array_base<ChildT>::word_type * cur_word,
+        std::size_t nwords) noexcept
+    {
+        using word_type_ = typename dpf::bit_array_base<ChildT>::word_type;
+        simde_type vec1 = loadu_word_vec<word_type_>(cur_word, nwords, 0);
+        simde_type vec2 = loadu_word_vec<word_type_>(cur_word, nwords, 4);
+        simde_type vec3 = loadu_word_vec<word_type_>(cur_word, nwords, 8);
+        simde_type vec4 = loadu_word_vec<word_type_>(cur_word, nwords, 12);
+
+        vec1 = simde_mm256_permutevar8x32_epi32(vec1, mask1);
+        vec2 = simde_mm256_permutevar8x32_epi32(vec2, mask1);
+        vec3 = simde_mm256_permutevar8x32_epi32(vec3, mask1);
+        vec4 = simde_mm256_permutevar8x32_epi32(vec4, mask1);
+
+        vec1 = simde_mm256_shuffle_epi8(vec1, mask2);
+        vec2 = simde_mm256_shuffle_epi8(vec2, mask2);
+        vec3 = simde_mm256_shuffle_epi8(vec3, mask2);
+        vec4 = simde_mm256_shuffle_epi8(vec4, mask2);
+
+        simde_type bld_0002 = simde_mm256_unpacklo_epi64(vec1, vec2);
+        simde_type bld_0406 = simde_mm256_unpacklo_epi64(vec3, vec4);
+        simde_type bld_0103 = simde_mm256_unpackhi_epi64(vec1, vec2);
+        simde_type bld_0507 = simde_mm256_unpackhi_epi64(vec3, vec4);
+
+        return {
+            simde_mm256_permute2x128_si256(bld_0002, bld_0406, 0b00100000),
+            simde_mm256_permute2x128_si256(bld_0103, bld_0507, 0b00100000),
+            simde_mm256_permute2x128_si256(bld_0002, bld_0406, 0b00110001),
+            simde_mm256_permute2x128_si256(bld_0103, bld_0507, 0b00110001) };
+    }
+  private:
+    static constexpr simde_type mask1 = {
+        0b0000000000000000000000000000001000000000000000000000000000000000,
+        0b0000000000000000000000000000011000000000000000000000000000000100,
+        0b0000000000000000000000000000001100000000000000000000000000000001,
+        0b0000000000000000000000000000011100000000000000000000000000000101};
+    static constexpr simde_type mask2 = {
+        0b0000110100001100000010010000100000000101000001000000000100000000,
+        0b0000111100001110000010110000101000000111000001100000001100000010,
+        0b0000110100001100000010010000100000000101000001000000000100000000,
+        0b0000111100001110000010110000101000000111000001100000001100000010};
+};  // struct parallel_bit_iterable_helper<4>
+
+/// @brief for batch_size in 17..32
+template <typename ChildT>
+struct parallel_bit_iterable_helper<5, ChildT>
+{
+  public:
+    using word_pointer = typename dpf::bit_array_base<ChildT>::word_pointer;
+    using simde_type = simde__m256i;
+    using simde_ptr = simde_type *;
+    using element_type = psnip_uint8_t;
+    static constexpr auto bits_per_word = dpf::bit_array_base<ChildT>::bits_per_word;
+    static constexpr auto bits_per_element = std::numeric_limits<element_type>::digits;
+    static constexpr auto elements_per_word = bits_per_word / bits_per_element;
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    using simde_array = std::array<simde_type, elements_per_word>;
+HEDLEY_PRAGMA(GCC diagnostic pop)
+    static constexpr auto left_shift = simde_mm256_slli_epi64;
+    static constexpr auto right_shift = simde_mm256_srli_epi64;
+    static constexpr auto bit_and = simde_mm256_and_si256;
+    static auto get_mask() noexcept
+    {
+        return simde_mm256_set1_epi8(1);
+    }
+    static simde_array build_vecs(const typename dpf::bit_array_base<ChildT>::word_type * cur_word,
+        std::size_t nwords) noexcept
+    {
+        using word_type_ = typename dpf::bit_array_base<ChildT>::word_type;
+        simde_type vec1 = loadu_word_vec<word_type_>(cur_word, nwords, 0);
+        simde_type vec2 = loadu_word_vec<word_type_>(cur_word, nwords, 4);
+        simde_type vec3 = loadu_word_vec<word_type_>(cur_word, nwords, 8);
+        simde_type vec4 = loadu_word_vec<word_type_>(cur_word, nwords, 12);
+        simde_type vec5 = loadu_word_vec<word_type_>(cur_word, nwords, 16);
+        simde_type vec6 = loadu_word_vec<word_type_>(cur_word, nwords, 20);
+        simde_type vec7 = loadu_word_vec<word_type_>(cur_word, nwords, 24);
+        simde_type vec8 = loadu_word_vec<word_type_>(cur_word, nwords, 28);
+
+        vec1 = simde_mm256_permutevar8x32_epi32(vec1, mask1);
+        vec2 = simde_mm256_permutevar8x32_epi32(vec2, mask1);
+        vec3 = simde_mm256_permutevar8x32_epi32(vec3, mask1);
+        vec4 = simde_mm256_permutevar8x32_epi32(vec4, mask1);
+        vec5 = simde_mm256_permutevar8x32_epi32(vec5, mask1);
+        vec6 = simde_mm256_permutevar8x32_epi32(vec6, mask1);
+        vec7 = simde_mm256_permutevar8x32_epi32(vec7, mask1);
+        vec8 = simde_mm256_permutevar8x32_epi32(vec8, mask1);
+
+        vec1 = simde_mm256_shuffle_epi8(vec1, mask2);
+        vec2 = simde_mm256_shuffle_epi8(vec2, mask2);
+        vec3 = simde_mm256_shuffle_epi8(vec3, mask2);
+        vec4 = simde_mm256_shuffle_epi8(vec4, mask2);
+        vec5 = simde_mm256_shuffle_epi8(vec5, mask2);
+        vec6 = simde_mm256_shuffle_epi8(vec6, mask2);
+        vec7 = simde_mm256_shuffle_epi8(vec7, mask2);
+        vec8 = simde_mm256_shuffle_epi8(vec8, mask2);
+
+        simde_type bld_00010405 = simde_mm256_unpacklo_epi32(vec1, vec2);
+        simde_type bld_08091213 = simde_mm256_unpacklo_epi32(vec3, vec4);
+        simde_type bld_16172021 = simde_mm256_unpacklo_epi32(vec5, vec6);
+        simde_type bld_24252829 = simde_mm256_unpacklo_epi32(vec7, vec8);
+        simde_type bld_02030607 = simde_mm256_unpackhi_epi32(vec1, vec2);
+        simde_type bld_10111415 = simde_mm256_unpackhi_epi32(vec3, vec4);
+        simde_type bld_18192223 = simde_mm256_unpackhi_epi32(vec5, vec6);
+        simde_type bld_26273031 = simde_mm256_unpackhi_epi32(vec7, vec8);
+
+        simde_type bld_0004 = simde_mm256_unpacklo_epi64(bld_00010405, bld_08091213);
+        simde_type bld_1620 = simde_mm256_unpacklo_epi64(bld_16172021, bld_24252829);
+        simde_type bld_0206 = simde_mm256_unpacklo_epi64(bld_02030607, bld_10111415);
+        simde_type bld_1822 = simde_mm256_unpacklo_epi64(bld_18192223, bld_26273031);
+        simde_type bld_0105 = simde_mm256_unpackhi_epi64(bld_00010405, bld_08091213);
+        simde_type bld_1721 = simde_mm256_unpackhi_epi64(bld_16172021, bld_24252829);
+        simde_type bld_0307 = simde_mm256_unpackhi_epi64(bld_02030607, bld_10111415);
+        simde_type bld_1923 = simde_mm256_unpackhi_epi64(bld_18192223, bld_26273031);
+
+        return {
+            simde_mm256_permute2x128_si256(bld_0004, bld_1620, 0b00100000),
+            simde_mm256_permute2x128_si256(bld_0105, bld_1721, 0b00100000),
+            simde_mm256_permute2x128_si256(bld_0206, bld_1822, 0b00100000),
+            simde_mm256_permute2x128_si256(bld_0307, bld_1923, 0b00100000),
+            simde_mm256_permute2x128_si256(bld_0004, bld_1620, 0b00110001),
+            simde_mm256_permute2x128_si256(bld_0105, bld_1721, 0b00110001),
+            simde_mm256_permute2x128_si256(bld_0206, bld_1822, 0b00110001),
+            simde_mm256_permute2x128_si256(bld_0307, bld_1923, 0b00110001) };
+    }
+  private:
+    static constexpr simde_type mask1 = {
+        0b0000000000000000000000000000001000000000000000000000000000000000,
+        0b0000000000000000000000000000011000000000000000000000000000000100,
+        0b0000000000000000000000000000001100000000000000000000000000000001,
+        0b0000000000000000000000000000011100000000000000000000000000000101};
+    static constexpr simde_type mask2 = {
+        0b0000110100001001000001010000000100001100000010000000010000000000,
+        0b0000111100001011000001110000001100001110000010100000011000000010,
+        0b0000110100001001000001010000000100001100000010000000010000000000,
+        0b0000111100001011000001110000001100001110000010100000011000000010};
+};  // struct parallel_bit_iterable_helper<5>
+
+}  // anonymous namespace
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_PARALLEL_BIT_ITERABLE_HELPERS_HPP__

include/dpf/path_memoizer.hpp

@@ -0,0 +1,289 @@
+/// @file dpf/path_memoizer.hpp
+/// @brief
+/// @details
+/// @author Ryan Henry <ryan.henry@ucalgary.ca>
+/// @author Christopher Jiang <christopher.jiang@ucalgary.ca>
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_PATH_MEMOIZER_HPP__
+#define LIBDPF_INCLUDE_DPF_PATH_MEMOIZER_HPP__
+
+#include "hedley/hedley.h"
+
+#include <cstddef>
+#include <cstring>
+#include <algorithm>
+#include <type_traits>
+#include <functional>
+#include <memory>
+#include <array>
+#include <optional>
+
+#include "dpf/utils.hpp"
+#include "dpf/secret_share.hpp"
+
+namespace dpf
+{
+
+/// Path memoizers key on the underlying DPF key type. `party_key` wrappers
+/// share the same tree layout, so a memoizer built for party 0 also accepts
+/// party 1 (and bare keys).
+template <typename T>
+using path_memoizer_key_t = unwrap_party_key_t<T>;
+
+template <typename DpfKey,
+          typename ReturnT = const typename DpfKey::interior_node *>
+struct path_memoizer_base
+{
+  public:
+    using dpf_type = DpfKey;
+    using input_type = typename DpfKey::input_type;
+    using node_type = typename DpfKey::interior_node;
+    using return_type = ReturnT;
+    using iterator_type = return_type;
+
+    virtual std::size_t assign_x(const dpf_type &, input_type) noexcept = 0;
+    virtual node_type & operator[](std::size_t) noexcept = 0;
+
+    virtual return_type begin() const noexcept = 0;
+    virtual return_type end() const noexcept = 0;
+};
+
+template <typename DpfKey>
+struct alignas(alignof(typename path_memoizer_key_t<DpfKey>::interior_node))
+basic_path_memoizer final
+    : public path_memoizer_base<path_memoizer_key_t<DpfKey>>
+{
+  public:
+    using dpf_type = path_memoizer_key_t<DpfKey>;
+    using input_type = typename dpf_type::input_type;
+    using node_type = typename dpf_type::interior_node;
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    using return_type = std::add_pointer_t<std::add_const_t<node_type>>;
+HEDLEY_PRAGMA(GCC diagnostic pop)
+    using iterator_type = return_type;
+    static constexpr auto depth = dpf_type::depth;
+
+    basic_path_memoizer()
+      : dpf_{std::nullopt}, x_{std::nullopt}, filled_to_{0} { }
+    basic_path_memoizer(basic_path_memoizer &&) noexcept = default;
+    basic_path_memoizer(const basic_path_memoizer &) = default;
+    basic_path_memoizer & operator=(basic_path_memoizer &&) noexcept = default;
+    basic_path_memoizer & operator=(const basic_path_memoizer &) = default;
+    ~basic_path_memoizer() = default;
+
+    std::size_t assign_x(const dpf_type & dpf, input_type new_x) noexcept override
+    {
+        static constexpr auto clz_xor = utils::countl_zero_symmetric_difference<input_type>{};
+        if (dpf_.has_value() == true && std::memcmp(&dpf_root_, &dpf.root(), sizeof(node_type)) == 0
+            && std::memcmp(&dpf_common_part_hash_, &dpf.common_part_hash(), sizeof(digest_type)) == 0)
+        {
+            static constexpr auto complement_of = std::bit_not{};
+            input_type old_x = x_.value_or(complement_of(new_x));
+            x_ = new_x;
+            const std::size_t resume = clz_xor(old_x, new_x) + 1;
+            // Path slots below `resume` remain valid; invalidate the rest.
+            if (filled_to_ >= resume)
+                filled_to_ = resume - 1;
+            return resume;
+        }
+
+        this->operator[](0) = dpf.root();
+        dpf_ = std::cref(dpf);
+        dpf_root_ = dpf.root();
+        dpf_common_part_hash_ = dpf.common_part_hash();
+        x_ = new_x;
+        filled_to_ = 0;
+        return 1;
+    }
+
+    node_type & operator[](std::size_t i) noexcept override
+    {
+        return arr_[i];
+    }
+
+    return_type begin() const noexcept override
+    {
+        if (x_.has_value() == true)
+        {
+            return std::addressof(arr_[depth]);
+        }
+        else
+        {
+            return end();
+        }
+    }
+
+    return_type end() const noexcept override
+    {
+        return std::addressof(arr_[depth+1]);
+    }
+
+    /// Inclusive high-water: `arr_[0..filled_to_]` are valid for the current x.
+    std::size_t filled_to() const noexcept { return filled_to_; }
+
+    void note_filled(std::size_t level) noexcept
+    {
+        if (level > filled_to_)
+            filled_to_ = level;
+    }
+
+  private:
+    std::optional<std::reference_wrapper<const dpf_type>> dpf_;
+    node_type dpf_root_;
+    digest_type dpf_common_part_hash_;
+    std::optional<input_type> x_;
+    std::size_t filled_to_;
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    std::array<node_type, depth+1> arr_;
+HEDLEY_PRAGMA(GCC diagnostic pop)
+};
+
+template <typename DpfKey>
+struct nonmemoizing_path_memoizer final
+    : public path_memoizer_base<path_memoizer_key_t<DpfKey>>
+{
+  public:
+    using dpf_type = path_memoizer_key_t<DpfKey>;
+    using input_type = typename dpf_type::input_type;
+    using node_type = typename dpf_type::interior_node;
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    using return_type = std::add_pointer_t<std::add_const_t<node_type>>;
+HEDLEY_PRAGMA(GCC diagnostic pop)
+    using iterator_type = return_type;
+
+    nonmemoizing_path_memoizer()
+      : dpf_{std::nullopt} { }
+    nonmemoizing_path_memoizer(nonmemoizing_path_memoizer &&) noexcept = default;
+    nonmemoizing_path_memoizer(const nonmemoizing_path_memoizer &) = default;
+    nonmemoizing_path_memoizer & operator=(nonmemoizing_path_memoizer &&) noexcept = default;
+    nonmemoizing_path_memoizer & operator=(const nonmemoizing_path_memoizer &) = default;
+    ~nonmemoizing_path_memoizer() = default;
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    std::size_t assign_x(const dpf_type & dpf, input_type) noexcept override
+    {
+        dpf_ = dpf;
+        v = dpf.root();
+        return 1;
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    node_type & operator[](std::size_t) noexcept override
+    {
+        return v;
+    }
+
+    return_type begin() const noexcept override
+    {
+        return std::addressof(v);
+    }
+
+    return_type end() const noexcept override
+    {
+        return std::addressof(v) + 1;
+    }
+
+  private:
+    std::optional<std::reference_wrapper<const dpf_type>> dpf_;
+    node_type v;
+};
+
+namespace detail
+{
+
+template <typename MemoizerT>
+HEDLEY_ALWAYS_INLINE
+auto make_path_memoizer()
+{
+    return MemoizerT();
+}
+
+template <typename T, typename = void>
+struct has_path_high_water : std::false_type {};
+
+template <typename T>
+struct has_path_high_water<T,
+    std::void_t<decltype(std::declval<T &>().filled_to()),
+                decltype(std::declval<T &>().note_filled(std::size_t{}))>>
+    : std::true_type {};
+
+template <typename DpfKey, typename PathMemoizer>
+std::size_t path_resume_for_level(PathMemoizer & path, const DpfKey & dpf,
+    typename DpfKey::input_type x, std::size_t to_level)
+{
+    auto resume = path.assign_x(dpf, x);
+    if constexpr (has_path_high_water<PathMemoizer>::value)
+    {
+        resume = std::min(resume, path.filled_to() + 1);
+        if (resume == 0)
+            resume = 1;
+    }
+    else if (resume > to_level)
+    {
+        resume = 1;
+    }
+    return resume;
+}
+
+template <typename PathMemoizer>
+void path_note_filled_to(PathMemoizer & path, std::size_t level)
+{
+    if constexpr (has_path_high_water<PathMemoizer>::value)
+        path.note_filled(level);
+}
+
+/// Walk interior nodes so `path[0..to_level]` is valid for `x`.
+template <typename DpfKey, typename PathMemoizer>
+void ensure_level(const DpfKey & dpf, typename DpfKey::input_type x,
+    PathMemoizer & path, std::size_t to_level)
+{
+    auto level_index = path_resume_for_level(path, dpf, x, to_level);
+    DPF_UNROLL_LOOP
+    for (auto mask = dpf.msb_mask >> (level_index - 1);
+         level_index <= to_level; ++level_index, mask >>= 1)
+    {
+        bool bit = !!(mask & x);
+        auto cw = dpf.correction_word(level_index - 1, bit);
+        path[level_index] =
+            DpfKey::traverse_interior(path[level_index - 1], cw, bit);
+    }
+    path_note_filled_to(path, to_level);
+}
+
+}  // namespace detail
+
+template <typename DpfKey>
+auto make_basic_path_memoizer()
+{
+    return detail::make_path_memoizer<basic_path_memoizer<DpfKey>>();
+}
+
+template <typename DpfKey>
+auto make_basic_path_memoizer(const DpfKey &)
+{
+    return make_basic_path_memoizer<DpfKey>();
+}
+
+template <typename DpfKey>
+auto make_nonmemoizing_path_memoizer()
+{
+    return detail::make_path_memoizer<nonmemoizing_path_memoizer<DpfKey>>();
+}
+
+template <typename DpfKey>
+auto make_nonmemoizing_path_memoizer(const DpfKey &)
+{
+    return make_nonmemoizing_path_memoizer<DpfKey>();
+}
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_PATH_MEMOIZER_HPP__

include/dpf/placement.hpp

@@ -0,0 +1,444 @@
+/// @file dpf/placement.hpp
+/// @brief Prefix placement (`at<N>`), phantom cmp tag, and slot-meta machinery.
+/// @details Shared by `dpf_key` (unified key) and `incremental.hpp` (gen/eval).
+///          Holds the pure type-level pieces so `dpf_key.hpp` can build a
+///          `slot_meta` table for multi-level / cmp keys without depending on
+///          the generation / evaluation code that lives in `incremental.hpp`.
+/// @copyright Copyright (c) 2019-2026 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license.
+
+#ifndef LIBDPF_INCLUDE_DPF_PLACEMENT_HPP__
+#define LIBDPF_INCLUDE_DPF_PLACEMENT_HPP__
+
+#include <cstddef>
+#include <array>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+#include <algorithm>
+
+#include "dpf/utils.hpp"
+#include "dpf/leaf_node.hpp"
+#include "dpf/wildcard.hpp"
+
+namespace dpf
+{
+
+// ---------------------------------------------------------------------------
+// Public placement sugar: `at<N>(y, ys...)`
+// ---------------------------------------------------------------------------
+
+template <std::size_t N, typename OutputT, typename ...OutputTs>
+struct at_pack
+{
+    static constexpr std::size_t prefix = N;
+    using outputs_tuple = std::tuple<OutputT, OutputTs...>;
+    outputs_tuple values;
+
+    at_pack() = delete;
+    explicit at_pack(OutputT y, OutputTs ...ys)
+      : values{std::move(y), std::move(ys)...} { }
+};
+
+template <std::size_t N>
+struct at_fn
+{
+    template <typename OutputT, typename ...OutputTs>
+    constexpr auto operator()(OutputT y, OutputTs ...ys) const
+    {
+        return at_pack<N, OutputT, OutputTs...>(std::move(y), std::move(ys)...);
+    }
+};
+
+template <std::size_t N>
+inline constexpr at_fn<N> at{};
+
+template <typename T> struct is_at : std::false_type {};
+template <std::size_t N, typename O, typename ...Os>
+struct is_at<at_pack<N, O, Os...>> : std::true_type {};
+template <typename T> inline constexpr bool is_at_v = is_at<T>::value;
+
+/// Phantom pack element for a key's comparison (DCF) channel. Not a leaf: it
+/// only records the cmp prefix depth in the key's type. `Depth` is the number
+/// of tree levels the comparison walks (0 is reserved for "no cmp").
+/// `OutBits` is the comparison output group width (bits of the β payload),
+/// so the value CWs / addend can be stored at group width instead of a full
+/// padded `uint64_t` per level.
+template <std::size_t Depth, std::size_t OutBits = 0, bool Wild = false>
+struct cmp_channel_tag
+{
+    static constexpr std::size_t depth = Depth;
+    static constexpr std::size_t out_bits = OutBits;
+    /// True when the comparison payload (β) is a wildcard to be assigned
+    /// after keygen. Concrete (non-wildcard) cmp keys keep `Wild == false`
+    /// so their layout / type name is unchanged.
+    static constexpr bool wild = Wild;
+};
+
+template <typename T> struct is_cmp_channel_tag : std::false_type {};
+template <std::size_t Depth, std::size_t OutBits, bool Wild>
+struct is_cmp_channel_tag<cmp_channel_tag<Depth, OutBits, Wild>>
+    : std::true_type {};
+template <typename T>
+inline constexpr bool is_cmp_channel_tag_v =
+    is_cmp_channel_tag<std::decay_t<T>>::value;
+
+namespace detail
+{
+namespace incr
+{
+
+// ---------------------------------------------------------------------------
+// A concrete placed output: an output type `OutputT` planted at prefix `N`.
+// ---------------------------------------------------------------------------
+
+template <std::size_t N, typename OutputT>
+struct placed
+{
+    static constexpr std::size_t prefix = N;
+    using output_type = OutputT;
+    OutputT value;
+    OutputT addend{};  // public if_false for eq(...); party 0 absorbs at eval
+};
+
+template <typename T> struct is_placed : std::false_type {};
+template <std::size_t N, typename O>
+struct is_placed<placed<N, O>> : std::true_type {};
+template <typename T>
+inline constexpr bool is_placed_v = is_placed<std::decay_t<T>>::value;
+
+template <typename NodeT, typename OutputT>
+inline constexpr std::size_t out_bits_v =
+    utils::bitlength_of_output_v<concrete_type_t<OutputT>, NodeT>;
+
+template <typename NodeT, typename OutputT>
+inline constexpr std::size_t lg_opl_v =
+    dpf::lg_outputs_per_leaf_v<concrete_type_t<OutputT>, NodeT>;
+
+template <typename NodeT, std::size_t N, typename OutputT>
+inline constexpr std::size_t level_of_v = N - lg_opl_v<NodeT, OutputT>;
+
+template <typename NodeT, std::size_t N, typename OutputT>
+inline constexpr bool prefix_ok_v = (N >= lg_opl_v<NodeT, OutputT>);
+
+// ---------------------------------------------------------------------------
+// Per-output slot metadata (packing / tree-level table).
+// ---------------------------------------------------------------------------
+
+struct slot_meta
+{
+    std::size_t prefix;
+    std::size_t tree_level;
+    std::size_t pos_base;
+    std::size_t group_id;
+    std::size_t index_in_group;
+    std::size_t out_bits;
+    std::size_t lg_opl;
+    std::size_t block_len;
+};
+
+template <typename NodeT, typename PlacedTuple, std::size_t ...Is>
+constexpr std::size_t max_tree_level_impl(std::index_sequence<Is...>)
+{
+    std::size_t m = 0;
+    ((m = std::max(m, level_of_v<NodeT,
+        std::tuple_element_t<Is, PlacedTuple>::prefix,
+        typename std::tuple_element_t<Is, PlacedTuple>::output_type>)), ...);
+    return m;
+}
+
+template <typename NodeT, typename PlacedTuple>
+inline constexpr std::size_t max_tree_level_v =
+    max_tree_level_impl<NodeT, PlacedTuple>(
+        std::make_index_sequence<std::tuple_size_v<PlacedTuple>>{});
+
+template <typename NodeT, typename PlacedTuple, std::size_t ...Is>
+constexpr bool all_prefixes_ok_impl(std::index_sequence<Is...>)
+{
+    return (prefix_ok_v<NodeT,
+        std::tuple_element_t<Is, PlacedTuple>::prefix,
+        typename std::tuple_element_t<Is, PlacedTuple>::output_type> && ...);
+}
+
+template <typename NodeT, typename PlacedTuple>
+inline constexpr bool all_prefixes_ok_v =
+    all_prefixes_ok_impl<NodeT, PlacedTuple>(
+        std::make_index_sequence<std::tuple_size_v<PlacedTuple>>{});
+
+template <typename NodeT, typename PlacedTuple, std::size_t I>
+constexpr void fill_slot_basics(
+    std::array<slot_meta, std::tuple_size_v<PlacedTuple>> & meta)
+{
+    using P = std::tuple_element_t<I, PlacedTuple>;
+    using O = typename P::output_type;
+    meta[I].prefix = P::prefix;
+    meta[I].lg_opl = lg_opl_v<NodeT, O>;
+    meta[I].out_bits = out_bits_v<NodeT, O>;
+    meta[I].tree_level = P::prefix - meta[I].lg_opl;
+    meta[I].block_len =
+        dpf::block_length_of_leaf_v<concrete_type_t<O>, NodeT>;
+    meta[I].pos_base = 0;
+    meta[I].group_id = 0;
+    meta[I].index_in_group = 0;
+}
+
+template <typename NodeT, typename PlacedTuple, std::size_t ...Is>
+constexpr void fill_all_basics(
+    std::array<slot_meta, std::tuple_size_v<PlacedTuple>> & meta,
+    std::index_sequence<Is...>)
+{
+    (fill_slot_basics<NodeT, PlacedTuple, Is>(meta), ...);
+}
+
+template <typename NodeT, typename PlacedTuple>
+constexpr auto build_meta()
+{
+    constexpr std::size_t n = std::tuple_size_v<PlacedTuple>;
+    std::array<slot_meta, n> meta{};
+    fill_all_basics<NodeT, PlacedTuple>(meta, std::make_index_sequence<n>{});
+
+    std::size_t next_gid = 0;
+    for (std::size_t i = 0; i < n; ++i)
+    {
+        std::size_t gid = static_cast<std::size_t>(-1);
+        for (std::size_t j = 0; j < i; ++j)
+        {
+            if (meta[j].prefix == meta[i].prefix
+                && meta[j].out_bits == meta[i].out_bits)
+            {
+                gid = meta[j].group_id;
+                break;
+            }
+        }
+        if (gid == static_cast<std::size_t>(-1))
+            gid = next_gid++;
+        meta[i].group_id = gid;
+    }
+
+    const std::size_t ngroups = next_gid;
+    std::array<std::size_t, n> group_count{};
+    for (std::size_t i = 0; i < n; ++i)
+        group_count[i] = 0;
+    for (std::size_t i = 0; i < n; ++i)
+        meta[i].index_in_group = group_count[meta[i].group_id]++;
+
+    std::array<std::size_t, n> g_level{};
+    std::array<std::size_t, n> g_blocks{};
+    std::array<std::size_t, n> g_first{};
+    for (std::size_t g = 0; g < ngroups; ++g)
+    {
+        g_blocks[g] = 0;
+        g_first[g] = n;
+        g_level[g] = 0;
+    }
+    for (std::size_t i = 0; i < n; ++i)
+    {
+        const auto g = meta[i].group_id;
+        g_level[g] = meta[i].tree_level;
+        g_blocks[g] += meta[i].block_len;
+        if (i < g_first[g])
+            g_first[g] = i;
+    }
+
+    constexpr std::size_t depth = max_tree_level_v<NodeT, PlacedTuple>;
+    std::array<bool, n> g_done{};
+    for (std::size_t g = 0; g < ngroups; ++g)
+        g_done[g] = false;
+
+    for (std::size_t level = 0; level <= depth; ++level)
+    {
+        std::size_t cursor = (level == depth) ? 0 : 2;
+        for (;;)
+        {
+            std::size_t best_g = n;
+            std::size_t best_first = n;
+            for (std::size_t g = 0; g < ngroups; ++g)
+            {
+                if (g_done[g] || g_level[g] != level)
+                    continue;
+                if (g_first[g] < best_first)
+                {
+                    best_first = g_first[g];
+                    best_g = g;
+                }
+            }
+            if (best_g == n)
+                break;
+            for (std::size_t i = 0; i < n; ++i)
+            {
+                if (meta[i].group_id == best_g)
+                    meta[i].pos_base = cursor;
+            }
+            cursor += g_blocks[best_g];
+            g_done[best_g] = true;
+        }
+    }
+    return meta;
+}
+
+template <typename MetaArray, std::size_t NGroups>
+constexpr auto build_group_order(const MetaArray & meta, std::size_t n)
+{
+    std::array<std::size_t, NGroups == 0 ? 1 : NGroups> order{};
+    std::array<bool, NGroups == 0 ? 1 : NGroups> used{};
+    for (std::size_t g = 0; g < NGroups; ++g)
+        used[g] = false;
+    for (std::size_t k = 0; k < NGroups; ++k)
+    {
+        std::size_t best = NGroups;
+        std::size_t best_lvl = static_cast<std::size_t>(-1);
+        std::size_t best_first = n;
+        for (std::size_t g = 0; g < NGroups; ++g)
+        {
+            if (used[g])
+                continue;
+            std::size_t lvl = 0, first = n;
+            for (std::size_t i = 0; i < n; ++i)
+            {
+                if (meta[i].group_id == g)
+                {
+                    lvl = meta[i].tree_level;
+                    if (i < first)
+                        first = i;
+                }
+            }
+            if (lvl < best_lvl || (lvl == best_lvl && first < best_first))
+            {
+                best_lvl = lvl;
+                best_first = first;
+                best = g;
+            }
+        }
+        order[k] = best;
+        used[best] = true;
+    }
+    return order;
+}
+
+template <typename InputT>
+constexpr InputT lane_input(InputT x, std::size_t prefix, std::size_t bitlen)
+{
+    if (prefix >= bitlen)
+        return x;
+    // Shift on the integral representation so this works for `modint`,
+    // `keyword` (whose `>>` yields a parent `modint`, not the keyword), and
+    // signed/bitstring inputs. Reconstruct the input type from the shifted
+    // integral value via `make_from_integral_value` (a friend of `keyword`).
+    constexpr auto to_int = utils::to_integral_type<InputT>{};
+    using FromI = typename utils::make_from_integral_value<InputT>::integral_type;
+    const auto shifted = static_cast<FromI>(to_int(x) >> (bitlen - prefix));
+    return utils::make_from_integral_value<InputT>{}(shifted);
+}
+
+// Concatenate index sequences
+template <typename... Seqs> struct cat_seq;
+template <> struct cat_seq<> { using type = std::index_sequence<>; };
+template <std::size_t... Is>
+struct cat_seq<std::index_sequence<Is...>>
+{
+    using type = std::index_sequence<Is...>;
+};
+template <std::size_t... Is, std::size_t... Js, typename... Rest>
+struct cat_seq<std::index_sequence<Is...>, std::index_sequence<Js...>, Rest...>
+{
+    using type = typename cat_seq<std::index_sequence<Is..., Js...>, Rest...>::type;
+};
+template <typename... Seqs>
+using cat_seq_t = typename cat_seq<Seqs...>::type;
+
+template <std::size_t G, typename MetaHolder, std::size_t I>
+using keep_if_group = std::conditional_t<
+    MetaHolder::value[I].group_id == G,
+    std::index_sequence<I>,
+    std::index_sequence<>>;
+
+template <std::size_t G, typename MetaHolder, typename Seq>
+struct filter_group;
+template <std::size_t G, typename MetaHolder, std::size_t... Is>
+struct filter_group<G, MetaHolder, std::index_sequence<Is...>>
+{
+    using type = cat_seq_t<keep_if_group<G, MetaHolder, Is>...>;
+};
+template <std::size_t G, typename MetaHolder, std::size_t N>
+using filter_group_t = typename filter_group<G, MetaHolder,
+    std::make_index_sequence<N>>::type;
+
+template <typename KeyT>
+struct meta_holder
+{
+    static constexpr auto value = KeyT::meta;
+};
+
+// ---------------------------------------------------------------------------
+// Normalize a `dpf_key` output pack into (PlacedTuple, CmpDepth).
+//
+// Each pack element is one of:
+//   - a bare output `T`            -> placed<BitLen, T>
+//   - a `placed<N, T>`             -> placed<N, T>  (from `at<N>`)
+//   - a `cmp_channel_tag<Depth>`   -> not a leaf; contributes Depth to CmpDepth
+// ---------------------------------------------------------------------------
+
+template <std::size_t BitLen, typename Elem>
+struct normalize_one
+{
+    using placed_tuple = std::tuple<placed<BitLen, Elem>>;
+    static constexpr std::size_t cmp_depth = 0;
+    static constexpr std::size_t cmp_out_bits = 0;
+    static constexpr bool cmp_wild = false;
+};
+template <std::size_t BitLen, std::size_t N, typename T>
+struct normalize_one<BitLen, placed<N, T>>
+{
+    using placed_tuple = std::tuple<placed<N, T>>;
+    static constexpr std::size_t cmp_depth = 0;
+    static constexpr std::size_t cmp_out_bits = 0;
+    static constexpr bool cmp_wild = false;
+};
+template <std::size_t BitLen, std::size_t Depth, std::size_t OutBits, bool Wild>
+struct normalize_one<BitLen, cmp_channel_tag<Depth, OutBits, Wild>>
+{
+    using placed_tuple = std::tuple<>;
+    static constexpr std::size_t cmp_depth = Depth;
+    static constexpr std::size_t cmp_out_bits = OutBits;
+    static constexpr bool cmp_wild = Wild;
+};
+
+template <std::size_t BitLen, typename ...Elems>
+struct normalize_pack
+{
+    using placed_tuple =
+        decltype(std::tuple_cat(
+            std::declval<typename normalize_one<BitLen, Elems>::placed_tuple>()...));
+    static constexpr std::size_t cmp_depth =
+        (std::size_t{0} + ... + normalize_one<BitLen, Elems>::cmp_depth);
+    // At most one comparison channel per key, so the sum is that channel's
+    // output width (0 when there is no cmp channel).
+    static constexpr std::size_t cmp_out_bits =
+        (std::size_t{0} + ... + normalize_one<BitLen, Elems>::cmp_out_bits);
+    // At most one comparison channel per key, so the OR is that channel's
+    // wildcard flag (false when there is no cmp channel).
+    static constexpr bool cmp_wild =
+        (false || ... || normalize_one<BitLen, Elems>::cmp_wild);
+};
+
+template <std::size_t BitLen>
+struct normalize_pack<BitLen>
+{
+    using placed_tuple = std::tuple<>;
+    static constexpr std::size_t cmp_depth = 0;
+    static constexpr std::size_t cmp_out_bits = 0;
+    static constexpr bool cmp_wild = false;
+};
+
+/// True iff the pack is "classic-shaped": every element is a bare output (no
+/// `placed<>` from `at<>` and no `cmp_channel_tag<>`).
+template <typename ...Elems>
+inline constexpr bool is_classic_pack_v =
+    !((is_placed_v<Elems> || is_cmp_channel_tag_v<Elems>) || ...);
+
+}  // namespace incr
+}  // namespace detail
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_PLACEMENT_HPP__

include/dpf/prg.hpp

@@ -0,0 +1,168 @@
+/// @file dpf/prg.hpp
+/// @brief
+/// @details
+/// @author Ryan Henry <ryan.henry@ucalgary.ca>
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_PRG_HPP__
+#define LIBDPF_INCLUDE_DPF_PRG_HPP__
+
+#include "hedley/hedley.h"
+
+#include <atomic>
+#include <cstring>
+#include <type_traits>
+
+#include "dpf/prg_aes.hpp"
+#include "dpf/prg_dummy.hpp"
+#include "dpf/prg_lowmc.hpp"
+#include "dpf/secret_share.hpp"
+
+namespace dpf
+{
+
+namespace prg
+{
+
+namespace detail
+{
+
+/// Fill `T` from consecutive PRG blocks starting at `pos` (low bytes first).
+template <typename PRG, typename T>
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+T expand_raw(typename PRG::block_type seed, psnip_uint32_t pos) noexcept
+{
+    static_assert(std::is_trivially_copyable_v<T>,
+        "prg expand requires a trivially copyable value type");
+    constexpr std::size_t nbytes = sizeof(T);
+    constexpr std::size_t nblocks =
+        (nbytes + sizeof(typename PRG::block_type) - 1)
+            / sizeof(typename PRG::block_type);
+    alignas(typename PRG::block_type) typename PRG::block_type
+        blocks[nblocks ? nblocks : 1];
+    PRG::eval(seed, blocks, static_cast<psnip_uint32_t>(nblocks ? nblocks : 1),
+        pos);
+    T value{};
+    std::memcpy(&value, blocks, nbytes);
+    return value;
+}
+
+template <typename PRG, typename T, std::size_t Party>
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+auto expand_as_share(typename PRG::block_type seed,
+    psnip_uint32_t pos) noexcept
+{
+    return subtractive_share<T, Party>::from_raw(expand_raw<PRG, T>(seed, pos));
+}
+
+}  // namespace detail
+
+template <typename AesKey>
+template <typename T, std::size_t Party>
+auto aes<AesKey>::expand(block_type seed, psnip_uint32_t pos) noexcept
+{
+    return detail::expand_as_share<aes<AesKey>, T, Party>(seed, pos);
+}
+
+template <typename T, std::size_t Party>
+auto dummy::expand(block_type seed, psnip_uint32_t pos) noexcept
+{
+    return detail::expand_as_share<dummy, T, Party>(seed, pos);
+}
+
+template <typename T, std::size_t Party>
+auto lowmc128::expand(block_type seed, psnip_uint32_t pos) noexcept
+{
+    return detail::expand_as_share<lowmc128, T, Party>(seed, pos);
+}
+
+template <typename PRG>
+struct counter_wrapper final
+{
+    using block_type = typename PRG::block_type;
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    static block_type eval(block_type seed, psnip_uint32_t pos) noexcept
+    {
+        count_.fetch_add(1, std::memory_order::memory_order_relaxed);
+        return PRG::eval(seed, pos);
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    static auto eval01(block_type seed) noexcept
+    {
+        count_.fetch_add(2, std::memory_order::memory_order_relaxed);
+        return PRG::eval01(seed);
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    static void eval(block_type seed, block_type * HEDLEY_RESTRICT output,
+        psnip_uint32_t count, psnip_uint32_t pos = 0) noexcept
+    {
+        count_.fetch_add(count, std::memory_order::memory_order_relaxed);
+        PRG::eval(seed, output, count, pos);
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    static void eval01_x4(const block_type * HEDLEY_RESTRICT seeds,
+        block_type * HEDLEY_RESTRICT left,
+        block_type * HEDLEY_RESTRICT right) noexcept
+    {
+        count_.fetch_add(8, std::memory_order::memory_order_relaxed);
+        PRG::eval01_x4(seeds, left, right);
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    static void eval_x4(const block_type * HEDLEY_RESTRICT seeds,
+        block_type * HEDLEY_RESTRICT output, psnip_uint32_t pos = 0) noexcept
+    {
+        count_.fetch_add(4, std::memory_order::memory_order_relaxed);
+        PRG::eval_x4(seeds, output, pos);
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    static void eval_x8(const block_type * HEDLEY_RESTRICT seeds,
+        block_type * HEDLEY_RESTRICT output, psnip_uint32_t pos = 0) noexcept
+    {
+        count_.fetch_add(8, std::memory_order::memory_order_relaxed);
+        PRG::eval_x8(seeds, output, pos);
+    }
+
+    template <typename T, std::size_t Party>
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    static auto expand(block_type seed, psnip_uint32_t pos = 0) noexcept
+    {
+        count_.fetch_add(
+            static_cast<std::size_t>(
+                (sizeof(T) + sizeof(block_type) - 1) / sizeof(block_type)),
+            std::memory_order::memory_order_relaxed);
+        return detail::expand_as_share<PRG, T, Party>(seed, pos);
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    static std::size_t count() noexcept
+    {
+        return count_;
+    }
+
+  private:
+    inline static std::atomic_size_t count_{0};
+};  // struct counter_wrapper
+
+}  // namespace prg
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_PRG_HPP__

include/dpf/prg_aes.hpp

@@ -0,0 +1,498 @@
+/// @file dpf/prg_aes.hpp
+/// @brief
+/// @details
+/// @author Ryan Henry <ryan.henry@ucalgary.ca>
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_PRG_AES_HPP__
+#define LIBDPF_INCLUDE_DPF_PRG_AES_HPP__
+
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+#include <array>
+
+#include "hedley/hedley.h"
+#include "simde/simde/x86/avx2.h"
+#include "portable-snippets/exact-int/exact-int.h"
+
+#include "dpf/utils.hpp"
+
+namespace dpf
+{
+
+namespace prg
+{
+
+#ifdef __ARM_NEON
+
+#else
+#define simde_mm_aesenc_si128(a, RoundKey) _mm_aesenc_si128(a, RoundKey)
+#define simde_mm_aesenclast_si128(a, RoundKey) _mm_aesenclast_si128(a, RoundKey)
+#define simde_mm_aeskeygenassist_si128(a, inn8) _mm_aeskeygenassist_si128(a, inn8)
+#endif
+
+#if defined(__VAES__) && defined(__AVX2__)
+#define DPF_PRG_AES_HAS_VAES 1
+#include <immintrin.h>
+#endif
+
+template <typename AesKey>
+struct aes final
+{
+    using block_type = simde__m128i;
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    static void require_block_aligned(const void * p) noexcept
+    {
+        assert(p == nullptr
+            || reinterpret_cast<std::uintptr_t>(p) % alignof(block_type) == 0);
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_PURE
+    static block_type eval(block_type seed, psnip_uint32_t pos) noexcept
+    {
+        block_type rd_key0 = simde_mm_xor_si128(key.rd_key[0],
+            simde_mm_set_epi64x(0, pos));
+        block_type output = simde_mm_xor_si128(seed, rd_key0);
+
+        for (std::size_t j = 1; j < key.rounds; ++j)
+        {
+            output = simde_mm_aesenc_si128(output, key.rd_key[j]);
+        }
+        output = simde_mm_aesenclast_si128(output, key.rd_key[key.rounds]);
+        output = simde_mm_xor_si128(output, seed);
+
+        return output;
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_PURE
+    static auto eval01(block_type seed) noexcept
+    {
+        block_type rd_key00 = key.rd_key[0];
+        block_type rd_key01 = simde_mm_xor_si128(rd_key00,
+            simde_mm_set_epi64x(0, 1));
+
+        block_type output0 = simde_mm_xor_si128(seed, rd_key00);
+        block_type output1 = simde_mm_xor_si128(seed, rd_key01);
+
+HEDLEY_PRAGMA(GCC unroll(14))
+        for (std::size_t j = 1; j < key.rounds; ++j)
+        {
+            output0 = simde_mm_aesenc_si128(output0, key.rd_key[j]);
+            output1 = simde_mm_aesenc_si128(output1, key.rd_key[j]);
+        }
+        output0 = simde_mm_aesenclast_si128(output0, key.rd_key[key.rounds]);
+        output1 = simde_mm_aesenclast_si128(output1, key.rd_key[key.rounds]);
+        output0 = simde_mm_xor_si128(output0, seed);
+        output1 = simde_mm_xor_si128(output1, seed);
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+        return std::array<block_type, 2>{output0, output1};
+HEDLEY_PRAGMA(GCC diagnostic pop)
+    }
+
+    /// Round-major multi-block MMO. Positions use the same lane as
+    /// `eval` / `eval01` (`set_epi64x(0, pos)`). The first AddRoundKey
+    /// includes `rd_key[0]` so this matches the one-block `eval` for any
+    /// key, not only the all-zero key this PRG currently installs.
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    static void eval(block_type seed, block_type * HEDLEY_RESTRICT output,
+        psnip_uint32_t count, psnip_uint32_t pos = 0) noexcept
+    {
+        if (HEDLEY_UNLIKELY(count == 0))
+        {
+            return;
+        }
+
+        require_block_aligned(output);
+        block_type * HEDLEY_RESTRICT out =
+            static_cast<block_type *>(__builtin_assume_aligned(output,
+            alignof(block_type)));
+
+        if (HEDLEY_LIKELY(count == 1))
+        {
+            out[0] = eval(seed, pos);
+            return;
+        }
+        if (count == 2 && pos == 0)
+        {
+            auto kids = eval01(seed);
+            out[0] = kids[0];
+            out[1] = kids[1];
+            return;
+        }
+
+        auto whitened = simde_mm_xor_si128(seed, key.rd_key[0]);
+        DPF_UNROLL_LOOP
+        for (psnip_uint32_t i = 0; i < count; ++i)
+        {
+            out[i] = simde_mm_xor_si128(whitened,
+                simde_mm_set_epi64x(0, pos + i));
+        }
+
+        DPF_UNROLL_LOOP
+        for (std::size_t j = 1; j < key.rounds; ++j)
+        {
+            DPF_UNROLL_LOOP
+            for (psnip_uint32_t i = 0; i < count; ++i)
+            {
+                out[i] = simde_mm_aesenc_si128(out[i], key.rd_key[j]);
+            }
+        }
+        DPF_UNROLL_LOOP
+        for (psnip_uint32_t i = 0; i < count; ++i)
+        {
+            out[i] = simde_mm_aesenclast_si128(out[i],
+                key.rd_key[key.rounds]);
+            out[i] = simde_mm_xor_si128(out[i], seed);
+        }
+    }
+
+    /// Four independent `eval01` calls as one 8-block round-major AES.
+    /// `left[i] == eval(seeds[i], 0)`, `right[i] == eval(seeds[i], 1)`.
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    static void eval01_x4(const block_type * HEDLEY_RESTRICT seeds,
+        block_type * HEDLEY_RESTRICT left,
+        block_type * HEDLEY_RESTRICT right) noexcept
+    {
+        require_block_aligned(seeds);
+        require_block_aligned(left);
+        require_block_aligned(right);
+        const block_type * HEDLEY_RESTRICT s =
+            static_cast<const block_type *>(__builtin_assume_aligned(seeds,
+            alignof(block_type)));
+        block_type rk0 = key.rd_key[0];
+        block_type rk1 = simde_mm_xor_si128(rk0, simde_mm_set_epi64x(0, 1));
+        block_type blk[8], feed[8];
+        DPF_UNROLL_LOOP
+        for (std::size_t i = 0; i < 4; ++i)
+        {
+            feed[2*i] = feed[2*i + 1] = s[i];
+            blk[2*i]     = simde_mm_xor_si128(s[i], rk0);
+            blk[2*i + 1] = simde_mm_xor_si128(s[i], rk1);
+        }
+        aes_mmo_rounds_x8(blk, feed);
+        DPF_UNROLL_LOOP
+        for (std::size_t i = 0; i < 4; ++i)
+        {
+            left[i] = blk[2*i];
+            right[i] = blk[2*i + 1];
+        }
+    }
+
+    /// Four independent `eval(seed, pos)` as one 4-block round-major AES.
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    static void eval_x4(const block_type * HEDLEY_RESTRICT seeds,
+        block_type * HEDLEY_RESTRICT output, psnip_uint32_t pos) noexcept
+    {
+        require_block_aligned(seeds);
+        require_block_aligned(output);
+        const block_type * HEDLEY_RESTRICT s =
+            static_cast<const block_type *>(__builtin_assume_aligned(seeds,
+            alignof(block_type)));
+        block_type * HEDLEY_RESTRICT out =
+            static_cast<block_type *>(__builtin_assume_aligned(output,
+            alignof(block_type)));
+        block_type rk0 = simde_mm_xor_si128(key.rd_key[0],
+            simde_mm_set_epi64x(0, pos));
+        block_type blk[4];
+        DPF_UNROLL_LOOP
+        for (std::size_t i = 0; i < 4; ++i)
+        {
+            blk[i] = simde_mm_xor_si128(s[i], rk0);
+        }
+        aes_mmo_rounds_x4(blk, s);
+        DPF_UNROLL_LOOP
+        for (std::size_t i = 0; i < 4; ++i)
+        {
+            out[i] = blk[i];
+        }
+    }
+
+    /// Eight independent `eval(seed, pos)` as one 8-block round-major AES.
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    static void eval_x8(const block_type * HEDLEY_RESTRICT seeds,
+        block_type * HEDLEY_RESTRICT output, psnip_uint32_t pos) noexcept
+    {
+        require_block_aligned(seeds);
+        require_block_aligned(output);
+        const block_type * HEDLEY_RESTRICT s =
+            static_cast<const block_type *>(__builtin_assume_aligned(seeds,
+            alignof(block_type)));
+        block_type * HEDLEY_RESTRICT out =
+            static_cast<block_type *>(__builtin_assume_aligned(output,
+            alignof(block_type)));
+        block_type rk0 = simde_mm_xor_si128(key.rd_key[0],
+            simde_mm_set_epi64x(0, pos));
+        block_type blk[8];
+        DPF_UNROLL_LOOP
+        for (std::size_t i = 0; i < 8; ++i)
+        {
+            blk[i] = simde_mm_xor_si128(s[i], rk0);
+        }
+        aes_mmo_rounds_x8(blk, s);
+        DPF_UNROLL_LOOP
+        for (std::size_t i = 0; i < 8; ++i)
+        {
+            out[i] = blk[i];
+        }
+    }
+
+    /// Raw-bit subtractive share of `T` for party `Party` (see `prg.hpp`).
+    template <typename T, std::size_t Party>
+    static auto expand(block_type seed, psnip_uint32_t pos = 0) noexcept;
+
+  private:
+    static const AesKey key;
+
+    /// `blk[i]` is already `seed[i] XOR rd_key[0] XOR pos_i`. Runs AES
+    /// rounds 1..last and the MMO feed-forward `XOR seed[i]`.
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    static void aes_mmo_rounds_x4(block_type * HEDLEY_RESTRICT blk,
+        const block_type * HEDLEY_RESTRICT seed) noexcept
+    {
+        block_type b0 = blk[0], b1 = blk[1], b2 = blk[2], b3 = blk[3];
+HEDLEY_PRAGMA(GCC unroll(14))
+        for (std::size_t j = 1; j < key.rounds; ++j)
+        {
+            const block_type rk = key.rd_key[j];
+            b0 = simde_mm_aesenc_si128(b0, rk);
+            b1 = simde_mm_aesenc_si128(b1, rk);
+            b2 = simde_mm_aesenc_si128(b2, rk);
+            b3 = simde_mm_aesenc_si128(b3, rk);
+        }
+        const block_type last = key.rd_key[key.rounds];
+        blk[0] = simde_mm_xor_si128(simde_mm_aesenclast_si128(b0, last), seed[0]);
+        blk[1] = simde_mm_xor_si128(simde_mm_aesenclast_si128(b1, last), seed[1]);
+        blk[2] = simde_mm_xor_si128(simde_mm_aesenclast_si128(b2, last), seed[2]);
+        blk[3] = simde_mm_xor_si128(simde_mm_aesenclast_si128(b3, last), seed[3]);
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    static void aes_mmo_rounds_x8(block_type * HEDLEY_RESTRICT blk,
+        const block_type * HEDLEY_RESTRICT seed) noexcept
+    {
+#if defined(DPF_PRG_AES_HAS_VAES) && defined(__AVX512F__)
+        // 4 blocks per VAES instruction. Same round keys as the 128-bit path.
+        __m512i v0 = _mm512_castsi128_si512(blk[0]);
+        v0 = _mm512_inserti32x4(v0, blk[1], 1);
+        v0 = _mm512_inserti32x4(v0, blk[2], 2);
+        v0 = _mm512_inserti32x4(v0, blk[3], 3);
+        __m512i v1 = _mm512_castsi128_si512(blk[4]);
+        v1 = _mm512_inserti32x4(v1, blk[5], 1);
+        v1 = _mm512_inserti32x4(v1, blk[6], 2);
+        v1 = _mm512_inserti32x4(v1, blk[7], 3);
+HEDLEY_PRAGMA(GCC unroll(14))
+        for (std::size_t j = 1; j < key.rounds; ++j)
+        {
+            const __m512i rk = _mm512_broadcast_i32x4(key.rd_key[j]);
+            v0 = _mm512_aesenc_epi128(v0, rk);
+            v1 = _mm512_aesenc_epi128(v1, rk);
+        }
+        const __m512i last = _mm512_broadcast_i32x4(key.rd_key[key.rounds]);
+        v0 = _mm512_aesenclast_epi128(v0, last);
+        v1 = _mm512_aesenclast_epi128(v1, last);
+        blk[0] = simde_mm_xor_si128(_mm512_extracti32x4_epi32(v0, 0), seed[0]);
+        blk[1] = simde_mm_xor_si128(_mm512_extracti32x4_epi32(v0, 1), seed[1]);
+        blk[2] = simde_mm_xor_si128(_mm512_extracti32x4_epi32(v0, 2), seed[2]);
+        blk[3] = simde_mm_xor_si128(_mm512_extracti32x4_epi32(v0, 3), seed[3]);
+        blk[4] = simde_mm_xor_si128(_mm512_extracti32x4_epi32(v1, 0), seed[4]);
+        blk[5] = simde_mm_xor_si128(_mm512_extracti32x4_epi32(v1, 1), seed[5]);
+        blk[6] = simde_mm_xor_si128(_mm512_extracti32x4_epi32(v1, 2), seed[6]);
+        blk[7] = simde_mm_xor_si128(_mm512_extracti32x4_epi32(v1, 3), seed[7]);
+#elif defined(DPF_PRG_AES_HAS_VAES)
+        __m256i v0 = _mm256_insertf128_si256(_mm256_castsi128_si256(blk[0]), blk[1], 1);
+        __m256i v1 = _mm256_insertf128_si256(_mm256_castsi128_si256(blk[2]), blk[3], 1);
+        __m256i v2 = _mm256_insertf128_si256(_mm256_castsi128_si256(blk[4]), blk[5], 1);
+        __m256i v3 = _mm256_insertf128_si256(_mm256_castsi128_si256(blk[6]), blk[7], 1);
+HEDLEY_PRAGMA(GCC unroll(14))
+        for (std::size_t j = 1; j < key.rounds; ++j)
+        {
+            const __m256i rk = _mm256_broadcastsi128_si256(key.rd_key[j]);
+            v0 = _mm256_aesenc_epi128(v0, rk);
+            v1 = _mm256_aesenc_epi128(v1, rk);
+            v2 = _mm256_aesenc_epi128(v2, rk);
+            v3 = _mm256_aesenc_epi128(v3, rk);
+        }
+        const __m256i last = _mm256_broadcastsi128_si256(key.rd_key[key.rounds]);
+        v0 = _mm256_aesenclast_epi128(v0, last);
+        v1 = _mm256_aesenclast_epi128(v1, last);
+        v2 = _mm256_aesenclast_epi128(v2, last);
+        v3 = _mm256_aesenclast_epi128(v3, last);
+        blk[0] = simde_mm_xor_si128(_mm256_castsi256_si128(v0), seed[0]);
+        blk[1] = simde_mm_xor_si128(_mm256_extracti128_si256(v0, 1), seed[1]);
+        blk[2] = simde_mm_xor_si128(_mm256_castsi256_si128(v1), seed[2]);
+        blk[3] = simde_mm_xor_si128(_mm256_extracti128_si256(v1, 1), seed[3]);
+        blk[4] = simde_mm_xor_si128(_mm256_castsi256_si128(v2), seed[4]);
+        blk[5] = simde_mm_xor_si128(_mm256_extracti128_si256(v2, 1), seed[5]);
+        blk[6] = simde_mm_xor_si128(_mm256_castsi256_si128(v3), seed[6]);
+        blk[7] = simde_mm_xor_si128(_mm256_extracti128_si256(v3, 1), seed[7]);
+#else
+        block_type b0 = blk[0], b1 = blk[1], b2 = blk[2], b3 = blk[3];
+        block_type b4 = blk[4], b5 = blk[5], b6 = blk[6], b7 = blk[7];
+HEDLEY_PRAGMA(GCC unroll(14))
+        for (std::size_t j = 1; j < key.rounds; ++j)
+        {
+            const block_type rk = key.rd_key[j];
+            b0 = simde_mm_aesenc_si128(b0, rk);
+            b1 = simde_mm_aesenc_si128(b1, rk);
+            b2 = simde_mm_aesenc_si128(b2, rk);
+            b3 = simde_mm_aesenc_si128(b3, rk);
+            b4 = simde_mm_aesenc_si128(b4, rk);
+            b5 = simde_mm_aesenc_si128(b5, rk);
+            b6 = simde_mm_aesenc_si128(b6, rk);
+            b7 = simde_mm_aesenc_si128(b7, rk);
+        }
+        const block_type last = key.rd_key[key.rounds];
+        blk[0] = simde_mm_xor_si128(simde_mm_aesenclast_si128(b0, last), seed[0]);
+        blk[1] = simde_mm_xor_si128(simde_mm_aesenclast_si128(b1, last), seed[1]);
+        blk[2] = simde_mm_xor_si128(simde_mm_aesenclast_si128(b2, last), seed[2]);
+        blk[3] = simde_mm_xor_si128(simde_mm_aesenclast_si128(b3, last), seed[3]);
+        blk[4] = simde_mm_xor_si128(simde_mm_aesenclast_si128(b4, last), seed[4]);
+        blk[5] = simde_mm_xor_si128(simde_mm_aesenclast_si128(b5, last), seed[5]);
+        blk[6] = simde_mm_xor_si128(simde_mm_aesenclast_si128(b6, last), seed[6]);
+        blk[7] = simde_mm_xor_si128(simde_mm_aesenclast_si128(b7, last), seed[7]);
+#endif
+    }
+};  // struct aes
+
+#define EXPAND_ASSIST(v1, v2, v3, v4, shuff_const, aes_const)  \
+  v2 = simde_mm_aeskeygenassist_si128(v4, aes_const);          \
+  v3 = simde_mm_castps_si128(_mm_shuffle_ps(                   \
+            simde_mm_castsi128_ps(v3),                         \
+            simde_mm_castsi128_ps(v1), 16));                   \
+  v1 = simde_mm_xor_si128(v1, v3);                             \
+  v3 = simde_mm_castps_si128(simde_mm_shuffle_ps(              \
+            simde_mm_castsi128_ps(v3),                         \
+            simde_mm_castsi128_ps(v1), 140));                  \
+  v1 = simde_mm_xor_si128(v1, v3);                             \
+  v2 = simde_mm_shuffle_epi32(v2, shuff_const);                \
+  v1 = simde_mm_xor_si128(v1, v2)
+
+struct aes128_key
+{
+  public:
+    static constexpr std::size_t rounds = 10;
+    HEDLEY_PRAGMA(GCC diagnostic push)
+    HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    using rd_key_array = std::array<simde__m128i, rounds+1>;
+    HEDLEY_PRAGMA(GCC diagnostic pop)
+    const rd_key_array rd_key;
+
+    explicit aes128_key(const simde__m128i & userkey)
+      : rd_key{compute_round_keys(userkey)} { }
+
+  private:
+    rd_key_array compute_round_keys(const simde__m128i & userkey)
+    {
+        rd_key_array rd_key;
+        simde__m128i x0, x1, x2;
+        rd_key[0] = x0 = userkey;
+        x2 = simde_mm_setzero_si128();
+        EXPAND_ASSIST(x0, x1, x2, x0, 255, 1);
+        rd_key[1] = x0;
+        EXPAND_ASSIST(x0, x1, x2, x0, 255, 2);
+        rd_key[2] = x0;
+        EXPAND_ASSIST(x0, x1, x2, x0, 255, 4);
+        rd_key[3] = x0;
+        EXPAND_ASSIST(x0, x1, x2, x0, 255, 8);
+        rd_key[4] = x0;
+        EXPAND_ASSIST(x0, x1, x2, x0, 255, 16);
+        rd_key[5] = x0;
+        EXPAND_ASSIST(x0, x1, x2, x0, 255, 32);
+        rd_key[6] = x0;
+        EXPAND_ASSIST(x0, x1, x2, x0, 255, 64);
+        rd_key[7] = x0;
+        EXPAND_ASSIST(x0, x1, x2, x0, 255, 128);
+        rd_key[8] = x0;
+        EXPAND_ASSIST(x0, x1, x2, x0, 255, 27);
+        rd_key[9] = x0;
+        EXPAND_ASSIST(x0, x1, x2, x0, 255, 54);
+        rd_key[10] = x0;
+
+        return rd_key;
+    }
+};  // struct aes128_key
+
+struct aes256_key
+{
+  public:
+    static constexpr std::size_t rounds = 14;
+
+  HEDLEY_PRAGMA(GCC diagnostic push)
+  HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    using rd_key_array = std::array<simde__m128i, rounds+1>;
+  HEDLEY_PRAGMA(GCC diagnostic pop)
+    const rd_key_array rd_key;
+
+    explicit aes256_key(const simde__m256i & userkey)
+      : rd_key{compute_round_keys(userkey)} { }
+
+  private:
+    rd_key_array compute_round_keys(const simde__m256i & userkey)
+    {
+        rd_key_array rd_key;
+        simde__m128i x0, x1, x2, x3;
+        rd_key[0] = x0 = simde_mm256_extracti128_si256(userkey, 0);
+        rd_key[1] = x3 = simde_mm256_extracti128_si256(userkey, 1);
+        x2 = simde_mm_setzero_si128();
+
+        EXPAND_ASSIST(x0, x1, x2, x3, 255, 1);
+        rd_key[2] = x0;
+        EXPAND_ASSIST(x3, x1, x2, x0, 170, 1);
+        rd_key[3] = x3;
+        EXPAND_ASSIST(x0, x1, x2, x3, 255, 2);
+        rd_key[4] = x0;
+        EXPAND_ASSIST(x3, x1, x2, x0, 170, 2);
+        rd_key[5] = x3;
+        EXPAND_ASSIST(x0, x1, x2, x3, 255, 4);
+        rd_key[6] = x0;
+        EXPAND_ASSIST(x3, x1, x2, x0, 170, 4);
+        rd_key[7] = x3;
+        EXPAND_ASSIST(x0, x1, x2, x3, 255, 8);
+        rd_key[8] = x0;
+        EXPAND_ASSIST(x3, x1, x2, x0, 170, 8);
+        rd_key[9] = x3;
+        EXPAND_ASSIST(x0, x1, x2, x3, 255, 16);
+        rd_key[10] = x0;
+        EXPAND_ASSIST(x3, x1, x2, x0, 170, 16);
+        rd_key[11] = x3;
+        EXPAND_ASSIST(x0, x1, x2, x3, 255, 32);
+        rd_key[12] = x0;
+        EXPAND_ASSIST(x3, x1, x2, x0, 170, 32);
+        rd_key[13] = x3;
+        EXPAND_ASSIST(x0, x1, x2, x3, 255, 64);
+        rd_key[14] = x0;
+
+        return rd_key;
+    }
+};  // struct aes256_key
+
+using aes128 = aes<aes128_key>;
+using aes256 = aes<aes256_key>;
+
+template <>
+const aes128_key aes128::key = aes128_key(simde__m128i{0, 0});
+
+template <>
+const aes256_key aes256::key = aes256_key(simde__m256i{0, 0, 0, 0});
+
+}  // namespace prg
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_PRG_AES_HPP__

include/dpf/prg_dummy.hpp

@@ -0,0 +1,95 @@
+/// @file dpf/prg_dummy.hpp
+/// @brief
+/// @details
+/// @author Ryan Henry <ryan.henry@ucalgary.ca>
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_PRG_DUMMY_HPP__
+#define LIBDPF_INCLUDE_DPF_PRG_DUMMY_HPP__
+
+#include <array>
+#include <algorithm>
+#include <cstddef>
+
+#include "hedley/hedley.h"
+#include "portable-snippets/exact-int/exact-int.h"
+
+#include "dpf/utils.hpp"
+
+namespace dpf
+{
+
+namespace prg
+{
+
+struct dummy final
+{
+    using block_type = simde__m128i;
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_CONST
+    static block_type eval(block_type seed, psnip_uint32_t) noexcept
+    {
+        return seed;
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_CONST
+    static auto eval01(block_type seed) noexcept
+    {
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+        return std::array<block_type, 2>{seed, seed};
+HEDLEY_PRAGMA(GCC diagnostic pop)
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    static void eval(block_type seed, block_type * HEDLEY_RESTRICT output,
+        psnip_uint32_t count_, psnip_uint32_t = 0) noexcept
+    {
+        std::fill_n(output, count_, seed);
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    static void eval01_x4(const block_type * HEDLEY_RESTRICT seeds,
+        block_type * HEDLEY_RESTRICT left,
+        block_type * HEDLEY_RESTRICT right) noexcept
+    {
+        for (std::size_t i = 0; i < 4; ++i)
+        {
+            left[i] = right[i] = seeds[i];
+        }
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    static void eval_x4(const block_type * HEDLEY_RESTRICT seeds,
+        block_type * HEDLEY_RESTRICT output, psnip_uint32_t = 0) noexcept
+    {
+        std::copy_n(seeds, 4, output);
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    static void eval_x8(const block_type * HEDLEY_RESTRICT seeds,
+        block_type * HEDLEY_RESTRICT output, psnip_uint32_t = 0) noexcept
+    {
+        std::copy_n(seeds, 8, output);
+    }
+
+    /// Raw-bit subtractive share of `T` for party `Party` (see `prg.hpp`).
+    template <typename T, std::size_t Party>
+    static auto expand(block_type seed, psnip_uint32_t pos = 0) noexcept;
+};  // struct dummy
+
+}  // namespace prg
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_PRG_DUMMY_HPP__

include/dpf/prg_lowmc.hpp

@@ -0,0 +1,141 @@
+/// @file dpf/prg_lowmc.hpp
+/// @brief Fixed-key LowMC PRG. Same Matyas–Meyer–Oseas stretch as `aes128`.
+/// @copyright Copyright (c) 2019-2026 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_PRG_LOWMC_HPP__
+#define LIBDPF_INCLUDE_DPF_PRG_LOWMC_HPP__
+
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+
+#include "hedley/hedley.h"
+#include "simde/simde/x86/avx2.h"
+#include "portable-snippets/exact-int/exact-int.h"
+
+#include "lowmc/LowMC.h"
+#include "lowmc/LowMC.cpp"
+
+namespace dpf
+{
+
+namespace prg
+{
+
+/// LowMCv3, 128-bit block and key, 10 S-boxes, 32 rounds, all-zero key.
+/// `eval(seed, pos)` is `E(seed ⊕ pos) ⊕ seed`, with `pos` in the low lane.
+struct lowmc128 final
+{
+    using block_type = simde__m128i;
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    static block_type eval(block_type seed, psnip_uint32_t pos) noexcept
+    {
+        block_type in = simde_mm_xor_si128(seed, simde_mm_set_epi64x(0, pos));
+        return simde_mm_xor_si128(permute(in), seed);
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    static auto eval01(block_type seed) noexcept
+    {
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+        return std::array<block_type, 2>{eval(seed, 0), eval(seed, 1)};
+HEDLEY_PRAGMA(GCC diagnostic pop)
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    static void eval(block_type seed, block_type * HEDLEY_RESTRICT output,
+        psnip_uint32_t count, psnip_uint32_t pos = 0) noexcept
+    {
+        for (psnip_uint32_t i = 0; i < count; ++i)
+        {
+            output[i] = eval(seed, pos + i);
+        }
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    static void eval01_x4(const block_type * HEDLEY_RESTRICT seeds,
+        block_type * HEDLEY_RESTRICT left,
+        block_type * HEDLEY_RESTRICT right) noexcept
+    {
+        for (std::size_t i = 0; i < 4; ++i)
+        {
+            auto kids = eval01(seeds[i]);
+            left[i] = kids[0];
+            right[i] = kids[1];
+        }
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    static void eval_x4(const block_type * HEDLEY_RESTRICT seeds,
+        block_type * HEDLEY_RESTRICT output, psnip_uint32_t pos = 0) noexcept
+    {
+        for (std::size_t i = 0; i < 4; ++i)
+        {
+            output[i] = eval(seeds[i], pos);
+        }
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    static void eval_x8(const block_type * HEDLEY_RESTRICT seeds,
+        block_type * HEDLEY_RESTRICT output, psnip_uint32_t pos = 0) noexcept
+    {
+        for (std::size_t i = 0; i < 8; ++i)
+        {
+            output[i] = eval(seeds[i], pos);
+        }
+    }
+
+    /// Raw-bit subtractive share of `T` for party `Party` (see `prg.hpp`).
+    template <typename T, std::size_t Party>
+    static auto expand(block_type seed, psnip_uint32_t pos = 0) noexcept;
+
+  private:
+    static lowmc::block to_block(block_type x) noexcept
+    {
+        std::uint64_t lane[2];
+        std::memcpy(lane, &x, sizeof(lane));
+        lowmc::block b;
+        for (unsigned i = 0; i < 64; ++i)
+        {
+            b[i] = (lane[0] >> i) & 1ull;
+            b[i + 64] = (lane[1] >> i) & 1ull;
+        }
+        return b;
+    }
+
+    static block_type from_block(const lowmc::block & b) noexcept
+    {
+        std::uint64_t lane[2] = {0, 0};
+        for (unsigned i = 0; i < 64; ++i)
+        {
+            lane[0] |= static_cast<std::uint64_t>(b[i]) << i;
+            lane[1] |= static_cast<std::uint64_t>(b[i + 64]) << i;
+        }
+        block_type x;
+        std::memcpy(&x, lane, sizeof(x));
+        return x;
+    }
+
+    static block_type permute(block_type x) noexcept
+    {
+        static lowmc::LowMC cipher;
+        return from_block(cipher.encrypt(to_block(x)));
+    }
+};  // struct lowmc128
+
+}  // namespace prg
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_PRG_LOWMC_HPP__

include/dpf/random.hpp

@@ -0,0 +1,233 @@
+/// @file dpf/random.hpp
+/// @brief
+/// @details
+/// @author Ryan Henry <ryan.henry@ucalgary.ca>
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_RANDOM_HPP__
+#define LIBDPF_INCLUDE_DPF_RANDOM_HPP__
+
+#include <bsd/stdlib.h>
+
+#include <cerrno>
+#include <cstddef>
+#include <cstdio>
+#include <cstring>
+#include <exception>
+#include <fcntl.h>
+#include <mutex>
+#include <type_traits>
+#include <unistd.h>
+#include <utility>
+
+#include "hedley/hedley.h"
+
+#include "dpf/secret_share.hpp"
+
+namespace dpf
+{
+
+namespace detail
+{
+
+/// When set, `uniform_fill` copies from this hook and does not read the
+/// system RNG. Used to feed the same beaver coins to dealer `make_dpf` and
+/// Doerner–Shelat gen. Null in normal use.
+inline thread_local void (*uniform_bytes_hook)(void *, std::size_t) = nullptr;
+
+template <typename T>
+HEDLEY_ALWAYS_INLINE
+bool fill_from_hook(T & buf) noexcept
+{
+    if (uniform_bytes_hook == nullptr)
+    {
+        return false;
+    }
+    uniform_bytes_hook(&buf, sizeof(buf));
+    return true;
+}
+
+/// `bool` and `enum : bool` (including `dpf::bit`) have only two valid
+/// representations. Filling them with a raw entropy byte is undefined.
+template <typename T>
+constexpr bool is_boolean_representation() noexcept
+{
+    using U = std::remove_cv_t<T>;
+    if constexpr (std::is_same_v<U, bool>)
+    {
+        return true;
+    }
+    else if constexpr (std::is_enum_v<U>)
+    {
+        return std::is_same_v<std::underlying_type_t<U>, bool>;
+    }
+    else
+    {
+        return false;
+    }
+}
+
+#if !defined(LIBDPF_USE_ARC4RANDOM)
+
+/// One unbuffered, exclusively locked read of the entropy device.
+/// Buffering would copy unread bytes into a `fork()` child, so parent and
+/// child would repeat the same key material. The lock keeps concurrent
+/// `fread` calls off the shared `FILE`.
+struct entropy_source
+{
+#if defined(LIBDPF_USE_DEV_RANDOM)
+    static constexpr const char * path = "/dev/random";
+    static constexpr const char * open_error = "dpf: cannot open /dev/random\n";
+#else
+    static constexpr const char * path = "/dev/urandom";
+    static constexpr const char * open_error = "dpf: cannot open /dev/urandom\n";
+#endif
+
+    FILE * fp = nullptr;
+    std::mutex mu;
+
+    entropy_source() = default;
+    entropy_source(const entropy_source &) = delete;
+    entropy_source & operator=(const entropy_source &) = delete;
+    entropy_source(entropy_source &&) = delete;
+    entropy_source & operator=(entropy_source &&) = delete;
+
+    ~entropy_source()
+    {
+        if (fp != nullptr)
+        {
+            std::fclose(fp);
+        }
+    }
+
+    void open_unlocked()
+    {
+        if (fp != nullptr)
+        {
+            return;
+        }
+        fp = std::fopen(path, "rb");
+        if (fp == nullptr)
+        {
+            std::fputs(open_error, stderr);
+            std::terminate();
+        }
+        // Before any read. A buffered FILE duplicates entropy across fork().
+        if (std::setvbuf(fp, nullptr, _IONBF, 0) != 0)
+        {
+            std::fclose(fp);
+            fp = nullptr;
+            std::fputs("dpf: cannot disable entropy buffering\n", stderr);
+            std::terminate();
+        }
+        int fd = ::fileno(fp);
+        if (fd >= 0)
+        {
+            ::fcntl(fd, F_SETFD, FD_CLOEXEC);
+        }
+    }
+
+    void read(void * dst, std::size_t n)
+    {
+        std::lock_guard<std::mutex> lock(mu);
+        if (fp == nullptr)
+        {
+            open_unlocked();
+        }
+        auto * p = static_cast<unsigned char *>(dst);
+        while (n > 0)
+        {
+            std::size_t got = std::fread(p, 1, n, fp);
+            if (got == 0)
+            {
+                if (std::ferror(fp) && errno == EINTR)
+                {
+                    std::clearerr(fp);
+                    continue;
+                }
+                std::fputs("dpf: entropy read failed\n", stderr);
+                std::terminate();
+            }
+            p += got;
+            n -= got;
+        }
+    }
+};
+
+inline entropy_source & entropy()
+{
+    static entropy_source source;
+    return source;
+}
+
+#endif  // !LIBDPF_USE_ARC4RANDOM
+
+}  // namespace detail
+
+template <typename T>
+HEDLEY_NO_THROW
+auto & uniform_fill(T & buf) noexcept  // NOLINT(runtime/references)
+{
+    static_assert(std::is_trivially_copyable_v<std::remove_cv_t<T>>,
+        "uniform_fill requires a trivially copyable type");
+
+    if constexpr (detail::is_boolean_representation<T>())
+    {
+        unsigned char raw = 0;
+        uniform_fill(raw);
+        buf = static_cast<T>(static_cast<bool>(raw & 1u));
+        return buf;
+    }
+    else
+    {
+        if (detail::fill_from_hook(buf)) return buf;
+#if defined(LIBDPF_USE_ARC4RANDOM)
+        arc4random_buf(&buf, sizeof(buf));
+#else
+        detail::entropy().read(&buf, sizeof(buf));
+#endif
+        return buf;
+    }
+}
+
+template <typename T>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+auto uniform_sample() noexcept
+{
+    using U = std::remove_cv_t<T>;
+    U buf;
+    uniform_fill(buf);
+    return buf;
+}
+
+template <typename T>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_NO_THROW
+auto additively_share(T secret) noexcept
+{
+    using T_ = std::remove_cv_t<std::remove_reference_t<T>>;
+    T_ tmp = uniform_sample<T_>();
+    // Signed subtraction overflows for extreme shares. Subtract in the
+    // unsigned width and copy the bits back so the group is mod 2^n.
+    T_ other;
+    if constexpr (std::is_integral_v<T_> && std::is_signed_v<T_>)
+    {
+        using U = std::make_unsigned_t<T_>;
+        U diff = static_cast<U>(static_cast<T_>(secret)) - static_cast<U>(tmp);
+        std::memcpy(&other, &diff, sizeof(other));
+    }
+    else
+    {
+        other = static_cast<T_>(static_cast<T_>(secret) - tmp);
+    }
+    return std::make_pair(
+        additive_share<T_, 0>::from_raw(tmp),
+        additive_share<T_, 1>::from_raw(other));
+}
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_RANDOM_HPP__

include/dpf/rotated_iterable.hpp

@@ -0,0 +1,240 @@
+// /// @file dpf/rotated_iterable.hpp
+// /// @author Ryan Henry <ryan.henry@ucalgary.ca>
+// /// @brief defines `dpf::rotated_iterable` and associated helpers
+// /// @details
+// /// @copyright Copyright (c) 2019-2023 Ryan Henry and others
+// /// @license Released under a GNU General Public v2.0 (GPLv2) license;
+// ///          see [LICENSE.md](@ref GPLv2) for details.
+
+// #ifndef LIBDPF_INCLUDE_DPF_ROTATED_VIEW_HPP__
+// #define LIBDPF_INCLUDE_DPF_ROTATED_VIEW_HPP__
+
+// namespace dpf
+// {
+
+// template <typename ContainerT>
+// struct rotated_iterable_iterator;        // forward declaration
+
+// template <typename ContainerT>
+// struct rotated_iterable_const_iterator;  // forward declaration
+
+// template <typename ContainerT>
+// struct rotated_iterable
+// {
+//     using container_type = ContainerT;
+
+//     using value_type = typename container_type::value_type;
+//     using size_type = typename container_type::size_type;
+//     using difference_type = typename container_type::difference_type;
+//     using reference = typename container_type::reference;
+//     using const_reference = typename container_type::const_reference;
+//     using pointer = typename container_type::pointer;
+//     using const_pointer = typename container_type::const_pointer;
+
+//     using iterator = rotated_iterable_iterator<container_type>;
+//     using const_iterator = rotated_iterable_const_iterator<container_type>;
+//     using wrapped_iterator = typename ContainerT::iterator;
+
+//     rotated_iterable(const ContainerT & container, difference_type distance)
+//       : container_{container},
+//         distance_{distance >= 0 ? distance % container_.size() : (distance % container_.size()) + container_.size()},
+//         wrap_to{std::begin(container)},
+//         wrap_after{std::next(std::end(container), -1)},
+//         end_after{std::next(wrap_to, distance-1)}
+//     {
+//         distance_ %= container_.size();
+//         if (distance_ < 0)
+//         {
+//             distance_ += container_.size();
+//         }
+//     }
+
+//     HEDLEY_ALWAYS_INLINE
+//     reference operator[](size_type index)
+//     {
+//         index += distance_;
+//         if (index > container_.size())
+//         {
+//             index -= container_.size();
+//         }
+//         return container_[index];
+//     }
+
+//     HEDLEY_ALWAYS_INLINE
+//     const_reference operator[](size_type index) const
+//     {
+//         index += distance_;
+//         if (index > container_.size())
+//         {
+//             index -= container_.size();
+//         }
+//         return container_[index];
+//     }
+
+//     HEDLEY_NO_THROW
+//     HEDLEY_ALWAYS_INLINE
+//     iterator begin() noexcept
+//     {
+//         return iterator{*this, std::next(end_after, 1)};
+//     }
+
+//     HEDLEY_NO_THROW
+//     HEDLEY_ALWAYS_INLINE
+//     const_iterator begin() const noexcept
+//     {
+//         return const_iterator{*this, std::next(end_after, 1)};
+//     }
+
+//     HEDLEY_NO_THROW
+//     HEDLEY_ALWAYS_INLINE
+//     const_iterator cbegin() const noexcept
+//     {
+//         return begin();
+//     }
+
+//     HEDLEY_NO_THROW
+//     HEDLEY_ALWAYS_INLINE
+//     iterator end() noexcept
+//     {
+//         return iterator{*this, std::next(wrap_after, 1)};
+//     }
+
+//     HEDLEY_NO_THROW
+//     HEDLEY_ALWAYS_INLINE
+//     const_iterator end() const noexcept
+//     {
+//         return const_iterator{*this, std::next(wrap_after, 1)};
+//     }
+
+//     HEDLEY_NO_THROW
+//     HEDLEY_ALWAYS_INLINE
+//     const_iterator cend() const noexcept
+//     {
+//         return end();
+//     }
+
+//     auto distance() const
+//     {
+//         return distance_;
+//     }
+
+//   private:
+//     container_type & container_;
+//     difference_type distance_;
+
+//     wrapped_iterator wrap_to;
+//     wrapped_iterator wrap_after;
+//     wrapped_iterator end_after;
+// };  // rotated_iterable
+
+// template <typename ContainerT>
+// struct rotated_iterator
+// {
+//     using wrapped_iterable_type = rotated_iterable<ContainerT>;
+//     using wrapped_iterator = typename wrapped_iterable_type::iterator;
+//     using size_type = typename wrapped_iterable_type::size_type;
+//     using reference = typename wrapped_iterable_type::reference;
+
+//     rotated_iterable<ContainerT> & v;
+//     wrapped_iterator it;
+//     rotated_iterator & operator++()
+//     {
+//         if (it == v.wrap_after)
+//         {
+//             it = v.wrap_to;
+//         }
+//         else if (it == v.end_after)
+//         {
+//             it = std::next(v.wrap_after, 1);
+//         }
+//         else
+//         {
+//             ++it;
+//         }
+        
+//         return *this;
+//     }
+//     rotated_iterator & operator--()
+//     {
+//         --it;
+//         if (it == v.wrap_after)
+//         {
+//             it = v.end_after;
+//         }
+//         else if (it == v.end_after)
+//         {
+//             it = std::next(v.wrap_to, -1);
+//         }
+
+//         return *this;
+//     }
+//     reference operator*() const { return *it; }
+//     bool operator!=(const rotated_iterator other) const
+//         { return &v != &other.v || it != other.it;  }
+// };
+
+// template <typename ContainerT>
+// struct rotated_const_iterator
+// {
+//     using wrapped_iterable_type = rotated_iterable<ContainerT>;
+//     using wrapped_iterator = typename wrapped_iterable_type::iterator;
+//     using size_type = typename wrapped_iterable_type::size_type;
+//     using const_reference = typename wrapped_iterable_type::const_reference;
+
+//     const rotated_iterable<ContainerT> & v;
+//     wrapped_iterator it;
+//     rotated_const_iterator & operator++()
+//     {
+//         if (it == v.wrap_after)
+//         {
+//             it = v.wrap_to;
+//         }
+//         else if (it == v.end_after)
+//         {
+//             it = std::next(v.wrap_after, 1);
+//         }
+//         else
+//         {
+//             ++it;
+//         }
+        
+//         return *this;
+//     }
+//     rotated_const_iterator & operator--()
+//     {
+//         --it;
+//         if (it == v.wrap_after)
+//         {
+//             it = v.end_after;
+//         }
+//         else if (it == v.end_after)
+//         {
+//             it = std::next(v.wrap_to, -1);
+//         }
+
+//         return *this;
+//     }
+//     const_reference operator*() const { return *it; }
+//     bool operator!=(const rotated_const_iterator other) const
+//         { return &v != &other.v || it != other.it;  }
+// };
+
+// template <typename ContainerT>
+// auto rotated_by(const ContainerT & container,
+//     typename ContainerT::size_type rotate_by)
+// {
+//     return rotated_iterable{container, rotate_by};
+// }
+
+// template <typename ContainerT,
+//           typename UnaryFunction>
+// auto for_each_rotated_by(const ContainerT & container,
+//     typename ContainerT::size_type rotate_by, UnaryFunction && f)
+// {
+//     for (auto i = rotate_by; i < container.size(); ++i) f(container[i]);
+//     for (auto i = 0;         i < rotate_by;        ++i) f(container[i]);
+// }
+
+// }  // namespace dpf
+
+// #endif  // LIBDPF_INCLUDE_DPF_ROTATED_VIEW_HPP__

include/dpf/rotation_iterable.hpp

@@ -0,0 +1,340 @@
+#ifndef LIBDPF_INCLUDE_DPF_ROTATED_VIEW_HPP__
+#define LIBDPF_INCLUDE_DPF_ROTATED_VIEW_HPP__
+
+#include <type_traits>
+#include <iterator>
+
+#include "hedley/hedley.h"
+
+namespace dpf
+{
+
+template <typename WrappedIterator> struct rotation_iterator;
+template <typename WrappedIterator> struct rotation_const_iterator;
+
+template <typename WrappedIterator>
+struct rotation_iterable
+{
+    using wrapped_iterator = WrappedIterator;
+    using iterator = rotation_iterator<wrapped_iterator>;
+    using const_iterator = rotation_const_iterator<wrapped_iterator>;
+
+    using reference = typename std::iterator_traits<wrapped_iterator>::reference;
+    using const_reference = std::conditional_t<
+        std::is_reference_v<reference>,
+        std::add_lvalue_reference_t<std::add_const_t<std::remove_reference_t<reference>>>,
+        std::add_const_t<reference>>;
+    using difference_type = typename std::iterator_traits<wrapped_iterator>::difference_type;
+
+    // O(1) if `WrappedIterator` is a random-access iterator
+    constexpr rotation_iterable(wrapped_iterator begin, wrapped_iterator end,
+        difference_type distance)
+      : size_{std::distance(begin, end)},
+        distance_{
+          [this, &distance]()
+            {
+                distance %= this->size_;
+                if (distance < 0)
+                {
+                    distance += this->size_;
+                }
+                return distance;
+            }()},
+        begin_{std::next(begin, static_cast<difference_type>(distance_))},
+        wrap_to_{begin},
+        wrap_after_{std::next(end, -difference_type(distance_ > 0))},
+        end_after_{std::next(wrap_to_, static_cast<difference_type>(distance_-1))},
+        end_{end}
+    { }
+
+    // UB if `begin` does not precede `middle` does not precede `end`
+    // O(1) if `WrappedIterator` is a random-access iterator
+    constexpr rotation_iterable(wrapped_iterator begin, wrapped_iterator end,
+        wrapped_iterator middle)
+      : size_{std::distance(begin, end)},
+        distance_{std::distance(begin, middle)},
+        begin_{middle},
+        wrap_to_{begin},
+        wrap_after_{std::next(end, difference_type(-1))},
+        end_after_{std::next(middle, difference_type(-1))},
+        end_{end}
+    { }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr difference_type distance() const noexcept
+    {
+        return distance_;
+    }
+
+    // UB if `index<0` or `index>size_`
+    // O(1) if `WrappedIterator` is a random-access iterator
+    HEDLEY_ALWAYS_INLINE
+    constexpr reference operator[](difference_type index)
+    {
+        index += distance_;
+        if (index >= size_)
+        {
+            index -= size_;
+        }
+        return *std::next(wrap_to_, static_cast<difference_type>(index));
+    }
+
+    // UB if `index<0` or `index>size_`
+    // O(1) if `WrappedIterator` is a random-access iterator
+    HEDLEY_ALWAYS_INLINE
+    constexpr const_reference operator[](difference_type index) const
+    {
+        index += distance_;
+        if (index >= size_)
+        {
+            index -= size_;
+        }
+        return *std::next(wrap_to_, static_cast<difference_type>(index));
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr iterator begin() noexcept
+    {
+        return iterator(*this, begin_);
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr const_iterator begin() const noexcept
+    {
+        return const_iterator(*this, begin_);
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr const_iterator cbegin() const noexcept
+    {
+        return begin();
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr iterator end() noexcept
+    {
+        return iterator(*this, end_);
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr const_iterator end() const noexcept
+    {
+        return const_iterator(*this, end_);
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr const_iterator cend() const noexcept
+    {
+        return end();
+    }
+
+  private:
+    difference_type size_;
+    difference_type distance_;
+    wrapped_iterator begin_;
+    wrapped_iterator wrap_to_;
+    wrapped_iterator wrap_after_;
+    wrapped_iterator end_after_;
+    wrapped_iterator end_;
+
+    template <typename Iter>
+    friend struct rotation_iterator_base;
+};  // rotation_iterable
+
+template <typename WrappedIterator>
+struct rotation_iterator_base
+{
+  public:
+    using wrapped_iterator = WrappedIterator;
+    using iterator_category = std::bidirectional_iterator_tag;
+    using difference_type = typename std::iterator_traits<WrappedIterator>::difference_type;
+    using value_type = typename std::iterator_traits<WrappedIterator>::value_type;
+    using reference = typename std::iterator_traits<WrappedIterator>::reference;
+    using const_reference = std::conditional_t<
+        std::is_reference_v<reference>,
+        std::add_lvalue_reference_t<std::add_const_t<std::remove_reference_t<reference>>>,
+        std::add_const_t<reference>>;
+    using pointer = typename std::iterator_traits<WrappedIterator>::pointer;
+
+    rotation_iterator_base(const rotation_iterable<wrapped_iterator> & iterable_,
+        wrapped_iterator iterator) noexcept
+      : iterable{iterable_}, it{iterator} { }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr rotation_iterator_base & operator++() noexcept
+    {
+        if (it == iterable.wrap_after_)
+        {
+            it = iterable.wrap_to_;
+        }
+        else if (it == iterable.end_after_)
+        {
+            it = iterable.end_;
+        }
+        else
+        {
+            ++it;
+        }
+
+        return *this;
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr rotation_iterator_base operator++(int) noexcept
+    {
+        auto tmp = *this;
+        this->operator++();
+        return tmp;
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr rotation_iterator_base & operator--() noexcept
+    {
+        --it;
+        if (it == iterable.wrap_after_)
+        {
+            it = iterable.end_after_;
+        }
+        else if (it == iterable.end_after_)
+        {
+            it = std::next(iterable.wrap_to_, -1);
+        }
+
+        return *this;
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr rotation_iterator_base operator--(int) noexcept
+    {
+        auto tmp = *this;
+        this->operator--();
+        return tmp;
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr bool operator==(const rotation_iterator_base & rhs) const noexcept
+    {
+        return iterable.begin_ == rhs.iterable.begin_
+                && iterable.wrap_to_ == rhs.iterable.wrap_to_
+                && iterable.wrap_after_ == rhs.iterable.wrap_after_
+                && iterable.end_after_ == rhs.iterable.end_after_
+                && iterable.end_ == rhs.iterable.end_
+                && it == rhs.it;
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr bool operator!=(const rotation_iterator_base & rhs) const noexcept
+    {
+        return !(*this == rhs);
+    }
+
+  protected:
+    const rotation_iterable<WrappedIterator> & iterable;
+    wrapped_iterator it;
+};  // rotation_iterator_base
+
+template <typename WrappedIterator>
+struct rotation_iterator final
+  : public rotation_iterator_base<WrappedIterator>
+{
+  private:
+    using base = rotation_iterator_base<WrappedIterator>;
+  public:
+    using wrapped_iterator = WrappedIterator;
+    using reference = typename base::reference;
+
+    rotation_iterator(const rotation_iterable<WrappedIterator> & iterable,
+        wrapped_iterator iterator) noexcept
+      : base{iterable, iterator} {}
+
+    HEDLEY_ALWAYS_INLINE
+    constexpr reference operator*() const
+    {
+        return *base::it;
+    }
+};  // rotation_iterator
+
+template <typename WrappedIterator>
+struct rotation_const_iterator final
+  : public rotation_iterator_base<WrappedIterator>
+{
+  private:
+    using base = rotation_iterator_base<WrappedIterator>;
+  public:
+    using wrapped_iterator = WrappedIterator;
+    using const_reference = typename base::const_reference;
+
+    rotation_const_iterator(const rotation_iterable<WrappedIterator> & iterable,
+        wrapped_iterator iterator) noexcept
+      : base{iterable, iterator} {}
+
+    HEDLEY_ALWAYS_INLINE
+    constexpr const_reference operator*() const
+    {
+        return *base::it;
+    }
+};  // rotation_const_iterator
+
+template <typename ContainerT>
+auto rotated_by(const ContainerT & container,
+    typename ContainerT::size_type rotate_by)
+{
+    return rotation_iterable{std::begin(container), std::end(container), rotate_by};
+}
+
+template <typename IteratorT,
+          typename UnaryFunction>
+constexpr void for_each_rotated_by(IteratorT begin, IteratorT end,
+    typename std::iterator_traits<IteratorT>::difference_type rotate_by,
+    UnaryFunction && f)
+{
+    auto size = std::distance(begin, end);
+    auto it = std::next(begin, rotate_by);
+    for (std::size_t i = rotate_by; i < size; ++i, ++it) f(i, *it);
+    it = begin;
+    for (std::size_t i = 0; i < rotate_by; ++i, ++it) f(i, *it);
+}
+
+}  // namespace dpf
+
+namespace std
+{
+
+template <typename WrappedIterator>
+struct iterator_traits<dpf::rotation_iterator<WrappedIterator>>
+{
+    using iterator_category = std::bidirectional_iterator_tag;
+    using difference_type = typename std::iterator_traits<WrappedIterator>::difference_type;
+    using value_type = typename std::iterator_traits<WrappedIterator>::value_type;
+    using reference = typename std::iterator_traits<WrappedIterator>::reference;
+    using const_reference = typename std::add_const_t<reference>;
+    using pointer = typename std::iterator_traits<WrappedIterator>::pointer;
+};
+
+template <typename WrappedIterator>
+struct iterator_traits<dpf::rotation_const_iterator<WrappedIterator>>
+{
+    using iterator_category = std::bidirectional_iterator_tag;
+    using difference_type = typename std::iterator_traits<WrappedIterator>::difference_type;
+    using value_type = typename std::iterator_traits<WrappedIterator>::value_type;
+    using reference = typename std::iterator_traits<WrappedIterator>::reference;
+    using const_reference = typename std::add_const_t<reference>;
+    using pointer = typename std::iterator_traits<WrappedIterator>::pointer;
+};
+
+}  // namespace std
+
+#endif  // LIBDPF_INCLUDE_DPF_ROTATED_VIEW_HPP__

include/dpf/secret_share.hpp

@@ -0,0 +1,544 @@
+/// @file dpf/secret_share.hpp
+/// @brief Thin (2,2) additive and subtractive secret-share wrappers.
+/// @details Layout-identical to `T`. Party is a compile-time `0` or `1`.
+///          Reconstruction: additive opens by sum, subtractive by
+///          `share0 - share1`. Linear combinations of same-party shares are
+///          supported; mixing additive with subtractive applies the correct
+///          party coefficient. A plaintext absorbs on party 0 only.
+/// @copyright Copyright (c) 2019-2026 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_SECRET_SHARE_HPP__
+#define LIBDPF_INCLUDE_DPF_SECRET_SHARE_HPP__
+
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+#include <ostream>
+#include <type_traits>
+#include <utility>
+
+#include "hedley/hedley.h"
+
+#include "dpf/twiddle.hpp"
+
+namespace dpf
+{
+
+/// Sharing scheme tag.
+enum class sharing : unsigned char
+{
+    additive = 0,
+    subtractive = 1
+};
+
+template <typename T, std::size_t Party, sharing Scheme>
+struct secret_share;
+
+template <typename T, std::size_t Party>
+using additive_share = secret_share<T, Party, sharing::additive>;
+
+template <typename T, std::size_t Party>
+using subtractive_share = secret_share<T, Party, sharing::subtractive>;
+
+template <typename T>
+struct is_secret_share : std::false_type
+{
+};
+
+template <typename T, std::size_t Party, sharing Scheme>
+struct is_secret_share<secret_share<T, Party, Scheme>> : std::true_type
+{
+};
+
+template <typename T>
+inline constexpr bool is_secret_share_v = is_secret_share<std::decay_t<T>>::value;
+
+template <typename T>
+struct share_party;
+
+template <typename T, std::size_t Party, sharing Scheme>
+struct share_party<secret_share<T, Party, Scheme>>
+  : std::integral_constant<std::size_t, Party>
+{
+};
+
+template <typename T>
+inline constexpr std::size_t share_party_v = share_party<std::decay_t<T>>::value;
+
+template <typename T>
+struct share_scheme;
+
+template <typename T, std::size_t Party, sharing Scheme>
+struct share_scheme<secret_share<T, Party, Scheme>>
+  : std::integral_constant<sharing, Scheme>
+{
+};
+
+template <typename T>
+inline constexpr sharing share_scheme_v = share_scheme<std::decay_t<T>>::value;
+
+template <typename T>
+struct share_value_type;
+
+template <typename T, std::size_t Party, sharing Scheme>
+struct share_value_type<secret_share<T, Party, Scheme>>
+{
+    using type = T;
+};
+
+template <typename T>
+using share_value_type_t = typename share_value_type<std::decay_t<T>>::type;
+
+namespace detail
+{
+
+/// Party coefficient of the secret for this scheme: additive always +1;
+/// subtractive is +1 for party 0 and −1 for party 1.
+template <sharing Scheme, std::size_t Party, typename T>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_PURE
+constexpr T party_coeff_times(const T & v) noexcept
+{
+    if constexpr (Scheme == sharing::additive || Party == 0)
+        return v;
+    else
+        return static_cast<T>(-v);
+}
+
+}  // namespace detail
+
+template <typename T, std::size_t Party, sharing Scheme>
+struct secret_share
+{
+    static_assert(Party == 0 || Party == 1,
+        "secret_share party must be 0 or 1");
+
+    using value_type = T;
+    static constexpr std::size_t party = Party;
+    static constexpr sharing scheme = Scheme;
+
+    T value{};
+
+    secret_share() = default;
+    secret_share(const secret_share &) noexcept = default;
+    secret_share(secret_share &&) noexcept = default;
+    secret_share & operator=(const secret_share &) noexcept = default;
+    secret_share & operator=(secret_share &&) noexcept = default;
+    ~secret_share() = default;
+
+    /// Bit-preserving construction. Does not apply a party coefficient.
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_CONST
+    static constexpr secret_share from_raw(T v) noexcept
+    {
+        secret_share s;
+        s.value = v;
+        return s;
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_CONST
+    constexpr const T & raw() const noexcept { return value; }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_PURE
+    constexpr T & raw() noexcept { return value; }
+
+    /// Secret-preserving conversion to an additive share of the same party.
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_PURE
+    constexpr additive_share<T, Party> as_additive() const noexcept
+    {
+        if constexpr (Scheme == sharing::additive)
+            return additive_share<T, Party>::from_raw(value);
+        // subtractive → additive: party 0 keeps bits; party 1 negates.
+        return additive_share<T, Party>::from_raw(
+            detail::party_coeff_times<sharing::subtractive, Party>(value));
+    }
+
+    /// Secret-preserving conversion to a subtractive share of the same party.
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_PURE
+    constexpr subtractive_share<T, Party> as_subtractive() const noexcept
+    {
+        if constexpr (Scheme == sharing::subtractive)
+            return subtractive_share<T, Party>::from_raw(value);
+        // additive → subtractive: party 0 keeps bits; party 1 negates.
+        return subtractive_share<T, Party>::from_raw(
+            detail::party_coeff_times<sharing::additive, Party>(value));
+    }
+
+    /// Bit-preserving retag (no secret-preserving sign fix).
+    template <sharing NewScheme, std::size_t NewParty = Party>
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_PURE
+    constexpr secret_share<T, NewParty, NewScheme> retag() const noexcept
+    {
+        return secret_share<T, NewParty, NewScheme>::from_raw(value);
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_PURE
+    constexpr secret_share operator-() const noexcept
+    {
+        return from_raw(static_cast<T>(-value));
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    constexpr secret_share & operator+=(const secret_share & rhs) noexcept
+    {
+        value = static_cast<T>(value + rhs.value);
+        return *this;
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    constexpr secret_share & operator-=(const secret_share & rhs) noexcept
+    {
+        value = static_cast<T>(value - rhs.value);
+        return *this;
+    }
+
+    template <typename Scalar,
+              std::enable_if_t<!is_secret_share_v<Scalar>, int> = 0>
+    HEDLEY_ALWAYS_INLINE
+    constexpr secret_share & operator*=(const Scalar & c) noexcept
+    {
+        value = static_cast<T>(value * static_cast<T>(c));
+        return *this;
+    }
+
+    /// Absorb a public plaintext on party 0 only.
+    template <typename Plain,
+              std::enable_if_t<!is_secret_share_v<Plain>
+                  && std::is_convertible_v<Plain, T>, int> = 0>
+    HEDLEY_ALWAYS_INLINE
+    constexpr secret_share & operator+=(const Plain & c) noexcept
+    {
+        if constexpr (Party == 0)
+            value = static_cast<T>(value + static_cast<T>(c));
+        return *this;
+    }
+
+    template <typename Plain,
+              std::enable_if_t<!is_secret_share_v<Plain>
+                  && std::is_convertible_v<Plain, T>, int> = 0>
+    HEDLEY_ALWAYS_INLINE
+    constexpr secret_share & operator-=(const Plain & c) noexcept
+    {
+        if constexpr (Party == 0)
+            value = static_cast<T>(value - static_cast<T>(c));
+        return *this;
+    }
+};
+
+// ---------------------------------------------------------------------------
+// Same-scheme, same-party arithmetic
+// ---------------------------------------------------------------------------
+
+template <typename T, std::size_t Party, sharing Scheme>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_PURE
+constexpr secret_share<T, Party, Scheme> operator+(
+    secret_share<T, Party, Scheme> lhs,
+    const secret_share<T, Party, Scheme> & rhs) noexcept
+{
+    lhs += rhs;
+    return lhs;
+}
+
+template <typename T, std::size_t Party, sharing Scheme>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_PURE
+constexpr secret_share<T, Party, Scheme> operator-(
+    secret_share<T, Party, Scheme> lhs,
+    const secret_share<T, Party, Scheme> & rhs) noexcept
+{
+    lhs -= rhs;
+    return lhs;
+}
+
+template <typename T, std::size_t Party, sharing Scheme, typename Scalar,
+          std::enable_if_t<!is_secret_share_v<Scalar>, int> = 0>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_PURE
+constexpr secret_share<T, Party, Scheme> operator*(
+    secret_share<T, Party, Scheme> lhs, const Scalar & c) noexcept
+{
+    lhs *= c;
+    return lhs;
+}
+
+template <typename T, std::size_t Party, sharing Scheme, typename Scalar,
+          std::enable_if_t<!is_secret_share_v<Scalar>, int> = 0>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_PURE
+constexpr secret_share<T, Party, Scheme> operator*(
+    const Scalar & c, secret_share<T, Party, Scheme> rhs) noexcept
+{
+    rhs *= c;
+    return rhs;
+}
+
+// ---------------------------------------------------------------------------
+// Cross-scheme, same-party: keep the left-hand scheme; party 1 flips the
+// operand whose scheme differs from the result.
+// ---------------------------------------------------------------------------
+
+template <typename T, std::size_t Party, sharing LhsScheme, sharing RhsScheme,
+          std::enable_if_t<LhsScheme != RhsScheme, int> = 0>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_PURE
+constexpr secret_share<T, Party, LhsScheme> operator+(
+    const secret_share<T, Party, LhsScheme> & lhs,
+    const secret_share<T, Party, RhsScheme> & rhs) noexcept
+{
+    if constexpr (Party == 0)
+        return secret_share<T, Party, LhsScheme>::from_raw(
+            static_cast<T>(lhs.raw() + rhs.raw()));
+    else
+        return secret_share<T, Party, LhsScheme>::from_raw(
+            static_cast<T>(lhs.raw() - rhs.raw()));
+}
+
+template <typename T, std::size_t Party, sharing LhsScheme, sharing RhsScheme,
+          std::enable_if_t<LhsScheme != RhsScheme, int> = 0>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_PURE
+constexpr secret_share<T, Party, LhsScheme> operator-(
+    const secret_share<T, Party, LhsScheme> & lhs,
+    const secret_share<T, Party, RhsScheme> & rhs) noexcept
+{
+    if constexpr (Party == 0)
+        return secret_share<T, Party, LhsScheme>::from_raw(
+            static_cast<T>(lhs.raw() - rhs.raw()));
+    else
+        return secret_share<T, Party, LhsScheme>::from_raw(
+            static_cast<T>(lhs.raw() + rhs.raw()));
+}
+
+// ---------------------------------------------------------------------------
+// Plaintext absorb (party 0 only)
+// ---------------------------------------------------------------------------
+
+template <typename T, std::size_t Party, sharing Scheme, typename Plain,
+          std::enable_if_t<!is_secret_share_v<Plain>
+              && std::is_convertible_v<Plain, T>, int> = 0>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_PURE
+constexpr secret_share<T, Party, Scheme> operator+(
+    secret_share<T, Party, Scheme> lhs, const Plain & c) noexcept
+{
+    lhs += c;
+    return lhs;
+}
+
+template <typename T, std::size_t Party, sharing Scheme, typename Plain,
+          std::enable_if_t<!is_secret_share_v<Plain>
+              && std::is_convertible_v<Plain, T>, int> = 0>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_PURE
+constexpr secret_share<T, Party, Scheme> operator+(
+    const Plain & c, secret_share<T, Party, Scheme> rhs) noexcept
+{
+    rhs += c;
+    return rhs;
+}
+
+template <typename T, std::size_t Party, sharing Scheme, typename Plain,
+          std::enable_if_t<!is_secret_share_v<Plain>
+              && std::is_convertible_v<Plain, T>, int> = 0>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_PURE
+constexpr secret_share<T, Party, Scheme> operator-(
+    secret_share<T, Party, Scheme> lhs, const Plain & c) noexcept
+{
+    lhs -= c;
+    return lhs;
+}
+
+// ---------------------------------------------------------------------------
+// Equality (same party, same scheme) — compare raw bits
+// ---------------------------------------------------------------------------
+
+template <typename T, std::size_t Party, sharing Scheme>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_PURE
+constexpr bool operator==(const secret_share<T, Party, Scheme> & lhs,
+    const secret_share<T, Party, Scheme> & rhs) noexcept
+{
+    return lhs.raw() == rhs.raw();
+}
+
+template <typename T, std::size_t Party, sharing Scheme>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_PURE
+constexpr bool operator!=(const secret_share<T, Party, Scheme> & lhs,
+    const secret_share<T, Party, Scheme> & rhs) noexcept
+{
+    return !(lhs == rhs);
+}
+
+// ---------------------------------------------------------------------------
+// Reconstruction
+// ---------------------------------------------------------------------------
+
+template <typename T, sharing Scheme>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_PURE
+constexpr T reconstruct(const secret_share<T, 0, Scheme> & s0,
+    const secret_share<T, 1, Scheme> & s1) noexcept
+{
+    if constexpr (Scheme == sharing::additive)
+        return static_cast<T>(s0.raw() + s1.raw());
+    else
+        return static_cast<T>(s0.raw() - s1.raw());
+}
+
+template <typename T, sharing Scheme>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_PURE
+constexpr T reconstruct(const secret_share<T, 1, Scheme> & s1,
+    const secret_share<T, 0, Scheme> & s0) noexcept
+{
+    return reconstruct(s0, s1);
+}
+
+// ---------------------------------------------------------------------------
+// Plaintext splits (share1 = 0)
+// ---------------------------------------------------------------------------
+
+template <typename T>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_CONST
+constexpr auto make_additive_shares(T secret) noexcept
+{
+    using T_ = std::remove_cv_t<std::remove_reference_t<T>>;
+    return std::make_pair(
+        additive_share<T_, 0>::from_raw(static_cast<T_>(secret)),
+        additive_share<T_, 1>::from_raw(T_{}));
+}
+
+template <typename T>
+HEDLEY_ALWAYS_INLINE
+HEDLEY_CONST
+constexpr auto make_subtractive_shares(T secret) noexcept
+{
+    using T_ = std::remove_cv_t<std::remove_reference_t<T>>;
+    return std::make_pair(
+        subtractive_share<T_, 0>::from_raw(static_cast<T_>(secret)),
+        subtractive_share<T_, 1>::from_raw(T_{}));
+}
+
+// ---------------------------------------------------------------------------
+// Party-tagged DPF key wrapper
+// ---------------------------------------------------------------------------
+
+template <typename T>
+struct is_party_key : std::false_type
+{
+};
+
+template <std::size_t Party, typename Key>
+struct party_key : Key
+{
+    static_assert(Party == 0 || Party == 1, "party_key party must be 0 or 1");
+
+    static constexpr std::size_t party = Party;
+    using key_type = Key;
+
+    party_key() = default;
+
+    HEDLEY_ALWAYS_INLINE
+    explicit party_key(Key k)
+      : Key(std::move(k))
+    {
+#ifndef NDEBUG
+        assert(static_cast<std::size_t>(
+            static_cast<bool>(dpf::get_lo_bit(this->root()))) == Party);
+#endif
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    Key & key() noexcept { return static_cast<Key &>(*this); }
+
+    HEDLEY_ALWAYS_INLINE
+    const Key & key() const noexcept { return static_cast<const Key &>(*this); }
+
+    /// Party-tagged additive share of the comparison absorb addend.
+    HEDLEY_ALWAYS_INLINE
+    auto cmp_addend() const noexcept
+    {
+        return additive_share<std::uint64_t, Party>::from_raw(
+            Key::cmp_addend());
+    }
+};
+
+template <std::size_t Party, typename Key>
+struct is_party_key<party_key<Party, Key>> : std::true_type
+{
+};
+
+template <typename T>
+inline constexpr bool is_party_key_v = is_party_key<std::decay_t<T>>::value;
+
+template <typename T>
+struct party_of;  // incomplete for non-`party_key` (fail loudly on misuse)
+
+template <std::size_t Party, typename Key>
+struct party_of<party_key<Party, Key>>
+  : std::integral_constant<std::size_t, Party>
+{
+};
+
+template <typename T>
+inline constexpr std::size_t party_of_v = party_of<std::decay_t<T>>::value;
+
+/// Strip a `party_key` wrapper; bare keys are unchanged. Memoizers and other
+/// tree-layout helpers key on the underlying DPF key type so a memoizer built
+/// for party 0 also accepts party 1.
+template <typename T>
+struct unwrap_party_key
+{
+    using type = std::decay_t<T>;
+};
+template <std::size_t Party, typename Key>
+struct unwrap_party_key<party_key<Party, Key>>
+{
+    using type = Key;
+};
+template <typename T>
+using unwrap_party_key_t = typename unwrap_party_key<std::decay_t<T>>::type;
+
+template <std::size_t Party, typename Key>
+HEDLEY_ALWAYS_INLINE
+auto make_party_key(Key && k)
+{
+    return party_key<Party, std::decay_t<Key>>(std::forward<Key>(k));
+}
+
+template <typename Key0, typename Key1>
+HEDLEY_ALWAYS_INLINE
+auto make_party_key_pair(Key0 && k0, Key1 && k1)
+{
+    using K = std::decay_t<Key0>;
+    static_assert(std::is_same_v<K, std::decay_t<Key1>>,
+        "make_party_key_pair: both keys must have the same type");
+    return std::make_pair(
+        party_key<0, K>(std::forward<Key0>(k0)),
+        party_key<1, K>(std::forward<Key1>(k1)));
+}
+
+template <typename CharT, typename Traits, typename T, std::size_t Party,
+          sharing Scheme>
+std::basic_ostream<CharT, Traits> & operator<<(
+    std::basic_ostream<CharT, Traits> & os,
+    const secret_share<T, Party, Scheme> & s)
+{
+    return os << s.raw();
+}
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_SECRET_SHARE_HPP__

include/dpf/sequence_memoizer.hpp

@@ -0,0 +1,543 @@
+/// @file dpf/sequence_memoizer.hpp
+/// @brief
+/// @details
+/// @author Ryan Henry <ryan.henry@ucalgary.ca>
+/// @author Christopher Jiang <christopher.jiang@ucalgary.ca>
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_SEQUENCE_MEMOIZER_HPP__
+#define LIBDPF_INCLUDE_DPF_SEQUENCE_MEMOIZER_HPP__
+
+#include "hedley/hedley.h"
+
+#include <cstddef>
+#include <cstring>
+#include <type_traits>
+#include <functional>
+#include <utility>
+#include <iterator>
+#include <algorithm>
+#include <stdexcept>
+#include <optional>
+
+#include "dpf/sequence_recipe.hpp"
+
+namespace dpf
+{
+
+struct sequence_memoizer_tag_ {};
+
+template <typename DpfKey,
+          typename ReturnT = typename DpfKey::interior_node *>
+struct sequence_recipe_memoizer_base : public sequence_memoizer_tag_
+{
+  public:
+    using dpf_type = DpfKey;
+    using return_type = ReturnT;
+    using iterator_type = return_type;
+    using node_type = typename DpfKey::interior_node;
+    const sequence_recipe & recipe;
+
+    // level 0 should access the root
+    // level goes up to (and including) depth
+    virtual return_type operator[](std::size_t) const noexcept = 0;
+
+    // iterators should access most recently completed level
+    virtual return_type begin() const noexcept = 0;
+    virtual return_type end() const noexcept = 0;
+
+    virtual std::size_t assign_dpf(const dpf_type & dpf, const sequence_recipe & r)
+    {
+        if (&recipe != &r)
+        {
+            throw std::logic_error("memoizer cannot be used with different recipe");
+        }
+
+        if (dpf_.has_value() == false || std::memcmp(&dpf_root_, &dpf.root(), sizeof(node_type)) != 0
+            || std::memcmp(&dpf_common_part_hash_, &dpf.common_part_hash(), sizeof(digest_type)) != 0)
+        {
+            if (dpf.depth != recipe.depth())
+            {
+                throw std::logic_error("incorrect dpf depth");
+            }
+            this->operator[](0)[0] = dpf.root();
+            dpf_ = std::cref(dpf);
+            dpf_root_ = dpf.root();
+            dpf_common_part_hash_ = dpf.common_part_hash();
+            level_index = 1;
+        }
+
+        return level_index;
+    }
+
+    std::size_t advance_level()
+    {
+        return ++level_index;
+    }
+
+    std::size_t get_nodes_at_level() const
+    {
+        return get_nodes_at_level(level_index);
+    }
+
+    std::size_t get_nodes_at_level(std::size_t level) const
+    {
+        if (level == size_t(-1))
+        {
+            return 0;
+        }
+
+        if (level == depth)
+        {
+            return recipe.num_leaf_nodes();
+        }
+
+        return recipe.level_endpoints()[level+1] - recipe.level_endpoints()[level];
+    }
+
+    // returns true if first traversal should be taken
+    // this usually means traversing left, but for the inplace_reversing memoizer
+    // if it is working in reverse, this could be a right traversal
+    virtual bool traverse_first(std::size_t step) const
+    {
+        return recipe.recipe_steps()[step] > int8_t(-1);
+    }
+
+    // returns true if second traversal should be taken
+    // this usually means traversing right, but for the inplace_reversing memoizer
+    // if it is working in reverse, this could be a left traversal
+    virtual bool traverse_second(std::size_t step) const
+    {
+        return recipe.recipe_steps()[step] < int8_t(1);
+    }
+
+    // returns true if traversal should be done to the right
+    // this usually means traversing in the same direction as supplied, but for the
+    // inplace_reversing memoizer if it is working in reverse, this could be
+    // the opposite of the supplied direction
+    virtual bool get_direction(bool right) const
+    {
+        return right;
+    }
+
+  protected:
+    std::size_t depth;
+    std::size_t level_index;  // indicates current level being built
+
+    explicit sequence_recipe_memoizer_base(const sequence_recipe & r)
+      : recipe{r},
+        depth{recipe.level_endpoints().size()-1},
+        level_index{0},
+        dpf_{std::nullopt}
+    { }
+
+  private:
+    std::optional<std::reference_wrapper<const dpf_type>> dpf_;
+    node_type dpf_root_;
+    digest_type dpf_common_part_hash_;
+};
+
+namespace detail
+{
+
+template <typename ForwardIterT,
+          typename ReverseIterT>
+struct pointer_facade
+{
+  public:
+    using forward_iter = ForwardIterT;
+    using reverse_iter = ReverseIterT;
+
+    using value_type = typename std::iterator_traits<ForwardIterT>::value_type;
+    using reference = value_type &;
+    using const_reference = const value_type &;
+    using pointer = std::add_pointer_t<value_type>;
+    using iterator_category = std::bidirectional_iterator_tag;
+    using difference_type = std::pair<std::ptrdiff_t, std::ptrdiff_t>;
+
+    HEDLEY_ALWAYS_INLINE
+    pointer_facade(bool flip, forward_iter it, reverse_iter rit)
+        : flip_{flip}, it_{it}, rit_{rit}
+    { }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    reference operator*() const noexcept
+    {
+        return flip_ ? *rit_ : *it_;
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    pointer_facade & operator++() noexcept
+    {
+        ++it_;
+        ++rit_;
+        return *this;
+    }
+
+    HEDLEY_NO_THROW
+    pointer_facade operator++(int) noexcept
+    {
+        auto tmp = *this;
+        pointer_facade::operator++();
+        return tmp;
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    pointer_facade & operator--() noexcept
+    {
+        --it_;
+        --rit_;
+        return *this;
+    }
+
+    HEDLEY_NO_THROW
+    pointer_facade operator--(int) noexcept
+    {
+        auto tmp = *this;
+        pointer_facade::operator--();
+        return tmp;
+    }
+
+    pointer_facade & operator+=(std::size_t n) noexcept
+    {
+        it_ += n;
+        rit_ += n;
+        return *this;
+    }
+
+    pointer_facade operator+(std::size_t n) const noexcept
+    {
+        return pointer_facade(flip_, it_ + n, rit_ + n);
+    }
+
+    pointer_facade & operator-=(std::size_t n) noexcept
+    {
+        it_ -= n;
+        rit_ -= n;
+        return *this;
+    }
+
+    pointer_facade operator-(std::size_t n) const noexcept
+    {
+        return pointer_facade(flip_, it_ - n, rit_ - n);
+    }
+
+    difference_type operator-(pointer_facade rhs) const noexcept
+    {
+        return std::make_pair(it_ - rhs.it_, rit_ - rhs.rit_);
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    HEDLEY_PURE
+    reference operator[](std::size_t i)
+    {
+        return flip_ ? rit_[i] : it_[i];
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    HEDLEY_PURE
+    const_reference operator[](std::size_t i) const
+    {
+        return flip_ ? rit_[i] : it_[i];
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr bool operator==(const pointer_facade & rhs) const noexcept
+    {
+        return flip_ == rhs.flip_ && it_ == rhs.it_ && rit_ == rhs.rit_;
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr bool operator!=(const pointer_facade & rhs) const noexcept
+    {
+        return !(*this == rhs);
+    }
+
+  private:
+    bool flip_;
+    forward_iter it_;
+    reverse_iter rit_;
+};
+
+}  // namespace detail
+
+template <typename DpfKey,
+          typename Allocator = aligned_allocator<typename DpfKey::interior_node>>
+struct inplace_reversing_sequence_memoizer final
+  : public sequence_recipe_memoizer_base<DpfKey,
+        detail::pointer_facade<typename DpfKey::interior_node *, std::reverse_iterator<typename DpfKey::interior_node *>>>
+{
+  public:
+    using unique_ptr = typename Allocator::unique_ptr;
+    using forward_iter = typename DpfKey::interior_node *;
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    using reverse_iter = std::reverse_iterator<forward_iter>;
+    using return_type = detail::pointer_facade<forward_iter, reverse_iter>;
+  private:
+    using parent = sequence_recipe_memoizer_base<DpfKey, return_type>;
+HEDLEY_PRAGMA(GCC diagnostic pop)
+  public:
+    using parent::recipe;
+    using parent::depth;
+    using parent::level_index;
+    using parent::get_nodes_at_level;
+
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    explicit inplace_reversing_sequence_memoizer(const sequence_recipe & r,
+        Allocator alloc = Allocator{})
+      : parent::sequence_recipe_memoizer_base(r),
+        buf{alloc.allocate_unique_ptr(std::max(r.num_leaf_nodes(), std::size_t{1}))}
+    { }
+HEDLEY_PRAGMA(GCC diagnostic pop)
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    return_type operator[](std::size_t level) const noexcept override
+    {
+        // flip false => forward traversal
+        bool flip = (depth ^ level) & 1;
+        // first check used to determine if previous or current level is being requested
+        // second check used to determine if the last layer is being requested
+        //     in which case it is setup to always return buf in normal order
+        if (level == level_index-1 && level != depth)
+        {
+            std::size_t nodes_at_level = get_nodes_at_level(level);
+            return return_type(!flip, &buf[recipe.num_leaf_nodes()-nodes_at_level],
+                std::make_reverse_iterator(&buf[nodes_at_level]));
+        }
+
+        return return_type(flip, &buf[0],
+            std::make_reverse_iterator(&buf[recipe.num_leaf_nodes()]));
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    return_type begin() const noexcept override
+    {
+        return this->operator[](level_index - 1);
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    return_type end() const noexcept override
+    {
+        auto it = begin();
+        it += get_nodes_at_level(level_index - 1);
+        return it;
+    }
+
+    bool traverse_first(std::size_t step) const override
+    {
+        // flip false => forward traversal
+        bool flip = (depth ^ level_index) & 1;
+        step = !flip ? step : recipe.level_endpoints()[level_index] - step - 1 + recipe.level_endpoints()[level_index-1];
+        return !flip ? (recipe.recipe_steps()[step] > int8_t(-1)) : (recipe.recipe_steps()[step] < int8_t(1));
+    }
+
+    bool traverse_second(std::size_t step) const override
+    {
+        // flip false => forward traversal
+        bool flip = (depth ^ level_index) & 1;
+        step = !flip ? step : recipe.level_endpoints()[level_index] - step - 1 + recipe.level_endpoints()[level_index-1];
+        return !flip ? (recipe.recipe_steps()[step] < int8_t(1)) : (recipe.recipe_steps()[step] > int8_t(-1));
+    }
+
+    bool get_direction(bool right) const override
+    {
+        // flip false => forward traversal
+        bool flip = (depth ^ level_index) & 1;
+        return !flip ? right : !right;
+    }
+
+  private:
+    unique_ptr buf;
+};
+
+template <typename DpfKey,
+          typename Allocator = aligned_allocator<typename DpfKey::interior_node>>
+struct double_space_sequence_memoizer final
+  : public sequence_recipe_memoizer_base<DpfKey>
+{
+  private:
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    using parent = sequence_recipe_memoizer_base<DpfKey>;
+HEDLEY_PRAGMA(GCC diagnostic pop)
+  public:
+    using unique_ptr = typename Allocator::unique_ptr;
+    using return_type = typename DpfKey::interior_node *;
+    using parent::recipe;
+    using parent::depth;
+    using parent::level_index;
+    using parent::get_nodes_at_level;
+
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    explicit double_space_sequence_memoizer(const sequence_recipe & r, Allocator alloc = Allocator{})
+      : parent::sequence_recipe_memoizer_base(r),
+        buf{alloc.allocate_unique_ptr(2 * std::max(recipe.num_leaf_nodes(), std::size_t{1}))}
+    { }
+HEDLEY_PRAGMA(GCC diagnostic pop)
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    return_type operator[](std::size_t level) const noexcept override
+    {
+        auto b = (depth ^ level) & 1;
+        return Allocator::assume_aligned(&buf[recipe.num_leaf_nodes()*b]);
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    return_type begin() const noexcept override
+    {
+        return this->operator[](level_index - 1);
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    return_type end() const noexcept override
+    {
+        return this->operator[](level_index - 1) + get_nodes_at_level(level_index - 1);
+    }
+
+  private:
+    unique_ptr buf;
+};
+
+template <typename DpfKey,
+          typename Allocator = aligned_allocator<typename DpfKey::interior_node>>
+struct full_tree_sequence_memoizer final
+  : public sequence_recipe_memoizer_base<DpfKey>
+{
+  private:
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    using parent = sequence_recipe_memoizer_base<DpfKey>;
+HEDLEY_PRAGMA(GCC diagnostic pop)
+  public:
+    using unique_ptr = typename Allocator::unique_ptr;
+    using return_type = typename DpfKey::interior_node *;
+    using parent::recipe;
+    using parent::level_index;
+    using parent::get_nodes_at_level;
+
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    explicit full_tree_sequence_memoizer(const sequence_recipe & r, Allocator alloc = Allocator{})
+      : parent::sequence_recipe_memoizer_base(r),
+        buf{alloc.allocate_unique_ptr(std::max(
+            recipe.level_endpoints()[recipe.level_endpoints().size()-1] + recipe.num_leaf_nodes(),
+            std::size_t{1}))}
+    { }
+HEDLEY_PRAGMA(GCC diagnostic pop)
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    return_type operator[](std::size_t level) const noexcept override
+    {
+        return Allocator::assume_aligned(&buf[recipe.level_endpoints()[level]]);
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    return_type begin() const noexcept override
+    {
+        return this->operator[](level_index - 1);
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    return_type end() const noexcept override
+    {
+        return this->operator[](level_index - 1) + get_nodes_at_level(level_index - 1);
+    }
+
+  private:
+    unique_ptr buf;
+};
+
+namespace detail
+{
+
+template <typename MemoizerT>
+HEDLEY_ALWAYS_INLINE
+auto make_sequence_memoizer(const sequence_recipe & recipe)
+{
+    return MemoizerT(recipe);
+}
+
+}  // namespace detail
+
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+template <typename DpfKey>
+inline auto make_inplace_reversing_sequence_memoizer(const sequence_recipe & recipe)
+{
+    return detail::make_sequence_memoizer<inplace_reversing_sequence_memoizer<DpfKey>>(recipe);
+}
+
+template <typename DpfKey>
+inline auto make_inplace_reversing_sequence_memoizer(const DpfKey &, const sequence_recipe & recipe)
+{
+    return make_inplace_reversing_sequence_memoizer<DpfKey>(recipe);
+}
+
+template <typename DpfKey>
+inline auto make_double_space_sequence_memoizer(const sequence_recipe & recipe)
+{
+    return detail::make_sequence_memoizer<double_space_sequence_memoizer<DpfKey>>(recipe);
+}
+
+template <typename DpfKey>
+inline auto make_double_space_sequence_memoizer(const DpfKey &, const sequence_recipe & recipe)
+{
+    return make_double_space_sequence_memoizer<DpfKey>(recipe);
+}
+
+template <typename DpfKey>
+inline auto make_full_tree_sequence_memoizer(const sequence_recipe & recipe)
+{
+    return detail::make_sequence_memoizer<full_tree_sequence_memoizer<DpfKey>>(recipe);
+}
+
+template <typename DpfKey>
+inline auto make_full_tree_sequence_memoizer(const DpfKey &, const sequence_recipe & recipe)
+{
+    return make_full_tree_sequence_memoizer<DpfKey>(recipe);
+}
+HEDLEY_PRAGMA(GCC diagnostic pop)
+
+}  // namespace dpf
+
+namespace std
+{
+
+template <typename Iterator>
+struct iterator_traits<dpf::detail::pointer_facade<Iterator, std::reverse_iterator<Iterator>>>
+{
+  private:
+    using type = dpf::detail::pointer_facade<Iterator, std::reverse_iterator<Iterator>>;
+  public:
+    using iterator_category = typename type::iterator_category;
+    using difference_type = typename type::difference_type;
+    using value_type = typename type::value_type;
+    using reference = typename type::reference;
+    using const_reference = typename type::const_reference;
+    using pointer = typename type::pointer;
+};
+
+}  // namespace std
+
+#endif  // LIBDPF_INCLUDE_DPF_SEQUENCE_MEMOIZER_HPP__

include/dpf/sequence_recipe.hpp

@@ -0,0 +1,208 @@
+/// @file dpf/sequence_recipe.hpp
+/// @brief
+/// @details
+/// @author Ryan Henry <ryan.henry@ucalgary.ca>
+/// @author Christopher Jiang <christopher.jiang@ucalgary.ca>
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_SEQUENCE_RECIPE_HPP__
+#define LIBDPF_INCLUDE_DPF_SEQUENCE_RECIPE_HPP__
+
+#include <cstddef>
+#include <type_traits>
+#include <algorithm>
+#include <vector>
+#include <stdexcept>
+#include <list>
+#include <iterator>
+
+namespace dpf
+{
+
+struct sequence_recipe
+{
+  public:
+    sequence_recipe(const std::vector<int8_t> & steps,
+                const std::vector<std::size_t> & subsequence_indexes,
+                std::size_t leaf_index,
+                const std::vector<std::size_t> & level_endpoints)
+      : recipe_steps_{steps},
+        output_indices_{subsequence_indexes},
+        num_leaf_nodes_{leaf_index},
+        level_endpoints_{level_endpoints}
+    { }
+
+    constexpr const std::vector<int8_t> & recipe_steps() const noexcept { return recipe_steps_; }
+    constexpr const std::vector<std::size_t> & output_indices() const noexcept { return output_indices_; }
+    constexpr std::size_t num_leaf_nodes() const noexcept { return num_leaf_nodes_; }
+    constexpr const std::vector<std::size_t> & level_endpoints() const noexcept { return level_endpoints_; }
+    std::size_t depth() const noexcept { return level_endpoints_.size()-1; }
+
+  private:
+    std::vector<int8_t> recipe_steps_;
+    std::vector<std::size_t> output_indices_;
+    std::size_t num_leaf_nodes_;
+    std::vector<std::size_t> level_endpoints_;  // level_endpoints.size() = depth+1
+};
+
+namespace detail
+{
+
+template <typename DpfKey,
+          typename ForwardIterator>
+auto make_sequence_recipe(ForwardIterator begin, ForwardIterator end)
+{
+    static_assert(std::is_same_v<typename DpfKey::input_type, std::decay_t<decltype(*begin)>>);
+
+    using dpf_type = DpfKey;
+    using input_type = typename DpfKey::input_type;
+
+    if (!std::is_sorted(begin, end))
+    {
+        throw std::runtime_error("list must be sorted");
+    }
+    if (begin == end)
+    {
+        std::vector<std::size_t> level_endpoints(dpf_type::depth + 1, 0);
+        return sequence_recipe{{}, {}, 0, level_endpoints};
+    }
+
+    auto mask = dpf_type::msb_mask;
+
+    std::list<ForwardIterator> splits{begin, end};
+
+    std::vector<std::size_t> level_endpoints;
+    level_endpoints.push_back(0);
+    std::vector<int8_t> recipe_steps;
+
+    auto func = [&](const bool flip = false)
+    {
+        // `lower` and `upper` are always adjacent elements of `splits` with `lower` < `upper`
+        // [lower, upper) = "block"
+        for (auto upper = std::begin(splits), lower = upper++; upper != std::end(splits); lower = upper++)
+        {
+            // `upper_bound()` returns iterator to first element where the relevant bit (based on `mask`) is set
+            auto it = std::upper_bound(*lower, *upper, mask,
+                [&flip](auto a, auto b){ return static_cast<bool>(a&b) ^ flip; });
+            if (it == *lower) recipe_steps.push_back(-1);       // right only since first element in "block" requires right traversal
+            else if (it == *upper) recipe_steps.push_back(+1);  // left only since no element in "block" requires right traversal
+            else
+            {
+                recipe_steps.push_back(0);                      // both ways since some (non-lower) element within "block" requires right traversal
+                splits.insert(upper, it);
+            }
+        }
+        level_endpoints.push_back(recipe_steps.size());
+    };
+    if (dpf_type::depth > 0)
+    {
+        func(utils::uses_signed_msb_v<input_type>);
+        mask >>= 1;
+    }
+    for (std::size_t level_index = 1; level_index < dpf_type::depth; ++level_index, mask>>=1)
+    {
+        func();
+    }
+
+    std::vector<std::size_t> output_indices;
+    // output_indices.push_back(*begin % outputs_per_leaf);
+    std::size_t leaf_index = 0;  // *begin/outputs_per_leaf < *(begin+1)/outs_per_leaf;
+    constexpr auto mod = utils::mod_pow_2<input_type>{};
+    constexpr auto clz = utils::countl_zero_symmetric_difference<input_type>{};
+    for (auto curr = begin, prev = curr; curr != end; prev = curr++)
+    {
+        leaf_index += (clz(*prev, *curr)) < dpf_type::depth;
+        output_indices.push_back(leaf_index * dpf_type::outputs_per_leaf + mod(*curr, dpf_type::lg_outputs_per_leaf));
+    }
+
+    return sequence_recipe{recipe_steps, output_indices, leaf_index+1, level_endpoints};
+}
+
+}  // namespace detail
+
+template <typename DpfKey,
+          typename ForwardIterator>
+auto make_sequence_recipe(ForwardIterator begin, ForwardIterator end)
+{
+    return detail::make_sequence_recipe<DpfKey>(begin, end);
+}
+
+template <typename DpfKey,
+          typename ForwardIterator>
+auto make_sequence_recipe(const DpfKey &, ForwardIterator begin, ForwardIterator end)
+{
+    return make_sequence_recipe<DpfKey>(begin, end);
+}
+
+/// Build a sequence recipe that stops at `StopLevel` with packing `LgOpl`
+/// (multi-level / `out<I>` slots). Lane points are in the slot's prefix domain.
+template <std::size_t StopLevel, std::size_t LgOpl, typename InputT,
+          typename ForwardIterator>
+auto make_sequence_recipe_at(InputT msb_mask, ForwardIterator begin,
+    ForwardIterator end)
+{
+    using input_type = InputT;
+    constexpr auto mod = utils::mod_pow_2<input_type>{};
+    constexpr auto clz = utils::countl_zero_symmetric_difference<input_type>{};
+    constexpr std::size_t opl = std::size_t{1} << LgOpl;
+
+    if (!std::is_sorted(begin, end))
+        throw std::runtime_error("list must be sorted");
+    if (begin == end)
+    {
+        std::vector<std::size_t> level_endpoints(StopLevel + 1, 0);
+        return sequence_recipe{{}, {}, 0, level_endpoints};
+    }
+
+    auto mask = msb_mask;
+    std::list<ForwardIterator> splits{begin, end};
+    std::vector<std::size_t> level_endpoints;
+    level_endpoints.push_back(0);
+    std::vector<int8_t> recipe_steps;
+
+    auto func = [&](const bool flip = false) {
+        for (auto upper = std::begin(splits), lower = upper++;
+             upper != std::end(splits); lower = upper++)
+        {
+            auto it = std::upper_bound(*lower, *upper, mask,
+                [&flip](auto a, auto b) {
+                    return static_cast<bool>(a & b) ^ flip;
+                });
+            if (it == *lower)
+                recipe_steps.push_back(-1);
+            else if (it == *upper)
+                recipe_steps.push_back(+1);
+            else
+            {
+                recipe_steps.push_back(0);
+                splits.insert(upper, it);
+            }
+        }
+        level_endpoints.push_back(recipe_steps.size());
+    };
+    if (StopLevel > 0)
+    {
+        func(utils::uses_signed_msb_v<input_type>);
+        mask >>= 1;
+    }
+    for (std::size_t level_index = 1; level_index < StopLevel;
+         ++level_index, mask >>= 1)
+        func();
+
+    std::vector<std::size_t> output_indices;
+    std::size_t leaf_index = 0;
+    for (auto curr = begin, prev = curr; curr != end; prev = curr++)
+    {
+        leaf_index += (clz(*prev, *curr)) < StopLevel;
+        output_indices.push_back(
+            leaf_index * opl + mod(*curr, LgOpl));
+    }
+    return sequence_recipe{recipe_steps, output_indices, leaf_index + 1,
+        level_endpoints};
+}
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_SEQUENCE_RECIPE_HPP__

include/dpf/sequence_utils.hpp

@@ -0,0 +1,17 @@
+#ifndef LIBDPF_INCLUDE_DPF_SEQUENCE_UTILS_HPP__
+#define LIBDPF_INCLUDE_DPF_SEQUENCE_UTILS_HPP__
+
+namespace dpf
+{
+
+struct return_type_tag_{};
+
+struct return_entire_node_tag_ final : public return_type_tag_ {};
+// static constexpr auto return_entire_node_tag = return_entire_node_tag_{};
+
+struct return_output_only_tag_ final : public return_type_tag_ {};
+// static constexpr auto return_output_only_tag = return_output_only_tag_{};
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_SEQUENCE_UTILS_HPP__

include/dpf/setbit_index_iterable.hpp

@@ -0,0 +1,357 @@
+/// @file dpf/setbit_index_iterable.hpp
+/// @brief
+/// @details
+/// @author Ryan Henry <ryan.henry@ucalgary.ca>
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_SETBIT_INDEX_ITERABLE_HPP__
+#define LIBDPF_INCLUDE_DPF_SETBIT_INDEX_ITERABLE_HPP__
+
+#include <cstddef>
+#include <type_traits>
+#include <iterator>
+#include <utility>
+
+#include "hedley/hedley.h"
+
+#include "dpf/bit_array.hpp"
+#include "dpf/subinterval_iterable.hpp"
+
+namespace dpf
+{
+
+template <typename ChildT,
+          typename WordT>
+class const_setbit_iterator;  // forward declaration
+
+template <typename ChildT,
+          typename WordT>
+class setbit_index_iterable
+{
+  public:
+    using iter = subinterval_iterable<bit_iterator<ChildT, WordT>>;
+    using size_type = typename bit_array_base<ChildT, WordT>::size_type;
+    using word_type = typename bit_array_base<ChildT, WordT>::word_type;
+    using word_pointer = typename bit_array_base<ChildT, WordT>::word_pointer;
+    using const_iterator = const_setbit_iterator<ChildT, WordT>;
+    static constexpr size_type bits_per_word = bit_array_base<ChildT, WordT>::bits_per_word;
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    explicit setbit_index_iterable(iter it) noexcept
+      : it_{std::move(it)},
+        begin_{it_.it_.word_ptr_},
+        end_{begin_ + it_.buf_size_},
+        base_index_{calc_base_index(it_)},
+        length_{calc_length(it_)}
+    {
+        update_bit_array();
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    const_iterator begin() const noexcept
+    {
+        return const_iterator{begin_, base_index_,
+            typename const_iterator::const_iterator_begin_tag{}};
+    }
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    const_iterator cbegin() const noexcept
+    {
+        return begin();
+    }
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    const_iterator end() const noexcept
+    {
+        return const_iterator{end_, length_,
+            typename const_iterator::const_iterator_end_tag{}};
+    }
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    const_iterator cend() const noexcept
+    {
+        return end();
+    }
+
+  private:
+    iter it_;
+    word_pointer begin_;
+    word_pointer end_;
+    size_type base_index_;
+    size_type length_;
+
+    static constexpr size_type calc_base_index(const iter & it)
+    {
+        return it.outputs_ == 0 ? it.from_ : utils::quotient_floor(it.from_, it.outputs_) * it.outputs_;
+    }
+
+    static constexpr size_type calc_length(const iter & it) noexcept
+    {
+        return it.outputs_ == 0 ? utils::quotient_ceiling(it.to_, bits_per_word) * bits_per_word : utils::quotient_ceiling(it.to_, it.outputs_) * it.outputs_;
+    }
+
+    // outputs_ == 0 implies generated from eval_sequence
+    //     so no bits need to be zero'd
+    // otherwise generated from eval_interval
+    //     depending on node size, word size, and where the from, to points fall within nodes
+    //     some words will need to be zero'd completely
+    //     while other words just need some bits to be masked out
+    constexpr void update_bit_array()
+    {
+        if (it_.outputs_ == 0)
+        {
+            return;
+        }
+
+        word_type zero = 0;
+        word_type mask = (~word_type(0)) << (it_.from_ % bits_per_word);
+        word_pointer cur = begin_;
+        std::size_t loc = it_.from_ % it_.outputs_;
+
+        while (loc >= bits_per_word)
+        {
+            *cur = zero;
+            ++cur;
+            loc -= bits_per_word;
+        }
+        *cur &= mask;
+
+        mask = (~word_type(0)) >> (bits_per_word - it_.to_ % bits_per_word - 1);
+        cur = end_-1;
+        loc = it_.to_ % it_.outputs_;
+
+        while (loc < it_.outputs_ - bits_per_word)
+        {
+            *cur = zero;
+            --cur;
+            loc += bits_per_word;
+        }
+        *cur &= mask;
+    }
+};  // class dpf::setbit_index_iterable
+
+template <typename ChildT,
+          typename WordT>
+class const_setbit_iterator
+{
+    using array_type = bit_array_base<ChildT, WordT>;
+  public:
+    using value_type = typename array_type::size_type;
+    using reference = value_type;
+    using const_reference = reference;
+    using pointer = std::add_pointer_t<reference>;
+    using iterator_category = std::bidirectional_iterator_tag;
+    using word_type = typename array_type::word_type;
+    using word_pointer = typename array_type::const_word_pointer;
+    using size_type = typename array_type::size_type;
+    using difference_type = std::ptrdiff_t;
+    static constexpr auto bits_per_word = array_type::bits_per_word;
+
+    HEDLEY_ALWAYS_INLINE
+    constexpr const_setbit_iterator(const_setbit_iterator &&) noexcept = default;
+    HEDLEY_ALWAYS_INLINE
+    constexpr const_setbit_iterator(const const_setbit_iterator &) noexcept = default;
+
+    ~const_setbit_iterator() noexcept = default;
+    const_setbit_iterator & operator=(const_setbit_iterator &&) noexcept = default;
+    const_setbit_iterator & operator=(const const_setbit_iterator &) = default;
+
+    HEDLEY_PURE
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr reference operator*() const noexcept
+    {
+        // count trailing zeros -> offset within current_word_
+        return base_index_ + utils::ctz(current_word_);
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr const_setbit_iterator & operator++() noexcept
+    {
+        current_word_ &= current_word_-1;  // clear first (lowest) set bit
+        seek_to_next_bit();  // advance `word_ptr` till nonzero word
+        return *this;
+    }
+
+    HEDLEY_NO_THROW
+    const_setbit_iterator operator++(int) noexcept
+    {
+        auto tmp = *this;
+        const_setbit_iterator::operator++();
+        return tmp;
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    const_setbit_iterator & operator--() noexcept
+    {
+        seek_to_prev_bit();
+        auto clz = utils::clz(current_word_ ^ *word_ptr_);
+        current_word_ |= ~(~word_type(0) >> 1) >> clz;
+        return *this;
+    }
+
+    HEDLEY_NO_THROW
+    const_setbit_iterator operator--(int) noexcept
+    {
+        auto tmp = *this;
+        const_setbit_iterator::operator--();
+        return tmp;
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr bool operator==(const const_setbit_iterator & rhs) const noexcept
+    {
+        return word_ptr_ == rhs.word_ptr_ &&
+                current_word_ == rhs.current_word_;
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr bool operator<(const const_setbit_iterator & rhs) const noexcept
+    {
+        return (word_ptr_ < rhs.word_ptr_ ||
+            (word_ptr_ == rhs.word_ptr_ &&
+                current_word_ > rhs.current_word_));
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr bool operator!=(const const_setbit_iterator & rhs) const noexcept
+    {
+        return !(*this == rhs);
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr bool operator>(const const_setbit_iterator & rhs) const noexcept
+    {
+        return rhs < *this;
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr bool operator<=(const const_setbit_iterator & rhs) const noexcept
+    {
+        return !(rhs < *this);
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr bool operator>=(const const_setbit_iterator & rhs) const noexcept
+    {
+        return !(*this < rhs);
+    }
+
+  private:
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr void seek_to_next_bit() noexcept
+    {
+        while (HEDLEY_PREDICT(current_word_ == 0, false,
+            2.0 / bits_per_word))
+        {
+            current_word_ = utils::le(*(++word_ptr_));
+            base_index_ += bits_per_word;
+        }
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr void seek_to_prev_bit() noexcept
+    {
+        word_type lo_bits = current_word_ ^ *word_ptr_;
+        while (HEDLEY_PREDICT(lo_bits == 0, false,
+            2.0 / bits_per_word))
+        {
+            lo_bits = utils::le(*(--word_ptr_));
+            current_word_ = 0;
+            base_index_ -= bits_per_word;
+        }
+    }
+
+    struct const_iterator_end_tag final {};
+    struct const_iterator_begin_tag final {};
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NON_NULL()
+    explicit constexpr const_setbit_iterator(word_pointer word_ptr,
+        size_type base_index, const_iterator_begin_tag) noexcept
+        : word_ptr_{word_ptr},
+        current_word_{utils::le(*word_ptr_)},
+        base_index_{base_index}
+    {
+        seek_to_next_bit();
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NON_NULL()
+    explicit constexpr const_setbit_iterator(word_pointer word_ptr,
+        size_type base_index, const_iterator_end_tag) noexcept
+        : word_ptr_{word_ptr},
+        current_word_{*word_ptr_},  // utils::le(end) is a nop
+        base_index_{base_index}
+    { }
+
+    word_pointer word_ptr_;
+    word_type current_word_;
+    size_type base_index_;
+
+    friend const_setbit_iterator setbit_index_iterable<ChildT, WordT>::begin() const noexcept;
+    friend const_setbit_iterator setbit_index_iterable<ChildT, WordT>::end() const noexcept;
+};  // class dpf::const_setbit_iterator
+
+template <typename ChildT,
+          typename WordT>
+HEDLEY_ALWAYS_INLINE
+dpf::setbit_index_iterable<ChildT, WordT> indices_set_in(const subinterval_iterable<bit_iterator<ChildT, WordT>> & iter) noexcept
+{
+    return dpf::setbit_index_iterable<ChildT, WordT>{iter};
+}
+
+template <typename ChildT,
+          typename WordT>
+HEDLEY_ALWAYS_INLINE
+dpf::setbit_index_iterable<ChildT, WordT> indices_set_in(subinterval_iterable<bit_iterator<ChildT, WordT>> && iter) noexcept
+{
+    return dpf::setbit_index_iterable<ChildT, WordT>{std::forward<subinterval_iterable<bit_iterator<ChildT, WordT>>>(iter)};
+}
+
+template <typename ChildT,
+          typename WordT,
+          class UnaryFunction>
+HEDLEY_ALWAYS_INLINE
+void for_each_set_index(const subinterval_iterable<bit_iterator<ChildT, WordT>> & arr, UnaryFunction f)
+{
+    for (auto i : indices_set_in(arr)) f(i);
+}
+
+}  // namespace dpf
+
+namespace std
+{
+
+template <typename ChildT,
+          typename WordT>
+struct iterator_traits<dpf::const_setbit_iterator<ChildT, WordT>>
+{
+  private:
+    using type = dpf::const_setbit_iterator<ChildT, WordT>;
+  public:
+    using iterator_category = typename type::iterator_category;
+    using difference_type = typename type::difference_type;
+    using value_type = typename type::value_type;
+    using reference = typename type::reference;
+    using const_reference = typename type::const_reference;
+    using pointer = typename type::pointer;
+};
+
+}  // namespace std
+
+#endif  // LIBDPF_INCLUDE_DPF_SETBIT_INDEX_ITERABLE_HPP__

include/dpf/subinterval_iterable.hpp

@@ -0,0 +1,81 @@
+/// @file dpf/subinterval_iterable.hpp
+/// @author Ryan Henry <ryan.henry@ucalgary.ca>
+/// @brief defines `dpf::subinterval_iterable` and associated helpers
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_SUBINTERVAL_ITERABLE_HPP__
+#define LIBDPF_INCLUDE_DPF_SUBINTERVAL_ITERABLE_HPP__
+
+#include <cstddef>
+
+#include "hedley/hedley.h"
+
+namespace dpf
+{
+
+template <typename ChildT,
+          typename WordT>
+class setbit_index_iterable;  // forward declaration
+
+template <typename IteratorT>
+class subinterval_iterable
+{
+  public:
+    using iterator = IteratorT;
+    using size_type = std::size_t;
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    explicit subinterval_iterable(iterator it, size_type buf_size, size_type from, size_type to, size_type preclip, size_type outputs_per_leaf, bool nonempty = true) noexcept
+      : it_{it}, buf_size_{buf_size}, from_{from}, to_{to},
+        length_{nonempty ? to_ - from_ : 0}, preclip_{preclip},
+        outputs_{outputs_per_leaf}, nonempty_{nonempty}
+    { }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    iterator begin() const noexcept
+    {
+        return it_ + preclip_;
+    }
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    iterator cbegin() const noexcept
+    {
+        return it_ + preclip_;
+    }
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    iterator end() const noexcept
+    {
+        if (!nonempty_)
+            return it_ + preclip_;
+        return it_ + preclip_ + length_ + 1;
+    }
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    iterator cend() const noexcept
+    {
+        return end();
+    }
+
+  private:
+    iterator it_;
+    size_type buf_size_;
+    size_type from_;
+    size_type to_;
+    size_type length_;
+    size_type preclip_;
+    size_type outputs_;
+    bool nonempty_;
+
+    template <typename ChildT,
+              typename WordT>
+    friend class setbit_index_iterable;
+};  // class dpf::subinterval_iterable
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_SUBINTERVAL_ITERABLE_HPP__

include/dpf/subsequence_iterable.hpp

@@ -0,0 +1,422 @@
+/// @file dpf/subsequence_iterable.hpp
+/// @author Ryan Henry <ryan.henry@ucalgary.ca>
+/// @brief defines `dpf::subsequence_iterable` and associated helpers
+/// @details
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_SUBSEQUENCE_ITERABLE_HPP__
+#define LIBDPF_INCLUDE_DPF_SUBSEQUENCE_ITERABLE_HPP__
+
+#include <cstddef>
+#include <type_traits>
+#include <iterator>
+#include <vector>
+
+#include "hedley/hedley.h"
+
+namespace dpf
+{
+
+template <typename DpfKey,
+          typename OutputIterT,
+          typename PointsIterT>
+class subsequence_iterable
+{
+  public:
+    using output_type = typename std::iterator_traits<OutputIterT>::value_type;
+    using input_type = typename DpfKey::input_type;
+    using output_iterator = OutputIterT;
+    using points_iterator = PointsIterT;
+    static constexpr std::size_t outputs_per_leaf = DpfKey::outputs_per_leaf;
+    static constexpr std::size_t lg_outputs_per_leaf = DpfKey::lg_outputs_per_leaf;
+    static constexpr auto mod = utils::mod_pow_2<input_type>{};
+    class const_iterator;  // forward declaration
+    using iterator = const_iterator;
+
+    subsequence_iterable(output_iterator out_it, points_iterator begin, points_iterator end)
+      : out_it_{out_it}, begin_{begin}, end_{end}, count_{std::distance(begin_, end_)}
+    { }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    const_iterator begin() const noexcept
+    {
+        return const_iterator(out_it_, begin_);
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    const_iterator cbegin() const noexcept
+    {
+        return begin();
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    const_iterator end() const noexcept
+    {
+        return const_iterator(out_it_ + count_ * outputs_per_leaf, end_);
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    const_iterator cend() const noexcept
+    {
+        return end();
+    }
+
+    class const_iterator
+    {
+      public:
+        using value_type = output_type;
+        using reference = value_type;
+        using const_reference = reference;
+        using pointer = std::add_pointer_t<reference>;
+        using iterator_category = std::random_access_iterator_tag;
+        using size_type = std::size_t;
+        using difference_type = std::ptrdiff_t;
+
+        using subsequence_iterator_type = points_iterator;
+
+        HEDLEY_ALWAYS_INLINE
+        constexpr const_iterator(output_iterator out_it, subsequence_iterator_type it) noexcept
+          : out_it_{out_it}, it_{it}
+        { }
+
+        HEDLEY_ALWAYS_INLINE
+        constexpr const_iterator(const_iterator &&) noexcept = default;
+        HEDLEY_ALWAYS_INLINE
+        constexpr const_iterator(const const_iterator &) noexcept = default;
+
+        const_iterator & operator=(const_iterator &&) noexcept = default;
+        const_iterator & operator=(const const_iterator &) = default;
+        ~const_iterator() = default;
+
+        HEDLEY_PURE
+        HEDLEY_NO_THROW
+        HEDLEY_ALWAYS_INLINE
+        constexpr reference operator*() const noexcept
+        {
+            return out_it_[mod(*it_, lg_outputs_per_leaf)];
+        }
+
+        HEDLEY_NO_THROW
+        HEDLEY_ALWAYS_INLINE
+        constexpr const_iterator & operator++() noexcept
+        {
+            ++it_;
+            out_it_ += outputs_per_leaf;
+            return *this;
+        }
+
+        HEDLEY_NO_THROW
+        const_iterator operator++(int) noexcept
+        {
+            auto tmp = *this;
+            const_iterator::operator++();
+            return tmp;
+        }
+
+        HEDLEY_NO_THROW
+        HEDLEY_ALWAYS_INLINE
+        const_iterator & operator--() noexcept
+        {
+            --it_;
+            out_it_ -= outputs_per_leaf;
+            return *this;
+        }
+
+        HEDLEY_NO_THROW
+        const_iterator operator--(int) noexcept
+        {
+            auto tmp = *this;
+            const_iterator::operator--();
+            return tmp;
+        }
+
+        const_iterator & operator+=(std::size_t n) noexcept
+        {
+            it_ += n;
+            out_it_ += outputs_per_leaf*n;
+            return *this;
+        }
+
+        const_iterator operator+(std::size_t n) const noexcept
+        {
+            return const_iterator(out_it_ + outputs_per_leaf*n, it_ + n);
+        }
+
+        const_iterator & operator-=(std::size_t n) noexcept
+        {
+            it_ -= n;
+            out_it_ -= outputs_per_leaf*n;
+            return *this;
+        }
+
+        const_iterator operator-(std::size_t n) const noexcept
+        {
+            return const_iterator(out_it_ - outputs_per_leaf*n, it_ - n);
+        }
+
+        difference_type operator-(const_iterator rhs) const noexcept
+        {
+            return it_ - rhs.it_;
+        }
+
+        reference operator[](std::size_t i) const noexcept
+        {
+            return out_it_[i*outputs_per_leaf + mod(it_[i], lg_outputs_per_leaf)];
+        }
+
+        HEDLEY_NO_THROW
+        HEDLEY_ALWAYS_INLINE
+        constexpr bool operator==(const const_iterator & rhs) const noexcept
+        {
+            return out_it_ == rhs.out_it_ && it_ == rhs.it_;
+        }
+
+        HEDLEY_NO_THROW
+        HEDLEY_ALWAYS_INLINE
+        constexpr bool operator<(const const_iterator & rhs) const noexcept
+        {
+            return out_it_ < rhs.out_it_ && it_ < rhs.it_;
+        }
+
+        HEDLEY_NO_THROW
+        HEDLEY_ALWAYS_INLINE
+        constexpr bool operator!=(const const_iterator & rhs) const noexcept
+        {
+            return !(*this == rhs);
+        }
+
+        HEDLEY_NO_THROW
+        HEDLEY_ALWAYS_INLINE
+        constexpr bool operator>(const const_iterator & rhs) const noexcept
+        {
+            return rhs < *this;
+        }
+
+        HEDLEY_NO_THROW
+        HEDLEY_ALWAYS_INLINE
+        constexpr bool operator<=(const const_iterator & rhs) const noexcept
+        {
+            return !(rhs < *this);
+        }
+
+        HEDLEY_NO_THROW
+        HEDLEY_ALWAYS_INLINE
+        constexpr bool operator>=(const const_iterator & rhs) const noexcept
+        {
+            return !(*this < rhs);
+        }
+
+      private:
+        output_iterator out_it_;
+        subsequence_iterator_type it_;
+    };  // class dpf::subsequence_iterable::const_iterator
+
+  private:
+    const output_iterator out_it_;
+    const points_iterator begin_;
+    const points_iterator end_;
+    const typename std::iterator_traits<points_iterator>::difference_type count_;
+};  // class dpf::subsequence_iterable
+
+template <typename IteratorT>
+class recipe_subsequence_iterable
+{
+  public:
+    using output_type = typename std::iterator_traits<IteratorT>::value_type;
+    using output_iterator = IteratorT;
+    class const_iterator;  // forward declaration
+    using iterator = const_iterator;
+
+    recipe_subsequence_iterable(output_iterator out_it, const std::vector<std::size_t> & indices)
+      : out_it_{out_it}, indices_{indices}
+    {
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    const_iterator begin() const noexcept
+    {
+        return const_iterator(out_it_, std::begin(indices_));
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    const_iterator cbegin() const noexcept
+    {
+        return begin();
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    const_iterator end() const noexcept
+    {
+        return const_iterator(out_it_, std::end(indices_));
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    const_iterator cend() const noexcept
+    {
+        return end();
+    }
+
+    class const_iterator
+    {
+      public:
+        using value_type = output_type;
+        using reference = value_type;
+        using const_reference = reference;
+        using pointer = std::add_pointer_t<reference>;
+        using iterator_category = std::random_access_iterator_tag;
+        using size_type = std::size_t;
+        using difference_type = std::ptrdiff_t;
+
+        using subsequence_iterator_type = typename std::vector<std::size_t>::const_iterator;
+
+        HEDLEY_ALWAYS_INLINE
+        constexpr const_iterator(output_iterator out_it, subsequence_iterator_type it) noexcept
+          : out_it_{out_it}, it_{it}
+        { }
+
+        HEDLEY_ALWAYS_INLINE
+        constexpr const_iterator(const_iterator &&) noexcept = default;
+        HEDLEY_ALWAYS_INLINE
+        constexpr const_iterator(const const_iterator &) noexcept = default;
+
+        const_iterator & operator=(const const_iterator &) = default;
+        const_iterator & operator=(const_iterator &&) noexcept = default;
+        ~const_iterator() = default;
+
+        HEDLEY_PURE
+        HEDLEY_NO_THROW
+        HEDLEY_ALWAYS_INLINE
+        constexpr reference operator*() const noexcept
+        {
+            return out_it_[*it_];
+        }
+
+        HEDLEY_NO_THROW
+        HEDLEY_ALWAYS_INLINE
+        constexpr const_iterator & operator++() noexcept
+        {
+            ++it_;
+            return *this;
+        }
+
+        HEDLEY_NO_THROW
+        const_iterator operator++(int) noexcept
+        {
+            auto tmp = *this;
+            const_iterator::operator++();
+            return tmp;
+        }
+
+        HEDLEY_NO_THROW
+        HEDLEY_ALWAYS_INLINE
+        const_iterator & operator--() noexcept
+        {
+            --it_;
+            return *this;
+        }
+
+        HEDLEY_NO_THROW
+        const_iterator operator--(int) noexcept
+        {
+            auto tmp = *this;
+            const_iterator::operator--();
+            return tmp;
+        }
+
+        const_iterator & operator+=(std::size_t n) noexcept
+        {
+            it_ += n;
+            return *this;
+        }
+
+        const_iterator operator+(std::size_t n) const noexcept
+        {
+            return const_iterator(out_it_, it_ + n);
+        }
+
+        const_iterator & operator-=(std::size_t n) noexcept
+        {
+            it_ -= n;
+            return *this;
+        }
+
+        const_iterator operator-(std::size_t n) const noexcept
+        {
+            return const_iterator(out_it_, it_ - n);
+        }
+
+        difference_type operator-(const_iterator rhs) const noexcept
+        {
+            return it_ - rhs.it_;
+        }
+
+        reference operator[](std::size_t i) const noexcept
+        {
+            return out_it_[it_[i]];
+        }
+
+        HEDLEY_NO_THROW
+        HEDLEY_ALWAYS_INLINE
+        constexpr bool operator==(const const_iterator & rhs) const noexcept
+        {
+            return it_ == rhs.it_;
+        }
+
+        HEDLEY_NO_THROW
+        HEDLEY_ALWAYS_INLINE
+        constexpr bool operator<(const const_iterator & rhs) const noexcept
+        {
+            return it_ < rhs.it_;
+        }
+
+        HEDLEY_NO_THROW
+        HEDLEY_ALWAYS_INLINE
+        constexpr bool operator!=(const const_iterator & rhs) const noexcept
+        {
+            return !(*this == rhs);
+        }
+
+        HEDLEY_NO_THROW
+        HEDLEY_ALWAYS_INLINE
+        constexpr bool operator>(const const_iterator & rhs) const noexcept
+        {
+            return rhs < *this;
+        }
+
+        HEDLEY_NO_THROW
+        HEDLEY_ALWAYS_INLINE
+        constexpr bool operator<=(const const_iterator & rhs) const noexcept
+        {
+            return !(rhs < *this);
+        }
+
+        HEDLEY_NO_THROW
+        HEDLEY_ALWAYS_INLINE
+        constexpr bool operator>=(const const_iterator & rhs) const noexcept
+        {
+            return !(*this < rhs);
+        }
+
+      private:
+        output_iterator out_it_;
+        subsequence_iterator_type it_;
+    };  // class dpf::recipe_subsequence_iterable::const_iterator
+
+  private:
+    const output_iterator out_it_;
+    const std::vector<std::size_t> & indices_;
+};  // class dpf::recipe_subsequence_iterable
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_SUBSEQUENCE_ITERABLE_HPP__

include/dpf/twiddle.hpp

@@ -0,0 +1,156 @@
+/// @file dpf/twiddle.hpp
+/// @brief
+/// @details
+/// @author Ryan Henry <ryan.henry@ucalgary.ca>
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_TWIDDLE_HPP__
+#define LIBDPF_INCLUDE_DPF_TWIDDLE_HPP__
+
+#include <cstddef>
+#include <algorithm>
+#include <iterator>
+#include <array>
+
+#include "hedley/hedley.h"
+#include "simde/simde/x86/avx2.h"
+
+namespace dpf
+{
+
+static constexpr simde__m128i lo_bit128{int64_t{1}, int64_t{0}};
+static constexpr simde__m128i lo_2bits128{int64_t{3}, int64_t{0}};
+
+// if low bit of a is set, return all 1's, else return all 0's
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+HEDLEY_CONST
+simde__m128i mask_from_lo_bit(simde__m128i a) noexcept
+{
+    a = simde_mm_cmpeq_epi32(lo_bit128, simde_mm_and_si128(lo_bit128, a));
+    return simde_mm_shuffle_epi32(a, _MM_SHUFFLE(0, 0, 0, 0));
+}
+
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+HEDLEY_CONST
+uint_fast8_t get_lo_bit(simde__m128i a) noexcept
+{
+    return !simde_mm_testz_si128(simde_mm_and_si128(lo_bit128, a), lo_bit128);
+}
+
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+HEDLEY_CONST
+simde__m128i set_lo_bit(simde__m128i a, uint_fast8_t b = 1) noexcept
+{
+     return b&1 ? simde_mm_or_si128(lo_bit128, a)
+                : simde_mm_andnot_si128(lo_bit128, a);
+}
+
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+HEDLEY_CONST
+simde__m128i unset_lo_bit(simde__m128i a) noexcept
+{
+    return simde_mm_andnot_si128(lo_bit128, a);
+}
+
+// if low bit of b is set, return a, else 0
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+HEDLEY_CONST
+auto get_if_lo_bit(simde__m128i a, simde__m128i b) noexcept
+{
+    return simde_mm_and_si128(a, dpf::mask_from_lo_bit(b));
+}
+
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+template <std::size_t N>
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+HEDLEY_CONST
+auto get_if_lo_bit(std::array<simde__m128i, N> a, simde__m128i b) noexcept
+{
+    auto mask = dpf::mask_from_lo_bit(b);
+    std::transform(std::begin(a), std::end(a), std::begin(a), [mask](simde__m128i & a){return simde_mm_and_si128(a, mask);});
+    return a;
+}
+
+HEDLEY_PRAGMA(GCC diagnostic pop)
+
+// if low bit of c is set, then return xor of a and b, else return a
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+HEDLEY_CONST
+simde__m128i xor_if_lo_bit(simde__m128i a, simde__m128i b, simde__m128i c) noexcept
+{
+    return simde_mm_xor_si128(a, dpf::get_if_lo_bit(b, c));
+}
+
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+HEDLEY_CONST
+uint_fast8_t get_lo_2bits(simde__m128i a) noexcept
+{
+    a = simde_mm_and_si128(a, lo_2bits128);
+    return simde_mm_extract_epi64(a, 0);
+}
+
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+HEDLEY_CONST
+simde__m128i unset_lo_2bits(simde__m128i a) noexcept
+{
+    return simde_mm_andnot_si128(lo_2bits128, a);
+}
+
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+uint_fast8_t get_lo_bit_and_clear_lo_2bits(simde__m128i & a) noexcept  // NOLINT(runtime/references)
+{
+    uint_fast8_t lo_bit = get_lo_bit(a);
+    a = unset_lo_2bits(a);
+    return lo_bit;
+}
+
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+HEDLEY_CONST
+simde__m128i set_lo_2bits(simde__m128i a, uint_fast8_t i) noexcept
+{
+    static constexpr simde__m128i lo2[4] = {
+        {int64_t(0), int64_t(0)},
+        {int64_t(1), int64_t(0)},
+        {int64_t(2), int64_t(0)},
+        {int64_t(3), int64_t(0)}
+    };
+    return simde_mm_or_si128(dpf::unset_lo_2bits(a), lo2[i&3]);
+}
+
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+HEDLEY_CONST
+simde__m128i get_if(simde__m128i c, bool b) noexcept
+{
+    static constexpr simde__m128i mask[] = {
+        {int64_t(0), int64_t(0)},
+        {~int64_t(0), ~int64_t(0)}
+    };
+    return simde_mm_and_si128(c, mask[b]);
+}
+
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+HEDLEY_CONST
+simde__m128i xor_if(simde__m128i a, simde__m128i c, bool b) noexcept
+{
+    return simde_mm_xor_si128(a, get_if(c, b));
+}
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_TWIDDLE_HPP__

include/dpf/twobit.hpp

@@ -0,0 +1,204 @@
+/// @file dpf/twobit.hpp
+/// @brief `dpf::twobit`, a 2-bit output lane in the ring Z/4Z.
+/// @details Values are `0..3`. Scalar `+` and `-` wrap mod 4. A leaf node
+///          packs one lane every two bits, low lane in the low bits of the
+///          first byte, matching `dpf::bit`. Leaf addition is not XOR: a
+///          carry stays inside the 2-bit lane. See `packed_lane_arithmetic.hpp`.
+
+#ifndef LIBDPF_INCLUDE_DPF_TWOBIT_HPP__
+#define LIBDPF_INCLUDE_DPF_TWOBIT_HPP__
+
+#include <cstddef>
+#include <cstdint>
+#include <istream>
+#include <limits>
+#include <ostream>
+#include <stdexcept>
+#include <string>
+#include <type_traits>
+
+#include "hedley/hedley.h"
+
+#include "dpf/utils.hpp"
+
+namespace dpf
+{
+
+/// @brief 2-bit unsigned ring element, packed two bits per lane
+enum class twobit : std::uint8_t
+{
+    zero = 0,
+    one = 1,
+    two = 2,
+    three = 3
+};
+
+HEDLEY_CONST
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+static constexpr dpf::twobit to_twobit(unsigned value) noexcept
+{
+    return static_cast<dpf::twobit>(value & 3u);
+}
+
+HEDLEY_CONST
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+static constexpr dpf::twobit to_twobit(unsigned long long value) noexcept
+{
+    return static_cast<dpf::twobit>(value & 3ull);
+}
+
+/// @brief parse one character as a 2-bit digit
+/// @param zero character for 0 (default `'0'`)
+/// @throws std::domain_error if `value` is not one of the four digits
+template <typename CharT>
+HEDLEY_ALWAYS_INLINE
+static constexpr dpf::twobit to_twobit(CharT value, CharT zero = CharT('0'))
+{
+    auto delta = static_cast<unsigned>(value) - static_cast<unsigned>(zero);
+    if (delta > 3u)
+    {
+        throw std::domain_error("Unrecognized twobit character");
+    }
+    return static_cast<dpf::twobit>(delta);
+}
+
+inline std::string to_string(dpf::twobit value)
+{
+    return std::string(1, static_cast<char>('0' + static_cast<unsigned>(value)));
+}
+
+template <typename CharT, typename Traits>
+std::basic_ostream<CharT, Traits> &
+operator<<(std::basic_ostream<CharT, Traits> & os, dpf::twobit value)
+{
+    return os << static_cast<char>('0' + static_cast<unsigned>(value));
+}
+
+template <typename CharT, typename Traits>
+std::basic_istream<CharT, Traits> &
+operator>>(std::basic_istream<CharT, Traits> & is, dpf::twobit & value)
+{
+    try
+    {
+        value = to_twobit<CharT>(is.get(), is.widen('0'));
+    }
+    catch (const std::exception &)
+    {
+        is.setstate(std::ios::failbit);
+    }
+    return is;
+}
+
+/// @brief addition in Z/4Z
+HEDLEY_CONST
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+constexpr dpf::twobit operator+(dpf::twobit lhs, dpf::twobit rhs) noexcept
+{
+    return static_cast<dpf::twobit>(
+        (static_cast<unsigned>(lhs) + static_cast<unsigned>(rhs)) & 3u);
+}
+
+/// @brief subtraction in Z/4Z
+HEDLEY_CONST
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+constexpr dpf::twobit operator-(dpf::twobit lhs, dpf::twobit rhs) noexcept
+{
+    return static_cast<dpf::twobit>(
+        (static_cast<unsigned>(lhs) - static_cast<unsigned>(rhs)) & 3u);
+}
+
+/// @brief additive inverse in Z/4Z
+HEDLEY_CONST
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+constexpr dpf::twobit operator-(dpf::twobit value) noexcept
+{
+    return static_cast<dpf::twobit>((0u - static_cast<unsigned>(value)) & 3u);
+}
+
+/// @brief multiplication in Z/4Z
+HEDLEY_CONST
+HEDLEY_NO_THROW
+HEDLEY_ALWAYS_INLINE
+constexpr dpf::twobit operator*(dpf::twobit lhs, dpf::twobit rhs) noexcept
+{
+    return static_cast<dpf::twobit>(
+        (static_cast<unsigned>(lhs) * static_cast<unsigned>(rhs)) & 3u);
+}
+
+namespace utils
+{
+
+template <>
+struct bitlength_of<dpf::twobit>
+  : public std::integral_constant<std::size_t, 2> {};
+
+template <typename NodeT>
+struct bitlength_of_output<dpf::twobit, NodeT>
+  : public std::integral_constant<std::size_t, 2> {};
+
+template <>
+struct is_packed_subbyte<dpf::twobit> : std::true_type {};
+
+template <>
+struct packed_lane_bits<dpf::twobit>
+  : public std::integral_constant<std::size_t, 2> {};
+
+template <>
+struct make_from_integral_value<dpf::twobit>
+{
+    using integral_type = std::uint8_t;
+    constexpr dpf::twobit operator()(integral_type val) const noexcept
+    {
+        return dpf::to_twobit(val);
+    }
+};
+
+}  // namespace utils
+
+namespace literals
+{
+namespace twobit
+{
+
+constexpr dpf::twobit operator""_twobit(unsigned long long x)
+{
+    return dpf::to_twobit(x);
+}
+
+}  // namespace twobit
+}  // namespace literals
+
+}  // namespace dpf
+
+namespace std
+{
+
+template <>
+class numeric_limits<dpf::twobit> : public numeric_limits<std::uint8_t>
+{
+  public:
+    static constexpr int digits = 2;
+    static constexpr int digits10 = 0;
+    static constexpr dpf::twobit min() noexcept { return dpf::twobit::zero; }
+    static constexpr dpf::twobit max() noexcept { return dpf::twobit::three; }
+    static constexpr dpf::twobit lowest() noexcept { return min(); }
+};
+
+template <>
+class numeric_limits<dpf::twobit const> : public numeric_limits<dpf::twobit> {};
+
+template <>
+class numeric_limits<dpf::twobit volatile> : public numeric_limits<dpf::twobit> {};
+
+template <>
+class numeric_limits<dpf::twobit const volatile>
+  : public numeric_limits<dpf::twobit> {};
+
+}  // namespace std
+
+#endif  // LIBDPF_INCLUDE_DPF_TWOBIT_HPP__

include/dpf/uint256_t.hpp

@@ -0,0 +1,253 @@
+#ifndef LIBDPF_INCLUDE_DPF_UINT256_T_HPP__
+#define LIBDPF_INCLUDE_DPF_UINT256_T_HPP__
+
+#include "hedley/hedley.h"
+
+#include <type_traits>
+#include <limits>
+
+#include "uint256_t/uint256_t.hpp"
+
+#include "dpf/utils.hpp"
+#include "dpf/leaf_arithmetic.hpp"
+
+
+
+namespace dpf
+{
+
+namespace leaf_arithmetic
+{
+
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+template <> struct add_t<uint128_t, simde__m128i> final
+{
+    auto operator()(const simde__m128i & lhs, const simde__m128i & rhs) const
+    {
+        simde__m128i ret;
+        uint128_t lhs_, rhs_;
+        std::memcpy(&lhs_, &lhs, sizeof(uint128_t));
+        std::memcpy(&rhs_, &rhs, sizeof(uint128_t));
+        uint128_t sum = lhs_ + rhs_;
+        std::memcpy(&ret, &sum, sizeof(simde__m128i));
+        return ret;
+    }
+};
+
+template <> struct add_t<uint128_t, simde__m256i> final
+{
+    auto operator()(const simde__m256i & lhs, const simde__m256i & rhs) const
+    {
+        simde__m256i ret;
+        uint128_t lhs_[2], rhs_[2];
+        std::memcpy(&lhs_, &lhs, sizeof(uint128_t) * 2);
+        std::memcpy(&rhs_, &rhs, sizeof(uint128_t) * 2);
+        uint128_t sum[2] = { lhs_[0] + rhs_[0], lhs_[1] + rhs_[1] };
+        std::memcpy(&ret, &sum, sizeof(simde__m256i));
+        return ret;
+    }
+};
+template <> struct add_t<uint256_t, simde__m256i> final
+{
+    auto operator()(const simde__m256i & lhs, const simde__m256i & rhs) const
+    {
+        simde__m256i ret;
+        uint256_t lhs_, rhs_;
+        std::memcpy(&lhs_, &lhs, sizeof(uint256_t));
+        std::memcpy(&rhs_, &rhs, sizeof(uint256_t));
+        uint256_t sum = lhs_ + rhs_;
+        std::memcpy(&ret, &sum, sizeof(simde__m256i));
+        return ret;
+    }
+};
+template <> struct add_t<uint256_t, std::array<simde__m128i, 2>> final
+{
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    HEDLEY_PURE
+    auto operator()(const std::array<simde__m128i, 2> & a, const std::array<simde__m128i, 2> & b) const noexcept
+    {
+        std::array<simde__m128i, 2> c;
+        uint256_t a_, b_;
+        std::memcpy(&a_, std::data(a), sizeof(uint256_t));
+        std::memcpy(&b_, std::data(b), sizeof(uint256_t));
+        uint256_t c_ = a_ + b_;
+        std::memcpy(std::data(c), &c_, sizeof(std::array<simde__m128i, 2>));
+        return c;
+    }
+};
+
+template <> struct subtract_t<uint128_t, simde__m128i> final
+{
+    auto operator()(const simde__m128i & lhs, const simde__m128i & rhs) const
+    {
+        simde__m128i ret;
+        uint128_t lhs_, rhs_;
+        std::memcpy(&lhs_, &lhs, sizeof(uint128_t));
+        std::memcpy(&rhs_, &rhs, sizeof(uint128_t));
+        uint128_t sum = lhs_ - rhs_;
+        std::memcpy(&ret, &sum, sizeof(simde__m128i));
+        return ret;
+    }
+};
+
+template <> struct subtract_t<uint128_t, simde__m256i> final
+{
+    auto operator()(const simde__m256i & lhs, const simde__m256i & rhs) const
+    {
+        simde__m256i ret;
+        uint128_t lhs_[2], rhs_[2];
+        std::memcpy(&lhs_, &lhs, sizeof(simde_uint128) * 2);
+        std::memcpy(&rhs_, &rhs, sizeof(simde_uint128) * 2);
+        uint128_t sum[2] = { lhs_[0] - rhs_[0], lhs_[1] - rhs_[1] };
+        std::memcpy(&ret, &sum, sizeof(simde__m256i));
+        return ret;
+    }
+};
+template <> struct subtract_t<uint256_t, simde__m256i> final
+{
+    auto operator()(const simde__m256i & lhs, const simde__m256i & rhs) const
+    {
+        simde__m256i ret;
+        uint256_t lhs_, rhs_;
+        std::memcpy(&lhs_, &lhs, sizeof(uint256_t));
+        std::memcpy(&rhs_, &rhs, sizeof(uint256_t));
+        uint256_t sum = lhs_ - rhs_;
+        std::memcpy(&ret, &sum, sizeof(simde__m256i));
+        return ret;
+    }
+};
+template <> struct subtract_t<uint256_t, std::array<simde__m128i, 2>> final
+{
+    auto operator()(const std::array<simde__m128i, 2> & a, const std::array<simde__m128i, 2> & b) const
+    {
+        std::array<simde__m128i, 2> c;
+        uint256_t a_, b_;
+        std::memcpy(&a_, std::data(a), sizeof(uint256_t));
+        std::memcpy(&b_, std::data(b), sizeof(uint256_t));
+        uint256_t c_ = a_ - b_;
+        std::memcpy(std::data(c), &c_, sizeof(std::array<simde__m128i, 2>));
+        return c;
+    }
+};
+
+template <> struct multiply_t<uint128_t, simde__m128i> final
+{
+    auto operator()(const simde__m128i & a, uint128_t b) const
+    {
+        uint128_t a_;
+        simde__m128i c;
+        std::memcpy(&a_, &a, sizeof(uint128_t));
+        uint128_t c_ = a_ * b;
+        std::memcpy(&c, &c_, sizeof(simde__m128i));
+        return c;
+    }
+};
+
+template <> struct multiply_t<uint256_t, std::array<simde__m128i, 2>> final
+{
+    auto operator()(const std::array<simde__m128i, 2> & a, uint256_t b) const
+    {
+        uint256_t a_;
+        std::memcpy(&a_, &a, sizeof(uint256_t));
+        uint256_t c_ = a_ * b;
+        std::array<simde__m128i, 2> c;
+        std::memcpy(&c, &c_, sizeof(std::array<simde__m128i, 2>));
+        return c;
+    }
+};
+
+template <> struct multiply_t<uint128_t, simde__m256i> final
+{
+    auto operator()(const simde__m256i & a, uint128_t b) const
+    {
+        uint256_t a_;
+        simde__m256i c;
+        std::memcpy(&a_, &a, sizeof(uint256_t));
+        uint256_t c_{a_.upper() * b, a_.lower() * b};
+        std::memcpy(&c, &c_, sizeof(simde__m256i));
+        return c;
+    }
+};
+
+template <> struct multiply_t<uint256_t, simde__m256i> final
+{
+    auto operator()(const simde__m256i & a, uint256_t b) const
+    {
+        uint256_t a_;
+        simde__m256i c;
+        std::memcpy(&a_, &a, sizeof(uint256_t));
+        uint256_t c_ = a_ * b;
+        std::memcpy(&c, &c_, sizeof(simde__m256i));
+        return c;
+    }
+};
+HEDLEY_PRAGMA(GCC diagnostic pop)
+
+}  // namespace leaf_arithmetic
+
+namespace utils
+{
+
+template <>
+struct msb_of<uint128_t>
+{
+    constexpr static uint128_t value{1ul << 63, 0ul};
+};
+
+template <>
+struct msb_of<uint256_t>
+{
+    constexpr static uint256_t value{uint128_t{1ul << 63, 0ul}, uint128_t{0ul, 0ul}};
+};
+
+template <>
+struct mod_pow_2<uint128_t>
+{
+    std::size_t operator()(uint128_t val, std::size_t n) const noexcept
+    {
+        return mod_pow_2<uint64_t>{}(static_cast<uint64_t>(val.lower()), n);
+    }
+};
+
+template <>
+struct mod_pow_2<uint256_t>
+{
+    std::size_t operator()(uint256_t val, std::size_t n) const noexcept
+    {
+        return mod_pow_2<uint128_t>{}(val.lower(), n);
+    }
+};
+
+template <>
+struct to_integral_type<uint128_t>
+    : to_integral_type_base<uint128_t>
+{
+    using parent = to_integral_type_base<uint128_t>;
+    using typename parent::integral_type;
+
+    constexpr integral_type operator()(uint128_t val) const noexcept
+    {
+        return (simde_uint128(val.upper()) << 64) | simde_uint128(val.lower());
+    }
+};
+
+template <>
+struct to_integral_type<uint256_t>
+    : to_integral_type_base<uint256_t>
+{
+    using parent = to_integral_type_base<uint256_t>;
+    using typename parent::integral_type;
+
+    constexpr integral_type operator()(uint256_t val) const noexcept
+    {
+        return val;
+    }
+};
+
+}  // namespace dpf::utils
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_UINT256_T_HPP__

include/dpf/wildcard.hpp

@@ -0,0 +1,277 @@
+/// @file dpf/wildcard.hpp
+/// @brief defines the `dpf::wildcard_value` template and associated helpers
+/// @details A `dpf::wildcard` is a struct template with a single parameter
+///          `T`, which must be a trivially copyable type (as indicated by
+///          `std::is_trivially_copyable<T>`). It is used as a placeholder
+///          for an instance of type `T`, which can be assigned later. Its
+///          intended to wrap an [output type](@ref output_types) of a DPF.
+/// @author Ryan Henry <ryan.henry@ucalgary.ca>
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_WILDCARD_HPP__
+#define LIBDPF_INCLUDE_DPF_WILDCARD_HPP__
+
+#include "hedley/hedley.h"
+
+#include <type_traits>
+#include <limits>
+#include <array>
+#include <optional>
+
+#include "dpf/bit.hpp"
+#include "dpf/bitstring.hpp"
+#include "dpf/random.hpp"
+#include "dpf/xor_wrapper.hpp"
+
+#include "simde/simde/x86/avx2.h"
+// #include "simde/simde/x86/avx512.h"
+
+namespace dpf
+{
+
+// namespace
+// {
+
+/// @brief represents a placeholder value of a given type, with a concrete value to be assigned later
+/// @tparam T the underlying type
+template <typename T>
+struct wildcard_value
+{
+    static_assert(std::is_trivially_copyable_v<T>,
+        "T must be a trivially copyable type");
+    static_assert(std::numeric_limits<T>::is_iec559 ||
+        !(std::is_same_v<T, float> || std::is_same_v<T, double>),
+        "floating point types only supported for iec559");
+    inline constexpr wildcard_value() noexcept : val{std::nullopt} { }
+    inline constexpr wildcard_value(const T & t) noexcept : val{t} { }
+    inline constexpr wildcard_value(T && t) noexcept : val{std::move(t)} { }
+
+    HEDLEY_ALWAYS_INLINE
+    constexpr auto operator()(const T & t) const
+    {
+        return wildcard_value(t);
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    constexpr auto operator()(T && t) const
+    {
+        return wildcard_value(std::move(t));
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    auto operator()() const
+    {
+        auto t = val.value_or(dpf::uniform_sample<T>());
+        return std::tuple_cat(std::make_tuple(t), dpf::additively_share(t));
+    }
+
+  private:
+    std::optional<T> val;
+};
+
+// }  // namespace
+
+template <typename T> static constexpr wildcard_value<T> wildcard{};
+
+/// @brief Checks whether `T` is a wildcard type.
+/// @details A trait class that provides the member constant `value` which is
+///          equal to `true`, if `T` is a specialization of the `wildcard_value`
+///          template and `false` otherwise.
+/// @see dpf::is_wildcard_v
+template <typename T> struct is_wildcard : std::false_type { };
+template <typename T> struct is_wildcard<wildcard_value<T>> : std::true_type { };
+
+/// @brief Checks whether `T` is a wildcard type.
+template <typename T> constexpr bool is_wildcard_v = is_wildcard<T>::value;
+
+template <typename T> struct concrete_type { using type = T; };
+template <typename T> struct concrete_type<wildcard_value<T>>
+    : public concrete_type<T> { };
+template <typename T> using concrete_type_t = typename concrete_type<T>::type;
+
+template <typename T> struct concrete_value
+{
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_CONST
+    constexpr std::optional<T> operator()(T y) const noexcept
+    {
+        return y;
+    }
+};
+template <typename T> struct concrete_value<wildcard_value<T>>
+{
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_CONST
+    constexpr std::optional<T> operator()(wildcard_value<T>) const noexcept
+    {
+        return std::nullopt;
+    }
+};
+template <typename T> constexpr auto concrete_value_v = concrete_value<T>{};
+
+namespace wildcards
+{
+    using bit_t = wildcard_value<dpf::bit>;
+    static constexpr auto bit = wildcard<dpf::bit>;
+
+    using signed_char_t = wildcard_value<signed char>;
+    static constexpr auto signed_char = wildcard<signed char>;
+    using uchar_t = wildcard_value<unsigned char>;
+    static constexpr auto uchar = wildcard<unsigned char>;
+    using xchar_t = wildcard_value<dpf::xor_wrapper<unsigned char>>;
+    static constexpr auto xchar = wildcard<dpf::xor_wrapper<unsigned char>>;
+
+    using int8_t = wildcard_value<psnip_int8_t>;
+    static constexpr auto int8 = wildcard<psnip_int8_t>;
+    using uint8_t = wildcard_value<psnip_uint8_t>;
+    static constexpr auto uint8 = wildcard<psnip_uint8_t>;
+    using xint8_t = wildcard_value<dpf::xints::xint8_t>;
+    static constexpr auto xint8 = wildcard<dpf::xints::xint8_t>;
+
+    using int16_t = wildcard_value<psnip_int16_t>;
+    static constexpr auto int16 = wildcard<psnip_int16_t>;
+    using uint16_t = wildcard_value<psnip_uint16_t>;
+    static constexpr auto uint16 = wildcard<psnip_uint16_t>;
+    using xint16_t = wildcard_value<dpf::xints::xint16_t>;
+    static constexpr auto xint16 = wildcard<dpf::xints::xint16_t>;
+
+    using int32_t = wildcard_value<psnip_int32_t>;
+    static constexpr auto int32 = wildcard<psnip_int32_t>;
+    using uint32_t = wildcard_value<psnip_uint32_t>;
+    static constexpr auto uint32 = wildcard<psnip_uint32_t>;
+    using xint32_t = wildcard_value<dpf::xints::xint32_t>;
+    static constexpr auto xint32 = wildcard<dpf::xints::xint32_t>;
+
+    using int64_t = wildcard_value<psnip_int64_t>;
+    static constexpr auto int64 = wildcard<psnip_int64_t>;
+    using uint64_t = wildcard_value<psnip_uint64_t>;
+    static constexpr auto uint64 = wildcard<psnip_uint64_t>;
+    using xint64_t = wildcard_value<dpf::xints::xint64_t>;
+    static constexpr auto xint64 = wildcard<dpf::xints::xint64_t>;
+
+HEDLEY_PRAGMA(GCC diagnostic push)
+HEDLEY_PRAGMA(GCC diagnostic ignored "-Wignored-attributes")
+    using __int128_t = wildcard_value<simde_int128>;  //< gcc builtin `__int128` type
+    static constexpr auto _int128 = wildcard<simde_int128>;
+    using __uint128_t = wildcard_value<simde_uint128>;  //< gcc builtin `unsigned __int128` type
+    static constexpr auto _uint128 = wildcard<simde_uint128>;
+    using uint128_t = wildcard_value<::uint128_t>;  //< `uint128_t` type
+    static constexpr auto uint128 = wildcard<::uint128_t>;
+    using xint128_t = wildcard_value<dpf::xints::xint128_t>;
+    static constexpr auto xint128 = wildcard<dpf::xints::xint128_t>;
+
+    using uint256_t = wildcard_value<::uint256_t>;  //< `uint256_t` type
+    static constexpr auto uint256 = wildcard<::uint256_t>;
+    using xint256_t = wildcard_value<dpf::xints::xint256_t>;
+    static constexpr auto xint256 = wildcard<dpf::xints::xint256_t>;
+
+    template <std::size_t Nbits> using bitstring_t = wildcard_value<dpf::bitstring<Nbits>>;
+    template <std::size_t Nbits> static constexpr auto bitstring = wildcard<dpf::bitstring<Nbits>>;
+
+    template <std::size_t Nbits> using xint_t = wildcard_value<dpf::xint<Nbits>>;
+    template <std::size_t Nbits> static constexpr auto xint = wildcard<dpf::xint<Nbits>>;
+    
+    template <std::size_t Nbits> using modint_t = wildcard_value<dpf::modint<Nbits>>;
+    template <std::size_t Nbits> static constexpr auto modint = wildcard<dpf::modint<Nbits>>;
+
+    using m128_t = wildcard_value<simde__m128>;
+    static constexpr auto m128 = wildcard<simde__m128>;
+    using m128i_t = wildcard_value<simde__m128i>;
+    static constexpr auto m128i = wildcard<simde__m128i>;
+    using m128d_t = wildcard_value<simde__m128d>;
+    static constexpr auto m128d = wildcard<simde__m128d>;
+
+    using m256_t = wildcard_value<simde__m256>;
+    static constexpr auto m256 = wildcard<simde__m256>;
+    using m256i_t = wildcard_value<simde__m256i>;
+    static constexpr auto m256i = wildcard<simde__m256i>;
+    using m256d_t = wildcard_value<simde__m256d>;
+    static constexpr auto m256d = wildcard<simde__m256d>;
+
+    // using m512_t = wildcard_value<simde__m512>;
+    // static constexpr auto m512 = wildcard<simde__m512>;
+    // using m512i_t = wildcard_value<simde__m512i>;
+    // static constexpr auto m512i = wildcard<simde__m512i>;
+    // using m512d_t = wildcard_value<simde__m512d>;
+    // static constexpr auto m512d = wildcard<simde__m512d>;
+HEDLEY_PRAGMA(GCC diagnostic pop)
+
+    using ieee_float_t = wildcard_value<float>;
+    /// @brief Placeholder for a `float` whose leaf group is bitwise XOR
+    ///        (exact reconstruction), not IEEE addition.
+    static constexpr auto ieee_float = wildcard<float>;
+    using ieee_double_t = wildcard_value<double>;
+    /// @brief Placeholder for a `double` whose leaf group is bitwise XOR.
+    static constexpr auto ieee_double = wildcard<double>;
+
+}  // namespace wildcards
+
+template <std::size_t I,
+          typename DpfKey>
+void assert_wildcard_output(const DpfKey & dpf)
+{
+    if (HEDLEY_UNLIKELY(std::get<I>(dpf.leaves).is_ready()))
+    {
+        throw std::runtime_error("output not an unassigned wildcard");
+    }
+}
+
+template <typename DpfKey>
+void assert_wildcard_input(const DpfKey & dpf)
+{
+    if (HEDLEY_UNLIKELY(dpf.offset_x.is_ready()))
+    {
+        throw std::runtime_error("input is not an unassigned wildcard");
+    }
+}
+
+template <std::size_t ...Is,
+          typename DpfKey>
+HEDLEY_ALWAYS_INLINE
+void assert_not_wildcard_output(const DpfKey & dpf)
+{
+    if (HEDLEY_UNLIKELY(!std::get<Is>(dpf.leaf_nodes).is_ready() || ...))
+    {
+        throw std::runtime_error("one or more outputs is an unassigned wildcard");
+    }
+}
+
+template <typename DpfKey>
+HEDLEY_ALWAYS_INLINE
+void assert_not_wildcard_input(const DpfKey & dpf)
+{
+    if (HEDLEY_UNLIKELY(!dpf.offset_x.is_ready()))
+    {
+        throw std::runtime_error("input is unassigned wildcard");
+    }
+}
+
+namespace utils
+{
+
+/// @brief specializes `dpf::utils::bitlength_of` for `dpf::wildcard_value`
+template <typename T>
+struct bitlength_of<wildcard_value<T>>
+  : public bitlength_of<T>
+{ };
+
+/// @brief specializes `dpf::utils::bitlength_of_output` for `dpf::wildcard_value`
+template <typename T,
+          typename NodeT>
+struct bitlength_of_output<wildcard_value<T>, NodeT>
+  : public bitlength_of_output<T, NodeT>
+{ };
+
+template <typename T>
+struct make_default<wildcard_value<T>>
+{
+    static constexpr wildcard_value<T> value = wildcard_value<T>();
+};
+
+}  // namespace dpf::utils
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_WILDCARD_HPP__

include/dpf/xor_wrapper.hpp

@@ -0,0 +1,977 @@
+/// @file dpf/xor_wrapper.hpp
+/// @brief defines the `xor_wrapper` class and associated helpers
+/// @details A `xor_wrapper` is a struct template that adapts integral
+///          types to use bitwise arithmetic; that is, it makes an `N`-bit
+///          integer type behave as it it were an element of `GF(2)^N`.
+///          Specifically,
+/// @author Ryan Henry <ryan.henry@ucalgary.ca>
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_XOR_WRAPPER_HPP__
+#define LIBDPF_INCLUDE_DPF_XOR_WRAPPER_HPP__
+
+#include "hedley/hedley.h"
+
+#include <cstddef>
+#include <type_traits>
+#include <functional>
+#include <limits>
+#include <ostream>
+#include <istream>
+
+#include "simde/simde/x86/avx2.h"
+#include "portable-snippets/exact-int/exact-int.h"
+
+#include "dpf/modint.hpp"
+#include "dpf/literals.hpp"
+
+namespace dpf
+{
+
+template <typename T>
+struct xor_wrapper
+{
+  public:
+    using value_type = utils::make_unsigned_t<T>;
+    static constexpr auto bit_xor = std::bit_xor<value_type>{};
+    static constexpr auto bit_and = std::bit_and<value_type>{};
+    static constexpr auto bit_or = std::bit_or<value_type>{};
+    static constexpr auto bit_not = std::bit_not<value_type>{};
+
+    static constexpr std::size_t bits = utils::bitlength_of_v<value_type>;
+    using integral_type = utils::integral_type_from_bitlength_t<bits>;
+
+    /// @{
+
+    /// @brief Default c'tor
+    constexpr xor_wrapper() = default;
+
+    /// @brief Copy c'tor
+    constexpr xor_wrapper(const xor_wrapper &) noexcept = default;
+
+    /// @brief Move c'tor
+    constexpr xor_wrapper(xor_wrapper &&) noexcept = default;
+
+    /// @brief Value c'tor
+    // cppcheck-suppress noExplicitConstructor
+    constexpr xor_wrapper(value_type v) noexcept : value{v} { }  // NOLINT(runtime/explicit)
+
+    /// @}
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    constexpr xor_wrapper & operator=(const xor_wrapper &) noexcept = default;
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    constexpr xor_wrapper & operator=(xor_wrapper &&) noexcept = default;
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    constexpr xor_wrapper & operator=(value_type v) noexcept
+    {
+        value = v;
+        return *this;
+    }
+
+    ~xor_wrapper() = default;
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    HEDLEY_PURE
+    constexpr xor_wrapper operator-() const noexcept
+    {
+        return xor_wrapper{value};
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    HEDLEY_PURE
+    constexpr operator bool() const noexcept
+    {
+        return static_cast<bool>(value);
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    HEDLEY_PURE
+    constexpr operator T() const noexcept {
+        if constexpr (std::is_same_v<T, value_type> == true)
+        {
+            return value;
+        }
+        return static_cast<T>(value);
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    HEDLEY_PURE
+    constexpr bool operator==(xor_wrapper rhs) const noexcept
+    {
+        return value == rhs.value;
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    HEDLEY_PURE
+    constexpr bool operator!=(xor_wrapper rhs) const noexcept
+    {
+        return value != rhs.value;
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    HEDLEY_PURE
+    constexpr bool operator<(xor_wrapper rhs) const noexcept
+    {
+        return value < rhs.value;
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    HEDLEY_PURE
+    constexpr bool operator<=(xor_wrapper rhs) const noexcept
+    {
+        return value <= rhs.value;
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    HEDLEY_PURE
+    constexpr bool operator>(xor_wrapper rhs) const noexcept
+    {
+        return value > rhs.value;
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    HEDLEY_PURE
+    constexpr bool operator>=(xor_wrapper rhs) const noexcept
+    {
+        return value >= rhs.value;
+    }
+
+    HEDLEY_PURE
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr value_type data() const noexcept
+    {
+        return value;
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    constexpr xor_wrapper & operator<<=(std::size_t amount)
+    {
+        this->value <<= amount;
+        return *this;
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    constexpr xor_wrapper & operator>>=(std::size_t amount)
+    {
+        this->value >>= amount;
+        return *this;
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr xor_wrapper & operator++() noexcept
+    {
+        // Characteristic 2: adding the multiplicative/additive unit bit is XOR 1,
+        // which is `*this + xor_wrapper{1}`, not an integer increment.
+        value = bit_xor(value, value_type{1});
+        return *this;
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr xor_wrapper operator++(int) noexcept
+    {
+        auto ret = *this;
+        this->operator++();
+        return ret;
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr xor_wrapper & operator--() noexcept
+    {
+        // In characteristic 2, subtracting 1 is the same XOR.
+        value = bit_xor(value, value_type{1});
+        return *this;
+    }
+
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr xor_wrapper operator--(int) noexcept
+    {
+        auto ret = *this;
+        this->operator--();
+        return ret;
+    }
+
+  private:
+    value_type value;
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    friend constexpr xor_wrapper operator+(const xor_wrapper & lhs,
+        const xor_wrapper & rhs) noexcept
+    {
+        return xor_wrapper(xor_wrapper::bit_xor(lhs.value, rhs.value));
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    friend constexpr xor_wrapper operator-(const xor_wrapper & lhs,
+        const xor_wrapper & rhs) noexcept
+    {
+        return xor_wrapper(xor_wrapper::bit_xor(lhs.value, rhs.value));
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    friend constexpr xor_wrapper operator*(const xor_wrapper & lhs,
+        const xor_wrapper & rhs) noexcept
+    {
+        return xor_wrapper(xor_wrapper::bit_and(lhs.value, rhs.value));
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    friend constexpr xor_wrapper operator&(const xor_wrapper & lhs,
+        const xor_wrapper & rhs) noexcept
+    {
+        return xor_wrapper(xor_wrapper::bit_and(lhs.value, rhs.value));
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    friend constexpr xor_wrapper operator|(const xor_wrapper & lhs,
+        const xor_wrapper & rhs) noexcept
+    {
+        return xor_wrapper(xor_wrapper::bit_or(lhs.value, rhs.value));
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    friend constexpr xor_wrapper operator^(const xor_wrapper & lhs,
+        const xor_wrapper & rhs) noexcept
+    {
+        return xor_wrapper(xor_wrapper::bit_xor(lhs.value, rhs.value));
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    friend constexpr xor_wrapper operator~(const xor_wrapper & val) noexcept
+    {
+        return xor_wrapper(xor_wrapper::bit_not(val.value));
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    friend constexpr xor_wrapper operator<<(const xor_wrapper & val, std::size_t amount)
+    {
+        return xor_wrapper(val.value << amount);
+    }
+
+    HEDLEY_ALWAYS_INLINE
+    HEDLEY_NO_THROW
+    friend constexpr xor_wrapper operator>>(const xor_wrapper & val, std::size_t amount)
+    {
+        return xor_wrapper(val.value >> amount);
+    }
+
+    friend std::ostream & operator<<(std::ostream & os, xor_wrapper<T> val)
+    {
+        return os << val.value;
+    }
+
+    friend std::istream & operator>>(std::istream & is, xor_wrapper<T> & val)
+    {
+        return is >> val.value;
+    }
+
+    friend struct utils::to_integral_type<xor_wrapper>;
+    friend struct utils::mod_pow_2<xor_wrapper>;
+};
+
+template <std::size_t Nbits>
+using xint = xor_wrapper<dpf::modint<Nbits>>;
+
+namespace xints
+{
+
+// 1--9
+using xint1_t = xint<1>;
+using xint2_t = xint<2>;
+using xint3_t = xint<3>;
+using xint4_t = xint<4>;
+using xint5_t = xint<5>;
+using xint6_t = xint<6>;
+using xint7_t = xint<7>;
+using xint8_t = xint<8>;
+using xint9_t = xint<9>;
+// 10--19
+using xint10_t = xint<10>;
+using xint11_t = xint<11>;
+using xint12_t = xint<12>;
+using xint13_t = xint<13>;
+using xint14_t = xint<14>;
+using xint15_t = xint<15>;
+using xint16_t = xint<16>;
+using xint17_t = xint<17>;
+using xint18_t = xint<18>;
+using xint19_t = xint<19>;
+// 20--29
+using xint20_t = xint<20>;
+using xint21_t = xint<21>;
+using xint22_t = xint<22>;
+using xint23_t = xint<23>;
+using xint24_t = xint<24>;
+using xint25_t = xint<25>;
+using xint26_t = xint<26>;
+using xint27_t = xint<27>;
+using xint28_t = xint<28>;
+using xint29_t = xint<29>;
+// 30--39
+using xint30_t = xint<30>;
+using xint31_t = xint<31>;
+using xint32_t = xint<32>;
+using xint33_t = xint<33>;
+using xint34_t = xint<34>;
+using xint35_t = xint<35>;
+using xint36_t = xint<36>;
+using xint37_t = xint<37>;
+using xint38_t = xint<38>;
+using xint39_t = xint<39>;
+// 40--49
+using xint40_t = xint<40>;
+using xint41_t = xint<41>;
+using xint42_t = xint<42>;
+using xint43_t = xint<43>;
+using xint44_t = xint<44>;
+using xint45_t = xint<45>;
+using xint46_t = xint<46>;
+using xint47_t = xint<47>;
+using xint48_t = xint<48>;
+using xint49_t = xint<49>;
+// 50--59
+using xint50_t = xint<50>;
+using xint51_t = xint<51>;
+using xint52_t = xint<52>;
+using xint53_t = xint<53>;
+using xint54_t = xint<54>;
+using xint55_t = xint<55>;
+using xint56_t = xint<56>;
+using xint57_t = xint<57>;
+using xint58_t = xint<58>;
+using xint59_t = xint<59>;
+// 60--69
+using xint60_t = xint<60>;
+using xint61_t = xint<61>;
+using xint62_t = xint<62>;
+using xint63_t = xint<63>;
+using xint64_t = xint<64>;
+using xint65_t = xint<65>;
+using xint66_t = xint<66>;
+using xint67_t = xint<67>;
+using xint68_t = xint<68>;
+using xint69_t = xint<69>;
+// 70--79
+using xint70_t = xint<70>;
+using xint71_t = xint<71>;
+using xint72_t = xint<72>;
+using xint73_t = xint<73>;
+using xint74_t = xint<74>;
+using xint75_t = xint<75>;
+using xint76_t = xint<76>;
+using xint77_t = xint<77>;
+using xint78_t = xint<78>;
+using xint79_t = xint<79>;
+// 80--89
+using xint80_t = xint<80>;
+using xint81_t = xint<81>;
+using xint82_t = xint<82>;
+using xint83_t = xint<83>;
+using xint84_t = xint<84>;
+using xint85_t = xint<85>;
+using xint86_t = xint<86>;
+using xint87_t = xint<87>;
+using xint88_t = xint<88>;
+using xint89_t = xint<89>;
+// 90--99
+using xint90_t = xint<90>;
+using xint91_t = xint<91>;
+using xint92_t = xint<92>;
+using xint93_t = xint<93>;
+using xint94_t = xint<94>;
+using xint95_t = xint<95>;
+using xint96_t = xint<96>;
+using xint97_t = xint<97>;
+using xint98_t = xint<98>;
+using xint99_t = xint<99>;
+// 100--109
+using xint100_t = xint<100>;
+using xint101_t = xint<101>;
+using xint102_t = xint<102>;
+using xint103_t = xint<103>;
+using xint104_t = xint<104>;
+using xint105_t = xint<105>;
+using xint106_t = xint<106>;
+using xint107_t = xint<107>;
+using xint108_t = xint<108>;
+using xint109_t = xint<109>;
+// 110--119
+using xint110_t = xint<110>;
+using xint111_t = xint<111>;
+using xint112_t = xint<112>;
+using xint113_t = xint<113>;
+using xint114_t = xint<114>;
+using xint115_t = xint<115>;
+using xint116_t = xint<116>;
+using xint117_t = xint<117>;
+using xint118_t = xint<118>;
+using xint119_t = xint<119>;
+// 120--129
+using xint120_t = xint<120>;
+using xint121_t = xint<121>;
+using xint122_t = xint<122>;
+using xint123_t = xint<123>;
+using xint124_t = xint<124>;
+using xint125_t = xint<125>;
+using xint126_t = xint<126>;
+using xint127_t = xint<127>;
+using xint128_t = xint<128>;
+using xint129_t = xint<129>;
+// 130--139
+using xint130_t = xint<130>;
+using xint131_t = xint<131>;
+using xint132_t = xint<132>;
+using xint133_t = xint<133>;
+using xint134_t = xint<134>;
+using xint135_t = xint<135>;
+using xint136_t = xint<136>;
+using xint137_t = xint<137>;
+using xint138_t = xint<138>;
+using xint139_t = xint<139>;
+// 140--149
+using xint140_t = xint<140>;
+using xint141_t = xint<141>;
+using xint142_t = xint<142>;
+using xint143_t = xint<143>;
+using xint144_t = xint<144>;
+using xint145_t = xint<145>;
+using xint146_t = xint<146>;
+using xint147_t = xint<147>;
+using xint148_t = xint<148>;
+using xint149_t = xint<149>;
+// 150--159
+using xint150_t = xint<150>;
+using xint151_t = xint<151>;
+using xint152_t = xint<152>;
+using xint153_t = xint<153>;
+using xint154_t = xint<154>;
+using xint155_t = xint<155>;
+using xint156_t = xint<156>;
+using xint157_t = xint<157>;
+using xint158_t = xint<158>;
+using xint159_t = xint<159>;
+// 160--169
+using xint160_t = xint<160>;
+using xint161_t = xint<161>;
+using xint162_t = xint<162>;
+using xint163_t = xint<163>;
+using xint164_t = xint<164>;
+using xint165_t = xint<165>;
+using xint166_t = xint<166>;
+using xint167_t = xint<167>;
+using xint168_t = xint<168>;
+using xint169_t = xint<169>;
+// 170--179
+using xint170_t = xint<170>;
+using xint171_t = xint<171>;
+using xint172_t = xint<172>;
+using xint173_t = xint<173>;
+using xint174_t = xint<174>;
+using xint175_t = xint<175>;
+using xint176_t = xint<176>;
+using xint177_t = xint<177>;
+using xint178_t = xint<178>;
+using xint179_t = xint<179>;
+// 180--189
+using xint180_t = xint<180>;
+using xint181_t = xint<181>;
+using xint182_t = xint<182>;
+using xint183_t = xint<183>;
+using xint184_t = xint<184>;
+using xint185_t = xint<185>;
+using xint186_t = xint<186>;
+using xint187_t = xint<187>;
+using xint188_t = xint<188>;
+using xint189_t = xint<189>;
+// 190--199
+using xint190_t = xint<190>;
+using xint191_t = xint<191>;
+using xint192_t = xint<192>;
+using xint193_t = xint<193>;
+using xint194_t = xint<194>;
+using xint195_t = xint<195>;
+using xint196_t = xint<196>;
+using xint197_t = xint<197>;
+using xint198_t = xint<198>;
+using xint199_t = xint<199>;
+// 200--209
+using xint200_t = xint<200>;
+using xint201_t = xint<201>;
+using xint202_t = xint<202>;
+using xint203_t = xint<203>;
+using xint204_t = xint<204>;
+using xint205_t = xint<205>;
+using xint206_t = xint<206>;
+using xint207_t = xint<207>;
+using xint208_t = xint<208>;
+using xint209_t = xint<209>;
+// 210--219
+using xint210_t = xint<210>;
+using xint211_t = xint<211>;
+using xint212_t = xint<212>;
+using xint213_t = xint<213>;
+using xint214_t = xint<214>;
+using xint215_t = xint<215>;
+using xint216_t = xint<216>;
+using xint217_t = xint<217>;
+using xint218_t = xint<218>;
+using xint219_t = xint<219>;
+// 220--229
+using xint220_t = xint<220>;
+using xint221_t = xint<221>;
+using xint222_t = xint<222>;
+using xint223_t = xint<223>;
+using xint224_t = xint<224>;
+using xint225_t = xint<225>;
+using xint226_t = xint<226>;
+using xint227_t = xint<227>;
+using xint228_t = xint<228>;
+using xint229_t = xint<229>;
+// 230--239
+using xint230_t = xint<230>;
+using xint231_t = xint<231>;
+using xint232_t = xint<232>;
+using xint233_t = xint<233>;
+using xint234_t = xint<234>;
+using xint235_t = xint<235>;
+using xint236_t = xint<236>;
+using xint237_t = xint<237>;
+using xint238_t = xint<238>;
+using xint239_t = xint<239>;
+// 240--249
+using xint240_t = xint<240>;
+using xint241_t = xint<241>;
+using xint242_t = xint<242>;
+using xint243_t = xint<243>;
+using xint244_t = xint<244>;
+using xint245_t = xint<245>;
+using xint246_t = xint<246>;
+using xint247_t = xint<247>;
+using xint248_t = xint<248>;
+using xint249_t = xint<249>;
+// 250--256
+using xint250_t = xint<250>;
+using xint251_t = xint<251>;
+using xint252_t = xint<252>;
+using xint253_t = xint<253>;
+using xint254_t = xint<254>;
+using xint255_t = xint<255>;
+using xint256_t = xint<256>;
+
+namespace literals = dpf::literals::xints;
+
+}  // namespace xints
+
+namespace literals
+{
+
+namespace xints
+{
+
+// 1--9
+constexpr static auto operator "" _x1(unsigned long long int x) { return dpf::xints::xint1_t{static_cast<psnip_uint8_t>(x)}; }
+constexpr static auto operator "" _x2(unsigned long long int x) { return dpf::xints::xint2_t{static_cast<psnip_uint8_t>(x)}; }
+constexpr static auto operator "" _x3(unsigned long long int x) { return dpf::xints::xint3_t{static_cast<psnip_uint8_t>(x)}; }
+constexpr static auto operator "" _x4(unsigned long long int x) { return dpf::xints::xint4_t{static_cast<psnip_uint8_t>(x)}; }
+constexpr static auto operator "" _x5(unsigned long long int x) { return dpf::xints::xint5_t{static_cast<psnip_uint8_t>(x)}; }
+constexpr static auto operator "" _x6(unsigned long long int x) { return dpf::xints::xint6_t{static_cast<psnip_uint8_t>(x)}; }
+constexpr static auto operator "" _x7(unsigned long long int x) { return dpf::xints::xint7_t{static_cast<psnip_uint8_t>(x)}; }
+constexpr static auto operator "" _x8(unsigned long long int x) { return dpf::xints::xint8_t{static_cast<psnip_uint8_t>(x)}; }
+constexpr static auto operator "" _x9(unsigned long long int x) { return dpf::xints::xint9_t{static_cast<psnip_uint16_t>(x)}; }
+// 10--19
+constexpr static auto operator "" _x10(unsigned long long int x) { return dpf::xints::xint10_t{static_cast<psnip_uint16_t>(x)}; }
+constexpr static auto operator "" _x11(unsigned long long int x) { return dpf::xints::xint11_t{static_cast<psnip_uint16_t>(x)}; }
+constexpr static auto operator "" _x12(unsigned long long int x) { return dpf::xints::xint12_t{static_cast<psnip_uint16_t>(x)}; }
+constexpr static auto operator "" _x13(unsigned long long int x) { return dpf::xints::xint13_t{static_cast<psnip_uint16_t>(x)}; }
+constexpr static auto operator "" _x14(unsigned long long int x) { return dpf::xints::xint14_t{static_cast<psnip_uint16_t>(x)}; }
+constexpr static auto operator "" _x15(unsigned long long int x) { return dpf::xints::xint15_t{static_cast<psnip_uint16_t>(x)}; }
+constexpr static auto operator "" _x16(unsigned long long int x) { return dpf::xints::xint16_t{static_cast<psnip_uint16_t>(x)}; }
+constexpr static auto operator "" _x17(unsigned long long int x) { return dpf::xints::xint17_t{static_cast<psnip_uint32_t>(x)}; }
+constexpr static auto operator "" _x18(unsigned long long int x) { return dpf::xints::xint18_t{static_cast<psnip_uint32_t>(x)}; }
+constexpr static auto operator "" _x19(unsigned long long int x) { return dpf::xints::xint19_t{static_cast<psnip_uint32_t>(x)}; }
+// 20--29
+constexpr static auto operator "" _x20(unsigned long long int x) { return dpf::xints::xint20_t{static_cast<psnip_uint32_t>(x)}; }
+constexpr static auto operator "" _x21(unsigned long long int x) { return dpf::xints::xint21_t{static_cast<psnip_uint32_t>(x)}; }
+constexpr static auto operator "" _x22(unsigned long long int x) { return dpf::xints::xint22_t{static_cast<psnip_uint32_t>(x)}; }
+constexpr static auto operator "" _x23(unsigned long long int x) { return dpf::xints::xint23_t{static_cast<psnip_uint32_t>(x)}; }
+constexpr static auto operator "" _x24(unsigned long long int x) { return dpf::xints::xint24_t{static_cast<psnip_uint32_t>(x)}; }
+constexpr static auto operator "" _x25(unsigned long long int x) { return dpf::xints::xint25_t{static_cast<psnip_uint32_t>(x)}; }
+constexpr static auto operator "" _x26(unsigned long long int x) { return dpf::xints::xint26_t{static_cast<psnip_uint32_t>(x)}; }
+constexpr static auto operator "" _x27(unsigned long long int x) { return dpf::xints::xint27_t{static_cast<psnip_uint32_t>(x)}; }
+constexpr static auto operator "" _x28(unsigned long long int x) { return dpf::xints::xint28_t{static_cast<psnip_uint32_t>(x)}; }
+constexpr static auto operator "" _x29(unsigned long long int x) { return dpf::xints::xint29_t{static_cast<psnip_uint32_t>(x)}; }
+// 30--39
+constexpr static auto operator "" _x30(unsigned long long int x) { return dpf::xints::xint30_t{static_cast<psnip_uint32_t>(x)}; }
+constexpr static auto operator "" _x31(unsigned long long int x) { return dpf::xints::xint31_t{static_cast<psnip_uint32_t>(x)}; }
+constexpr static auto operator "" _x32(unsigned long long int x) { return dpf::xints::xint32_t{static_cast<psnip_uint32_t>(x)}; }
+constexpr static auto operator "" _x33(unsigned long long int x) { return dpf::xints::xint33_t{static_cast<psnip_uint64_t>(x)}; }
+constexpr static auto operator "" _x34(unsigned long long int x) { return dpf::xints::xint34_t{static_cast<psnip_uint64_t>(x)}; }
+constexpr static auto operator "" _x35(unsigned long long int x) { return dpf::xints::xint35_t{static_cast<psnip_uint64_t>(x)}; }
+constexpr static auto operator "" _x36(unsigned long long int x) { return dpf::xints::xint36_t{static_cast<psnip_uint64_t>(x)}; }
+constexpr static auto operator "" _x37(unsigned long long int x) { return dpf::xints::xint37_t{static_cast<psnip_uint64_t>(x)}; }
+constexpr static auto operator "" _x38(unsigned long long int x) { return dpf::xints::xint38_t{static_cast<psnip_uint64_t>(x)}; }
+constexpr static auto operator "" _x39(unsigned long long int x) { return dpf::xints::xint39_t{static_cast<psnip_uint64_t>(x)}; }
+// 40--49
+constexpr static auto operator "" _x40(unsigned long long int x) { return dpf::xints::xint40_t{static_cast<psnip_uint64_t>(x)}; }
+constexpr static auto operator "" _x41(unsigned long long int x) { return dpf::xints::xint41_t{static_cast<psnip_uint64_t>(x)}; }
+constexpr static auto operator "" _x42(unsigned long long int x) { return dpf::xints::xint42_t{static_cast<psnip_uint64_t>(x)}; }
+constexpr static auto operator "" _x43(unsigned long long int x) { return dpf::xints::xint43_t{static_cast<psnip_uint64_t>(x)}; }
+constexpr static auto operator "" _x44(unsigned long long int x) { return dpf::xints::xint44_t{static_cast<psnip_uint64_t>(x)}; }
+constexpr static auto operator "" _x45(unsigned long long int x) { return dpf::xints::xint45_t{static_cast<psnip_uint64_t>(x)}; }
+constexpr static auto operator "" _x46(unsigned long long int x) { return dpf::xints::xint46_t{static_cast<psnip_uint64_t>(x)}; }
+constexpr static auto operator "" _x47(unsigned long long int x) { return dpf::xints::xint47_t{static_cast<psnip_uint64_t>(x)}; }
+constexpr static auto operator "" _x48(unsigned long long int x) { return dpf::xints::xint48_t{static_cast<psnip_uint64_t>(x)}; }
+constexpr static auto operator "" _x49(unsigned long long int x) { return dpf::xints::xint49_t{static_cast<psnip_uint64_t>(x)}; }
+// 50--59
+constexpr static auto operator "" _x50(unsigned long long int x) { return dpf::xints::xint50_t{static_cast<psnip_uint64_t>(x)}; }
+constexpr static auto operator "" _x51(unsigned long long int x) { return dpf::xints::xint51_t{static_cast<psnip_uint64_t>(x)}; }
+constexpr static auto operator "" _x52(unsigned long long int x) { return dpf::xints::xint52_t{static_cast<psnip_uint64_t>(x)}; }
+constexpr static auto operator "" _x53(unsigned long long int x) { return dpf::xints::xint53_t{static_cast<psnip_uint64_t>(x)}; }
+constexpr static auto operator "" _x54(unsigned long long int x) { return dpf::xints::xint54_t{static_cast<psnip_uint64_t>(x)}; }
+constexpr static auto operator "" _x55(unsigned long long int x) { return dpf::xints::xint55_t{static_cast<psnip_uint64_t>(x)}; }
+constexpr static auto operator "" _x56(unsigned long long int x) { return dpf::xints::xint56_t{static_cast<psnip_uint64_t>(x)}; }
+constexpr static auto operator "" _x57(unsigned long long int x) { return dpf::xints::xint57_t{static_cast<psnip_uint64_t>(x)}; }
+constexpr static auto operator "" _x58(unsigned long long int x) { return dpf::xints::xint58_t{static_cast<psnip_uint64_t>(x)}; }
+constexpr static auto operator "" _x59(unsigned long long int x) { return dpf::xints::xint59_t{static_cast<psnip_uint64_t>(x)}; }
+// 60--69
+constexpr static auto operator "" _x60(unsigned long long int x) { return dpf::xints::xint60_t{static_cast<psnip_uint64_t>(x)}; }
+constexpr static auto operator "" _x61(unsigned long long int x) { return dpf::xints::xint61_t{static_cast<psnip_uint64_t>(x)}; }
+constexpr static auto operator "" _x62(unsigned long long int x) { return dpf::xints::xint62_t{static_cast<psnip_uint64_t>(x)}; }
+constexpr static auto operator "" _x63(unsigned long long int x) { return dpf::xints::xint63_t{static_cast<psnip_uint64_t>(x)}; }
+constexpr static auto operator "" _x64(unsigned long long int x) { return dpf::xints::xint64_t{static_cast<psnip_uint64_t>(x)}; }
+template <char ...digits> constexpr static auto operator "" _x65() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint65_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x66() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint66_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x67() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint67_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x68() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint68_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x69() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint69_t{x}; }
+// 70--79
+template <char ...digits> constexpr static auto operator "" _x70() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint70_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x71() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint71_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x72() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint72_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x73() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint73_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x74() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint74_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x75() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint75_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x76() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint76_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x77() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint77_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x78() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint78_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x79() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint79_t{x}; }
+// 80--89
+template <char ...digits> constexpr static auto operator "" _x80() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint80_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x81() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint81_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x82() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint82_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x83() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint83_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x84() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint84_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x85() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint85_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x86() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint86_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x87() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint87_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x88() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint88_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x89() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint89_t{x}; }
+// 90--99
+template <char ...digits> constexpr static auto operator "" _x90() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint90_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x91() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint91_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x92() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint92_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x93() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint93_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x94() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint94_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x95() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint95_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x96() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint96_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x97() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint97_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x98() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint98_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x99() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint99_t{x}; }
+// 100--109
+template <char ...digits> constexpr static auto operator "" _x100() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint100_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x101() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint101_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x102() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint102_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x103() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint103_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x104() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint104_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x105() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint105_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x106() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint106_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x107() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint107_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x108() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint108_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x109() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint109_t{x}; }
+// 110--119
+template <char ...digits> constexpr static auto operator "" _x110() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint110_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x111() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint111_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x112() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint112_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x113() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint113_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x114() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint114_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x115() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint115_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x116() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint116_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x117() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint117_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x118() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint118_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x119() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint119_t{x}; }
+// 120--129
+template <char ...digits> constexpr static auto operator "" _x120() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint120_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x121() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint121_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x122() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint122_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x123() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint123_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x124() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint124_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x125() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint125_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x126() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint126_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x127() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint127_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x128() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); simde_uint128 x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint128_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x129() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint129_t{x}; }
+// 130--139
+template <char ...digits> constexpr static auto operator "" _x130() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint130_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x131() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint131_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x132() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint132_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x133() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint133_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x134() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint134_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x135() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint135_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x136() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint136_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x137() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint137_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x138() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint138_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x139() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint139_t{x}; }
+// 140--149
+template <char ...digits> constexpr static auto operator "" _x140() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint140_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x141() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint141_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x142() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint142_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x143() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint143_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x144() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint144_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x145() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint145_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x146() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint146_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x147() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint147_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x148() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint148_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x149() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint149_t{x}; }
+// 150--159
+template <char ...digits> constexpr static auto operator "" _x150() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint150_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x151() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint151_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x152() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint152_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x153() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint153_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x154() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint154_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x155() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint155_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x156() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint156_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x157() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint157_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x158() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint158_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x159() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint159_t{x}; }
+// 160--169
+template <char ...digits> constexpr static auto operator "" _x160() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint160_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x161() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint161_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x162() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint162_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x163() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint163_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x164() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint164_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x165() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint165_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x166() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint166_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x167() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint167_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x168() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint168_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x169() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint169_t{x}; }
+// 170--179
+template <char ...digits> constexpr static auto operator "" _x170() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint170_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x171() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint171_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x172() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint172_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x173() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint173_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x174() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint174_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x175() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint175_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x176() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint176_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x177() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint177_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x178() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint178_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x179() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint179_t{x}; }
+// 180--189
+template <char ...digits> constexpr static auto operator "" _x180() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint180_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x181() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint181_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x182() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint182_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x183() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint183_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x184() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint184_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x185() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint185_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x186() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint186_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x187() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint187_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x188() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint188_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x189() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint189_t{x}; }
+// 190--199
+template <char ...digits> constexpr static auto operator "" _x190() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint190_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x191() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint191_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x192() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint192_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x193() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint193_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x194() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint194_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x195() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint195_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x196() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint196_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x197() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint197_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x198() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint198_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x199() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint199_t{x}; }
+// 200--209
+template <char ...digits> constexpr static auto operator "" _x200() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint200_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x201() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint201_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x202() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint202_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x203() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint203_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x204() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint204_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x205() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint205_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x206() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint206_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x207() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint207_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x208() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint208_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x209() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint209_t{x}; }
+// 210--219
+template <char ...digits> constexpr static auto operator "" _x210() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint210_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x211() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint211_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x212() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint212_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x213() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint213_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x214() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint214_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x215() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint215_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x216() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint216_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x217() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint217_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x218() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint218_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x219() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint219_t{x}; }
+// 220--229
+template <char ...digits> constexpr static auto operator "" _x220() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint220_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x221() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint221_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x222() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint222_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x223() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint223_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x224() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint224_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x225() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint225_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x226() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint226_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x227() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint227_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x228() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint228_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x229() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint229_t{x}; }
+// 230--239
+template <char ...digits> constexpr static auto operator "" _x230() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint230_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x231() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint231_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x232() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint232_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x233() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint233_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x234() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint234_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x235() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint235_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x236() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint236_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x237() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint237_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x238() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint238_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x239() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint239_t{x}; }
+// 240--249
+template <char ...digits> constexpr static auto operator "" _x240() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint240_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x241() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint241_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x242() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint242_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x243() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint243_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x244() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint244_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x245() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint245_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x246() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint246_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x247() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint247_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x248() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint248_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x249() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint249_t{x}; }
+// 250--259
+template <char ...digits> constexpr static auto operator "" _x250() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint250_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x251() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint251_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x252() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint252_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x253() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint253_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x254() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint254_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x255() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint255_t{x}; }
+template <char ...digits> constexpr static auto operator "" _x256() { utils::constexpr_maybe_throw<std::runtime_error>(!(std::isdigit(digits) && ...), "invalid char"); uint256_t x{0}; (~((x *= 10) | (x += digits - '0')), ...); return dpf::xints::xint256_t{x}; }
+
+}  // namespace xints
+
+}  // namespace literals
+
+namespace utils
+{
+
+template <typename T>
+struct is_xor_wrapper<xor_wrapper<T>> : std::true_type {};
+
+/// @brief specializes `dpf::utils::bitlength_of` for `xor_wrapper`
+template <typename T>
+struct bitlength_of<xor_wrapper<T>>
+  : public bitlength_of<T>
+{ };
+
+template <typename T>
+struct msb_of<xor_wrapper<T>>
+{
+    static constexpr xor_wrapper<T> value = xor_wrapper<T>{msb_of_v<T>};
+};
+
+template <typename T>
+struct countl_zero_symmetric_difference<xor_wrapper<T>>
+  : public countl_zero_symmetric_difference<T>
+{ };
+
+template <typename T>
+struct to_integral_type<xor_wrapper<T>> : public to_integral_type_base<T>
+{
+    using parent = to_integral_type_base<T>;
+    using typename parent::integral_type;
+    static constexpr auto to_int = to_integral_type<T> {};
+
+    HEDLEY_PURE
+    HEDLEY_NO_THROW
+    HEDLEY_ALWAYS_INLINE
+    constexpr integral_type operator()(const xor_wrapper<T> & input) const noexcept
+    {
+        return to_int(input.value);
+    }
+};
+
+template <typename T>
+struct mod_pow_2<xor_wrapper<T>>
+{
+    static constexpr auto mod = mod_pow_2<T>{};
+    std::size_t operator()(xor_wrapper<T> val, std::size_t n) const noexcept
+    {
+        return mod(val.value, n);
+    }
+};
+
+template <typename T>
+struct has_characteristic_two<xor_wrapper<T>> : public std::true_type {};
+
+}  // namespace utils
+
+}  // namespace dpf
+
+namespace std
+{
+
+/// @brief specializes `std::numeric_limits` for CV-qualified `xor_wrapper`s
+/// @{
+
+/// @details specializes `std::numeric_limits` for `xor_wrapper<T>`
+template<typename T>
+class numeric_limits<dpf::xor_wrapper<T>>
+  : public numeric_limits<dpf::utils::make_unsigned_t<T>> {};
+
+/// @details specializes `std::numeric_limits` for `xor_wrapper<T> const`
+template<typename T>
+class numeric_limits<dpf::xor_wrapper<T> const>
+  : public numeric_limits<dpf::xor_wrapper<T>> {};
+
+/// @details specializes `std::numeric_limits` for
+///          `xor_wrapper<T> volatile`
+template<typename T>
+class numeric_limits<dpf::xor_wrapper<T> volatile>
+  : public numeric_limits<dpf::xor_wrapper<T>> {};
+
+/// @details specializes `std::numeric_limits` for
+///          `xor_wrapper<T> const volatile`
+template<typename T>
+class numeric_limits<dpf::xor_wrapper<T> const volatile>
+  : public numeric_limits<dpf::xor_wrapper<T>> {};
+
+/// @}
+
+}  // namespace std
+
+#endif  // LIBDPF_INCLUDE_DPF_XOR_WRAPPER_HPP__

include/dpf/zip_iterable.hpp

@@ -0,0 +1,120 @@
+/// @file dpf/zip_iterable.hpp
+/// @brief defines the `dpf::zip_itrable` class and associated helpers
+/// @details
+/// @author Ryan Henry <ryan.henry@ucalgary.ca>
+/// @copyright Copyright (c) 2019-2024 Ryan Henry and [others](@ref authors)
+/// @license Released under a GNU General Public v2.0 (GPLv2) license;
+///          see [LICENSE.md](@ref license) for details.
+
+#ifndef LIBDPF_INCLUDE_DPF_ZIP_ITERABLE_HPP__
+#define LIBDPF_INCLUDE_DPF_ZIP_ITERABLE_HPP__
+
+#include <tuple>
+#include <iterator>
+
+#include "hedley/hedley.h"
+
+namespace dpf
+{
+
+template <typename ...Iterables> struct zip_iterator;
+
+template <typename ...Iterables>
+struct zip_iterable
+{
+    using value_type = std::tuple<typename Iterables::iterator...>;
+    using iterator = zip_iterator<Iterables...>;
+
+    zip_iterable(value_type && begin, value_type && end)
+      : begin_{begin}, end_{end} { }
+
+    auto begin()
+    {
+        return iterator{begin_};
+    }
+    auto begin() const
+    {
+        return iterator{begin_};
+    }
+    auto cbegin() const
+    {
+        return begin();
+    }
+    auto end()
+    {
+        return iterator{end_};
+    }
+    auto end() const
+    {
+        return iterator{end_};
+    }
+    auto cend() const
+    {
+        return end();
+    }
+
+    value_type begin_, end_;
+};
+
+template <typename ...Iterables>
+struct zip_iterator
+{
+    using value_type = std::tuple<typename Iterables::iterator...>;
+
+    auto operator*() const
+    {
+        return std::apply([](auto && ...its){ return std::make_tuple(*its...); }, wrapped_iterators);
+    }
+    zip_iterator & operator++()
+    {
+        std::apply([](auto && ...its){ (++its, ...); }, wrapped_iterators);
+        return *this;
+    }
+    zip_iterator operator++(int)
+    {
+        zip_iterator old = *this;
+        ++(*this);
+        return old;
+    }
+    zip_iterator & operator--()
+    {
+        std::apply([](auto && ...its){ (--its, ...); }, wrapped_iterators);
+        return *this;
+    }
+    zip_iterator operator--(int)
+    {
+        zip_iterator old = *this;
+        --(*this);
+        return old;
+    }
+    bool operator==(const zip_iterator & rhs) const
+    {
+        return wrapped_iterators == rhs.wrapped_iterators;
+    }
+    bool operator!=(const zip_iterator & rhs) const
+    {
+        return wrapped_iterators != rhs.wrapped_iterators;
+    }
+    value_type wrapped_iterators;
+};
+
+template <typename ...Iterables>
+HEDLEY_ALWAYS_INLINE
+zip_iterable<Iterables...> tuple_as_zip(std::tuple<Iterables...> & tup) noexcept
+{
+    return zip_iterable<Iterables...>(
+        std::apply([](auto && ...elements){ return std::make_tuple(std::begin(elements)...); }, tup),
+        std::apply([](auto && ...elements){ return std::make_tuple(std::end(elements)...); }, tup));
+}
+
+template <typename TupleT,
+          typename UnaryFunction>
+HEDLEY_ALWAYS_INLINE
+void for_each_in_zip(TupleT && tuple, UnaryFunction f)
+{
+    for (auto i : tuple_as_zip(tuple)) f(i);
+}
+
+}  // namespace dpf
+
+#endif  // LIBDPF_INCLUDE_DPF_ZIP_ITERABLE_HPP__