/// @file dpf/net/party_session.hpp /// @brief One role's TCP/SCTP session: join, per-edge transport, dealer, reconnect. /// @details Socket edges run TLS 1.3 unless `session_options::security.encrypt` /// is off (see `dpf/net/security.hpp`): each side presents its /// identity key and authenticates the peer only if it holds that /// peer's key, so authentication can run in one direction, both, or /// neither; every unauthenticated direction is logged. After the TLS /// handshake (or directly, with encryption off) both ends exchange a /// fixed record (party id, epoch, transport, lane count, lane index, /// and whether the sender authenticated the receiver) under a /// deadline, so two ends that disagree fail at connect time with a /// message instead of corrupting frames later. Lower ids accept, /// higher ids connect; connects retry until the peer listens, so /// processes may start in any order. `join(table)` takes a static /// `host:port` table (separate processes); `join(host, ports)` is the /// in-process rendezvous. `reconnect(peer)` replaces one edge and /// returns the new link; `reconnector(peer)` adapts it for /// `sink_options::reconnect`. The dealer link has its own policy, /// `io_context`, epoch, and reconnect. SCTP edges cannot be encrypted /// and are refused while encryption is on. #ifndef LIBDPF_INCLUDE_DPF_NET_PARTY_SESSION_HPP__ #define LIBDPF_INCLUDE_DPF_NET_PARTY_SESSION_HPP__ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "dpf/net/asio_ns.hpp" #include "dpf/log.hpp" #include "dpf/net/async_sctp_stream_array.hpp" #include "dpf/net/async_stream_array.hpp" #include "dpf/net/connect.hpp" #include "dpf/net/identity.hpp" #include "dpf/net/link_log.hpp" #include "dpf/net/policy.hpp" #include "dpf/net/security.hpp" #include "dpf/net/mesh_rendezvous.hpp" #include "dpf/net/socket_tune.hpp" #include "dpf/net/tls.hpp" namespace dpf { namespace net { /// @brief Where one party listens. struct peer_address { std::string host = "127.0.0.1"; unsigned short port = 0; }; /// @brief Session-wide defaults. Per-edge overrides live on `party_session`. struct session_options { std::size_t n_lanes = 1; transport kind = transport::mux; wire_policy policy{}; deadlines limits{}; peer_security security{}; }; namespace detail { struct session_hello { static constexpr std::uint32_t magic = 0x53535044u; // 'DPSS' static constexpr std::size_t size = 28; /// The sender authenticated the receiver's key. static constexpr std::uint32_t verified_you = 1; std::uint32_t id = 0; std::uint32_t epoch = 0; std::uint32_t kind = 0; std::uint32_t lanes = 0; std::uint32_t lane = 0; std::uint32_t flags = 0; void pack(std::uint8_t * p) const { put_u32(p, magic); put_u32(p + 4, id); put_u32(p + 8, epoch); put_u32(p + 12, kind); put_u32(p + 16, lanes); put_u32(p + 20, lane); put_u32(p + 24, flags); } static session_hello unpack(const std::uint8_t * p) { if (p[0] == 0x16 && p[1] == 0x03) throw std::runtime_error("party_session: the peer started a TLS handshake " "but this side has encryption off (set encryption the same at both " "ends)"); if (get_u32(p) != magic) throw std::runtime_error( "party_session: peer is not a party_session (bad handshake magic)"); session_hello h; h.id = get_u32(p + 4); h.epoch = get_u32(p + 8); h.kind = get_u32(p + 12); h.lanes = get_u32(p + 16); h.lane = get_u32(p + 20); h.flags = get_u32(p + 24); return h; } }; inline std::string id_name(std::uint32_t id) { return id == 0xfffffffeu ? std::string("dealer") : "party " + std::to_string(id); } inline std::string role_name(std::uint32_t id) { return id == 0xfffffffeu ? std::string("dealer") : "p" + std::to_string(id); } /// @brief One TCP connection on its way into a link: plain, or TLS once the /// handshake is done. struct peer_conn { explicit peer_conn(asio::ip::tcp::socket s) : plain(std::move(s)), fd(plain.native_handle()) { } asio::ip::tcp::socket plain; #if DPF_HAS_OPENSSL std::unique_ptr tls; #endif int fd = -1; link_security sec; bool encrypted() const noexcept { #if DPF_HAS_OPENSSL return static_cast(tls); #else return false; #endif } }; inline std::chrono::milliseconds left(setup_clock::time_point deadline) { return std::chrono::milliseconds(std::max(1, remaining_ms(deadline))); } #if DPF_HAS_OPENSSL using peer_tls_context = tls_context; #else struct peer_tls_context { }; #endif /// @brief Tune `c` (TCP_NODELAY first: the handshake is several round trips), /// then run the TLS handshake when `ctx` is set. inline void secure(peer_conn & c, asio::io_context & io, peer_tls_context * ctx, bool server, const socket_options & so, std::chrono::milliseconds budget, const std::string & who) { tune_tcp(c.plain, so); if (ctx == nullptr) return; #if DPF_HAS_OPENSSL c.tls = std::make_unique(std::move(c.plain), *ctx); try { tls_handshake(io, *c.tls, server, budget, who); } catch (const std::system_error & e) { throw std::runtime_error(std::string(e.what()) + " (if the peer has encryption off, set it the same at both ends)"); } c.sec = tls_describe(*c.tls); #else (void)io; (void)server; (void)budget; (void)who; #endif } inline void send_hello(peer_conn & c, asio::io_context & io, const session_hello & h, setup_clock::time_point deadline, const std::string & what) { std::uint8_t out[session_hello::size]; h.pack(out); #if DPF_HAS_OPENSSL if (c.tls) { tls_write(io, *c.tls, out, sizeof(out), left(deadline), what); return; } #endif (void)io; send_all_until(c.plain.native_handle(), out, sizeof(out), deadline, what); } inline session_hello recv_hello(peer_conn & c, asio::io_context & io, setup_clock::time_point deadline, const std::string & what) { std::uint8_t in[session_hello::size]; #if DPF_HAS_OPENSSL if (c.tls) { tls_read(io, *c.tls, in, sizeof(in), left(deadline), what); return session_hello::unpack(in); } #endif (void)io; recv_all_until(c.plain.native_handle(), in, sizeof(in), deadline, what); return session_hello::unpack(in); } /// @brief Authenticate `party` on an encrypted connection (no-op when plain). inline void check_party(peer_conn & c, const peer_security & policy, std::uint32_t party, const std::string & who) { #if DPF_HAS_OPENSSL if (c.encrypted()) { try { check_peer(c.sec, policy, party, who); } catch (...) { std::error_code e; c.tls->lowest_layer().close(e); throw; } } #else (void)c; (void)policy; (void)party; (void)who; #endif } inline std::unique_ptr mux_link(asio::io_context & io, peer_conn & c, unsigned self, unsigned peer, std::size_t lanes, const wire_policy & pol) { #if DPF_HAS_OPENSSL if (c.tls) return std::make_unique(io, std::move(*c.tls), self, peer, lanes, pol); #endif return std::make_unique(io, std::move(c.plain), self, peer, lanes, pol); } inline std::unique_ptr parallel_link(asio::io_context & io, std::vector> & cs, const wire_policy & pol) { #if DPF_HAS_OPENSSL if (cs.front()->tls) { std::vector v; v.reserve(cs.size()); for (auto & c : cs) v.push_back(std::move(*c->tls)); return std::make_unique(io, std::move(v), pol); } #endif std::vector v; v.reserve(cs.size()); for (auto & c : cs) v.push_back(std::move(c->plain)); return std::make_unique(io, std::move(v), pol); } /// @brief This process's key for party links, or a fresh one (logged). inline std::shared_ptr link_identity(const peer_security & sec, const std::string & who) { if (sec.self) { DPF_LOG(info, "security.identity").kv("who", who) .kv("key", sec.self->key().base64()).kv("ephemeral", false) .kv("trusted", sec.trusted.size()); return sec.self; } auto id = std::make_shared(identity::generate()); DPF_LOG(info, "security.identity").kv("who", who).kv("key", id->key().base64()) .kv("ephemeral", true).kv("trusted", sec.trusted.size()); if (log::first_time("security.no_identity." + log::role() + "." + who)) DPF_LOG(warning, "security.no_identity").kv("who", who) .kv("detail", "no identity key configured: links are encrypted to a fresh " "key for this run, so peers cannot authenticate " + who); return id; } /// @brief Log a link direction this side could not authenticate. inline void note_link(const std::string & peer_role, const link_security & sec) { if (!sec.encrypted) return; const std::string me = log::role().empty() ? std::string("this party") : log::role(); if (sec.peer_auth == "none" && log::first_time("security.unauthenticated." + me + "." + peer_role)) DPF_LOG(warning, "security.unauthenticated").kv("peer", peer_role) .kv("peer_key", sec.peer_key ? sec.peer_key->base64() : std::string("none")) .kv("detail", "no key configured for " + peer_role + ": the link is " "encrypted but " + peer_role + " is not authenticated"); if (!sec.peer_verified_us && log::first_time("security.not_authenticated_by." + me + "." + peer_role)) DPF_LOG(info, "security.not_authenticated_by_peer").kv("peer", peer_role) .kv("detail", peer_role + " holds no key for " + me); } } // namespace detail class party_session { public: static constexpr unsigned k_dealer_id = dealer_id; party_session(asio::io_context & io, unsigned me, unsigned parties, session_options opt) : io_(&io), dealer_io_(&io), me_(me), parties_(parties), opt_(std::move(opt)) { if (parties_ < 2 || me_ >= parties_) throw std::invalid_argument("party_session: party/n"); if (opt_.n_lanes == 0) throw std::invalid_argument("party_session: n_lanes 0"); opt_.policy.validate(); links_.resize(parties_); sec_.resize(parties_); table_.resize(parties_); epoch_.assign(parties_, 0); kind_.assign(parties_, opt_.kind); policy_.assign(parties_, opt_.policy); dealer_policy_ = opt_.policy; if (opt_.security.encrypt) { refuse_sctp(opt_.kind); #if DPF_HAS_OPENSSL self_ = detail::link_identity(opt_.security, "p" + std::to_string(me_)); tls_ = make_peer_tls_context(*self_); #else throw std::logic_error("party_session: built without OpenSSL; set " "encryption=off for plaintext links"); #endif } } party_session(asio::io_context & io, unsigned me, unsigned parties, std::size_t nstreams) : party_session(io, me, parties, make_opts(nstreams)) { } unsigned me() const noexcept { return me_; } unsigned parties() const noexcept { return parties_; } std::size_t streams() const noexcept { return opt_.n_lanes; } asio::io_context & context() noexcept { return *io_; } const session_options & options() const noexcept { return opt_; } /// @brief This party's link key (null with encryption off). std::shared_ptr identity_key() const noexcept { return self_; } // --- configuration (before join, both ends must agree) --------------- void set_edge_transport(unsigned peer, transport t) { check_peer(peer); if (t != transport::mux && t != transport::parallel && t != transport::sctp) throw std::invalid_argument(std::string("party_session: transport ") + transport_name(t) + " is not a socket edge"); if (t == transport::sctp && !sctp_available()) throw std::logic_error("party_session: SCTP needs Linux + libsctp"); if (opt_.security.encrypt) refuse_sctp(t); kind_[peer] = t; } transport edge_transport(unsigned peer) const { check_peer(peer); return kind_[peer]; } void set_edge_policy(unsigned peer, const wire_policy & pol) { check_peer(peer); pol.validate(); policy_[peer] = pol; if (links_[peer]) links_[peer]->set_window_bytes(pol.window_bytes); } /// @brief Window for every edge and the dealer link, now and later. void set_window_bytes(std::size_t bytes) { for (auto & p : policy_) p.window_bytes = bytes; dealer_policy_.window_bytes = bytes; for (auto & link : links_) if (link) link->set_window_bytes(bytes); if (dealer_) dealer_->set_window_bytes(bytes); } void set_dealer_policy(const wire_policy & pol) { pol.validate(); dealer_policy_ = pol; if (dealer_) dealer_->set_window_bytes(pol.window_bytes); } /// @brief Run the dealer link on its own `io_context` (own failure domain). void set_dealer_context(asio::io_context & io) { dealer_io_ = &io; } void set_deadlines(const deadlines & d) { opt_.limits = d; } // --- listen / join --------------------------------------------------- /// @brief Bind TCP (and SCTP if any edge uses it) on `port`; 0 = ephemeral. unsigned short listen(unsigned short port = 0) { if (!acceptor_) { acceptor_ = std::make_unique(*io_); open_listener(*acceptor_, port); published_ = acceptor_->local_endpoint().port(); log_listen(published_, wants_sctp(), opt_.security.encrypt); } if (!sctp_ && wants_sctp()) sctp_ = std::make_unique(published_, opt_.n_lanes, opt_.policy.socket); return published_; } unsigned short port() const noexcept { return published_; } /// @brief Join with a static table: `table[i]` is party `i`'s address. void join(const std::vector & table) { if (table.size() != parties_) throw std::invalid_argument("party_session::join: table has " + std::to_string(table.size()) + " entries, expected " + std::to_string(parties_)); table_ = table; listen(table_[me_].port); accept_higher(); for (unsigned peer = 0; peer < me_; ++peer) connect_edge(peer); } /// @brief In-process rendezvous through shared `ports` (tests, one binary). void join(const std::string & host, mesh_ports & ports, std::size_t n_lanes = 0) { if (ports.size() != parties_) throw std::invalid_argument("party_session::join port table"); if (n_lanes != 0) opt_.n_lanes = n_lanes; ports[me_].store(listen(ports[me_].load())); const auto deadline = setup_clock::now() + opt_.limits.join; std::vector table(parties_); for (unsigned i = 0; i < parties_; ++i) { while (ports[i].load() == 0) { if (setup_clock::now() >= deadline) throw std::runtime_error("party_session: join timed out after " + std::to_string(opt_.limits.join.count()) + " ms waiting for party " + std::to_string(i)); std::this_thread::sleep_for(std::chrono::milliseconds(1)); } table[i] = peer_address{host, ports[i].load()}; } join(table); } // --- edges ----------------------------------------------------------- async_stream_array & peer(unsigned p) { check_peer(p); if (!links_[p]) throw std::logic_error("party_session: no link to party " + std::to_string(p) + " (join first)"); return *links_[p]; } stream_stats edge_stats(unsigned p) const { check_peer(p); return links_[p] ? links_[p]->stats() : stream_stats{}; } /// @brief How the edge to `p` was secured (encryption, cipher, who /// authenticated whom). const link_security & edge_security(unsigned p) const { check_peer(p); return sec_[p]; } std::uint32_t epoch(unsigned p) const { check_peer(p); return epoch_[p]; } /// @brief Replace the edge to `peer` and return it. Both ends must call. async_stream_array & reconnect(unsigned peer) { check_peer(peer); if (table_[peer].port == 0 && me_ > peer) throw std::logic_error("party_session::reconnect before join"); links_[peer].reset(); ++epoch_[peer]; if (me_ < peer) accept_one_edge(peer); else connect_edge(peer); return *links_[peer]; } /// @brief `sink_options::reconnect` for the edge to `peer`. std::function reconnector( unsigned peer) { return [this, peer](const std::error_code &) { return &reconnect(peer); }; } // --- dealer ---------------------------------------------------------- /// @brief Party side: connect to the dealer at `host:port`. void connect_dealer(const std::string & host, unsigned short port) { dealer_addr_ = peer_address{host, port}; const std::string who = "dealer at " + host + ":" + std::to_string(port); asio::ip::tcp::socket sock(*dealer_io_); connect_until(sock, host, port, opt_.limits.connect); detail::peer_conn c(std::move(sock)); const auto deadline = setup_clock::now() + opt_.limits.handshake; detail::secure(c, *dealer_io_, tls_ptr(), false, dealer_policy_.socket, opt_.limits.handshake, "handshake with " + who); detail::check_party(c, opt_.security, k_dealer_id, who); auto mine = dealer_hello(me_); mine.flags = c.sec.peer_auth == "key" ? detail::session_hello::verified_you : 0; detail::send_hello(c, *dealer_io_, mine, deadline, "dealer handshake"); const auto got = detail::recv_hello(c, *dealer_io_, deadline, "dealer handshake"); if (got.id != k_dealer_id) throw std::runtime_error("party_session: " + host + ":" + std::to_string(port) + " is " + detail::id_name(got.id) + ", not the dealer"); check_match(got, mine, "dealer"); c.sec.peer_verified_us = (got.flags & detail::session_hello::verified_you) != 0; dealer_sec_ = c.sec; dealer_ = detail::mux_link(*dealer_io_, c, me_, k_dealer_id, opt_.n_lanes, dealer_policy_); log_link_up("connect", "dealer", transport::mux, opt_.n_lanes, 0, dealer_epoch_, c.fd, dealer_policy_.socket, &dealer_sec_); detail::note_link("dealer", dealer_sec_); } /// @brief Dealer side of a one-party dealer link (see `dealer_session` /// for a dealer serving several parties). void accept_dealer() { listen(published_); asio::ip::tcp::socket sock(*dealer_io_); accept_until(*acceptor_, sock, opt_.limits.accept); detail::peer_conn c(std::move(sock)); const auto deadline = setup_clock::now() + opt_.limits.handshake; detail::secure(c, *dealer_io_, tls_ptr(), true, dealer_policy_.socket, opt_.limits.handshake, "dealer handshake"); const auto got = detail::recv_hello(c, *dealer_io_, deadline, "dealer handshake"); detail::check_party(c, opt_.security, got.id, detail::id_name(got.id)); auto mine = dealer_hello(k_dealer_id); mine.flags = c.sec.peer_auth == "key" ? detail::session_hello::verified_you : 0; detail::send_hello(c, *dealer_io_, mine, deadline, "dealer handshake"); check_match(got, mine, "dealer"); c.sec.peer_verified_us = (got.flags & detail::session_hello::verified_you) != 0; dealer_is_server_ = true; dealer_sec_ = c.sec; dealer_ = detail::mux_link(*dealer_io_, c, k_dealer_id, got.id, opt_.n_lanes, dealer_policy_); log_link_up("accept", detail::role_name(got.id), transport::mux, opt_.n_lanes, 0, dealer_epoch_, c.fd, dealer_policy_.socket, &dealer_sec_); detail::note_link(detail::role_name(got.id), dealer_sec_); } /// @brief Replace the dealer link (both ends must call). async_stream_array & reconnect_dealer() { dealer_.reset(); ++dealer_epoch_; if (dealer_is_server_) accept_dealer(); else connect_dealer(dealer_addr_.host, dealer_addr_.port); return *dealer_; } std::function dealer_reconnector() { return [this](const std::error_code &) { return &reconnect_dealer(); }; } bool has_dealer() const noexcept { return static_cast(dealer_); } async_stream_array & dealer() { if (!dealer_) throw std::logic_error("party_session: no dealer link"); return *dealer_; } stream_stats dealer_stats() const { return dealer_ ? dealer_->stats() : stream_stats{}; } const link_security & dealer_security() const noexcept { return dealer_sec_; } private: static session_options make_opts(std::size_t nstreams) { session_options o; o.n_lanes = nstreams; return o; } static void refuse_sctp(transport t) { if (t == transport::sctp) throw std::invalid_argument("party_session: SCTP links cannot be " "encrypted (OpenSSL has no TLS over SCTP streams); use mux or " "parallel, or set encryption=off"); } detail::peer_tls_context * tls_ptr() const noexcept { #if DPF_HAS_OPENSSL return tls_.get(); #else return nullptr; #endif } detail::session_hello dealer_hello(std::uint32_t id) const { detail::session_hello h; h.id = id; h.epoch = dealer_epoch_; h.kind = static_cast(transport::mux); h.lanes = static_cast(opt_.n_lanes); return h; } void check_peer(unsigned p) const { if (p == me_ || p >= parties_) throw std::invalid_argument("party_session: bad peer " + std::to_string(p)); } bool wants_sctp() const { for (unsigned p = 0; p < parties_; ++p) if (p != me_ && kind_[p] == transport::sctp) return true; return false; } std::size_t sockets_for(unsigned peer) const { return kind_[peer] == transport::parallel ? opt_.n_lanes : 1; } detail::session_hello my_hello(unsigned peer, std::uint32_t lane) const { detail::session_hello h; h.id = me_; h.epoch = epoch_[peer]; h.kind = static_cast(kind_[peer]); h.lanes = static_cast(opt_.n_lanes); h.lane = lane; return h; } void check_match(const detail::session_hello & got, const detail::session_hello & mine, const std::string & who) const { std::string why; if (got.kind != mine.kind) why += std::string(" transport ") + transport_name(static_cast(got.kind)) + " vs " + transport_name(static_cast(mine.kind)) + ";"; if (got.lanes != mine.lanes) why += " lanes " + std::to_string(got.lanes) + " vs " + std::to_string(mine.lanes) + ";"; if (got.epoch != mine.epoch) why += " epoch " + std::to_string(got.epoch) + " vs " + std::to_string(mine.epoch) + ";"; if (!why.empty()) throw std::runtime_error("party_session: " + who + " disagrees (peer vs this side):" + why); } struct parallel_in_progress { std::vector> conns; std::size_t got = 0; }; /// @brief Accept every edge from higher-numbered parties, in any order, /// on the TCP and SCTP listeners at once. void accept_higher() { std::size_t tcp_left = 0; std::size_t sctp_left = 0; for (unsigned p = me_ + 1; p < parties_; ++p) { if (kind_[p] == transport::sctp) ++sctp_left; else tcp_left += sockets_for(p); } std::map partial; const auto deadline = setup_clock::now() + opt_.limits.accept; while (tcp_left + sctp_left > 0) { pollfd pfds[2]{}; int nfds = 0; if (tcp_left > 0) { pfds[nfds].fd = acceptor_->native_handle(); pfds[nfds].events = POLLIN; ++nfds; } if (sctp_left > 0) { pfds[nfds].fd = sctp_fd(); pfds[nfds].events = POLLIN; ++nfds; } const int ms = detail::remaining_ms(deadline); if (ms == 0) throw std::system_error(std::make_error_code(std::errc::timed_out), "party_session " + std::to_string(me_) + ": accept timed out " "after " + std::to_string(opt_.limits.accept.count()) + " ms (" + std::to_string(tcp_left + sctp_left) + " connections missing)"); const int rc = ::poll(pfds, static_cast(nfds), ms); if (rc <= 0) continue; for (int k = 0; k < nfds; ++k) { if ((pfds[k].revents & POLLIN) == 0) continue; if (tcp_left > 0 && pfds[k].fd == acceptor_->native_handle()) { asio::ip::tcp::socket sock(*io_); acceptor_->accept(sock); accept_tcp(std::move(sock), partial, false, 0); --tcp_left; } else if (sctp_left > 0) { accept_sctp(0, false); --sctp_left; } } } } int sctp_fd() const { return sctp_ ? sctp_->native_handle() : -1; } void accept_tcp(asio::ip::tcp::socket sock, std::map & partial, bool expect_one, unsigned expect) { auto c = std::make_unique(std::move(sock)); const auto deadline = setup_clock::now() + opt_.limits.handshake; detail::secure(*c, *io_, tls_ptr(), true, opt_.policy.socket, opt_.limits.handshake, "party_session " + std::to_string(me_) + " handshake"); // The accepting side answers with the record for the announced peer. const auto theirs = detail::recv_hello(*c, *io_, deadline, "party_session handshake"); const unsigned p = theirs.id; if (p <= me_ || p >= parties_ || (expect_one && p != expect)) throw std::runtime_error("party_session " + std::to_string(me_) + ": unexpected connection from " + detail::id_name(p)); detail::check_party(*c, opt_.security, p, "party " + std::to_string(p)); auto mine = my_hello(p, theirs.lane); mine.flags = c->sec.peer_auth == "key" ? detail::session_hello::verified_you : 0; detail::send_hello(*c, *io_, mine, deadline, "party_session handshake"); check_match(theirs, mine, "party " + std::to_string(p)); c->sec.peer_verified_us = (theirs.flags & detail::session_hello::verified_you) != 0; const std::string role = "p" + std::to_string(p); if (kind_[p] == transport::mux) { if (links_[p]) throw std::runtime_error("party_session: duplicate edge from party " + std::to_string(p)); sec_[p] = c->sec; links_[p] = detail::mux_link(*io_, *c, me_, p, opt_.n_lanes, policy_[p]); log_link_up("accept", role, kind_[p], opt_.n_lanes, 0, epoch_[p], c->fd, policy_[p].socket, &sec_[p]); detail::note_link(role, sec_[p]); return; } auto & pp = partial[p]; if (pp.conns.empty()) pp.conns.resize(opt_.n_lanes); if (theirs.lane >= opt_.n_lanes || pp.conns[theirs.lane]) throw std::runtime_error("party_session: bad parallel lane " + std::to_string(theirs.lane) + " from party " + std::to_string(p)); pp.conns[theirs.lane] = std::move(c); if (++pp.got == opt_.n_lanes) adopt_parallel(p, pp.conns, "accept"); if (pp.got == opt_.n_lanes) partial.erase(p); } /// @brief Build a parallel link from one connection per lane. void adopt_parallel(unsigned p, std::vector> & cs, const char * how) { std::vector fds; std::vector secs; for (auto & c : cs) { fds.push_back(c->fd); secs.push_back(c->sec); } sec_[p] = secs.front(); for (const auto & s : secs) sec_[p].peer_verified_us = sec_[p].peer_verified_us && s.peer_verified_us; links_[p] = detail::parallel_link(*io_, cs, policy_[p]); const std::string role = "p" + std::to_string(p); for (std::size_t k = 0; k < fds.size(); ++k) log_link_up(how, role, kind_[p], opt_.n_lanes, static_cast(k), epoch_[p], fds[k], policy_[p].socket, &secs[k]); detail::note_link(role, sec_[p]); } void accept_sctp(unsigned expect, bool expect_one) { const int fd = sctp_->accept(opt_.limits.accept); std::uint8_t in[detail::session_hello::size]; const auto deadline = setup_clock::now() + opt_.limits.handshake; try { detail::recv_all_until(fd, in, sizeof(in), deadline, "party_session sctp handshake"); const auto theirs = detail::session_hello::unpack(in); const unsigned p = theirs.id; if (p <= me_ || p >= parties_ || (expect_one && p != expect)) throw std::runtime_error("party_session: unexpected SCTP " "association from " + detail::id_name(p)); const auto mine = my_hello(p, 0); std::uint8_t out[detail::session_hello::size]; mine.pack(out); detail::send_all_until(fd, out, sizeof(out), deadline, "party_session sctp handshake"); check_match(theirs, mine, "party " + std::to_string(p)); sec_[p] = link_security{}; links_[p] = std::make_unique(*io_, fd, opt_.n_lanes, policy_[p]); log_link_up("accept", "p" + std::to_string(p), transport::sctp, opt_.n_lanes, 0, epoch_[p], fd, policy_[p].socket); } catch (...) { ::close(fd); throw; } } void accept_one_edge(unsigned peer) { listen(published_); if (kind_[peer] == transport::sctp) { accept_sctp(peer, true); return; } std::map partial; for (std::size_t k = 0; k < sockets_for(peer); ++k) { asio::ip::tcp::socket sock(*io_); accept_until(*acceptor_, sock, opt_.limits.accept); accept_tcp(std::move(sock), partial, true, peer); } } void connect_edge(unsigned peer) { const auto & addr = table_[peer]; if (addr.port == 0) throw std::logic_error("party_session: no address for party " + std::to_string(peer)); const std::string who = "party " + std::to_string(peer) + " at " + addr.host + ":" + std::to_string(addr.port); if (kind_[peer] == transport::sctp) { const int fd = connect_sctp(addr.host, addr.port, opt_.n_lanes, opt_.limits.connect, policy_[peer].socket); try { const auto mine = my_hello(peer, 0); std::uint8_t out[detail::session_hello::size]; std::uint8_t in[detail::session_hello::size]; mine.pack(out); exchange_record(fd, out, in, detail::session_hello::size, opt_.limits.handshake, "sctp handshake with " + who); const auto got = detail::session_hello::unpack(in); if (got.id != peer) throw std::runtime_error("party_session: " + who + " is " + detail::id_name(got.id)); check_match(got, mine, who); sec_[peer] = link_security{}; links_[peer] = std::make_unique(*io_, fd, opt_.n_lanes, policy_[peer]); log_link_up("connect", "p" + std::to_string(peer), transport::sctp, opt_.n_lanes, 0, epoch_[peer], fd, policy_[peer].socket); } catch (...) { ::close(fd); throw; } return; } std::vector> conns; for (std::size_t k = 0; k < sockets_for(peer); ++k) { asio::ip::tcp::socket sock(*io_); connect_until(sock, addr.host, addr.port, opt_.limits.connect); auto c = std::make_unique(std::move(sock)); const auto deadline = setup_clock::now() + opt_.limits.handshake; detail::secure(*c, *io_, tls_ptr(), false, policy_[peer].socket, opt_.limits.handshake, "handshake with " + who); detail::check_party(*c, opt_.security, peer, who); auto mine = my_hello(peer, static_cast(k)); mine.flags = c->sec.peer_auth == "key" ? detail::session_hello::verified_you : 0; detail::send_hello(*c, *io_, mine, deadline, "handshake with " + who); const auto got = detail::recv_hello(*c, *io_, deadline, "handshake with " + who); if (got.id != peer) throw std::runtime_error("party_session: " + who + " is " + detail::id_name(got.id)); check_match(got, mine, who); c->sec.peer_verified_us = (got.flags & detail::session_hello::verified_you) != 0; conns.push_back(std::move(c)); } if (kind_[peer] == transport::mux) { auto & c = *conns.front(); sec_[peer] = c.sec; links_[peer] = detail::mux_link(*io_, c, me_, peer, opt_.n_lanes, policy_[peer]); const std::string role = "p" + std::to_string(peer); log_link_up("connect", role, kind_[peer], opt_.n_lanes, 0, epoch_[peer], c.fd, policy_[peer].socket, &sec_[peer]); detail::note_link(role, sec_[peer]); } else adopt_parallel(peer, conns, "connect"); } asio::io_context * io_ = nullptr; asio::io_context * dealer_io_ = nullptr; unsigned me_ = 0; unsigned parties_ = 0; session_options opt_; std::shared_ptr self_; #if DPF_HAS_OPENSSL std::shared_ptr tls_; #endif unsigned short published_ = 0; std::vector table_; std::vector epoch_; std::vector kind_; std::vector policy_; std::unique_ptr acceptor_; std::unique_ptr sctp_; std::vector sec_; std::vector> links_; std::unique_ptr dealer_; link_security dealer_sec_{}; wire_policy dealer_policy_{}; peer_address dealer_addr_{}; std::uint32_t dealer_epoch_ = 0; bool dealer_is_server_ = false; }; /// @brief Dealer serving several parties, each on its own link and epoch. /// @details Links are encrypted like party links; the dealer authenticates a /// party whose key is in `opt.security.trusted[p]`. class dealer_session { public: dealer_session(asio::io_context & io, unsigned served, session_options opt = {}) : io_(&io), served_(served), opt_(std::move(opt)) { if (served == 0) throw std::invalid_argument("dealer_session: serves no party"); opt_.policy.validate(); links_.resize(served); sec_.resize(served); epoch_.assign(served, 0); if (opt_.security.encrypt) { #if DPF_HAS_OPENSSL self_ = detail::link_identity(opt_.security, "dealer"); tls_ = make_peer_tls_context(*self_); #else throw std::logic_error("dealer_session: built without OpenSSL; set " "encryption=off for plaintext links"); #endif } } unsigned short listen(unsigned short port = 0) { if (!acceptor_) { acceptor_ = std::make_unique(*io_); open_listener(*acceptor_, port); log_listen(acceptor_->local_endpoint().port(), false, opt_.security.encrypt); } return acceptor_->local_endpoint().port(); } /// @brief Accept one link from every served party (any order). void accept_parties() { listen(0); for (unsigned k = 0; k < served_; ++k) accept_any(false, 0); } async_stream_array & party(unsigned p) { check(p); if (!links_[p]) throw std::logic_error("dealer_session: no link to party " + std::to_string(p)); return *links_[p]; } stream_stats party_stats(unsigned p) const { check(p); return links_[p] ? links_[p]->stats() : stream_stats{}; } const link_security & party_security(unsigned p) const { check(p); return sec_[p]; } std::shared_ptr identity_key() const noexcept { return self_; } async_stream_array & reconnect(unsigned p) { check(p); links_[p].reset(); ++epoch_[p]; accept_any(true, p); return *links_[p]; } std::function reconnector( unsigned p) { return [this, p](const std::error_code &) { return &reconnect(p); }; } private: void check(unsigned p) const { if (p >= served_) throw std::invalid_argument("dealer_session: bad party " + std::to_string(p)); } void accept_any(bool expect_one, unsigned expect) { asio::ip::tcp::socket sock(*io_); accept_until(*acceptor_, sock, opt_.limits.accept); detail::peer_conn c(std::move(sock)); const auto deadline = setup_clock::now() + opt_.limits.handshake; #if DPF_HAS_OPENSSL detail::secure(c, *io_, tls_.get(), true, opt_.policy.socket, opt_.limits.handshake, "dealer handshake"); #else detail::secure(c, *io_, nullptr, true, opt_.policy.socket, opt_.limits.handshake, "dealer handshake"); #endif const auto theirs = detail::recv_hello(c, *io_, deadline, "dealer handshake"); const unsigned p = theirs.id; if (p >= served_ || (expect_one && p != expect) || (!expect_one && links_[p])) throw std::runtime_error("dealer_session: unexpected " + detail::id_name(p)); detail::check_party(c, opt_.security, p, "party " + std::to_string(p)); detail::session_hello mine; mine.id = party_session::k_dealer_id; mine.epoch = epoch_[p]; mine.kind = static_cast(transport::mux); mine.lanes = static_cast(opt_.n_lanes); mine.flags = c.sec.peer_auth == "key" ? detail::session_hello::verified_you : 0; detail::send_hello(c, *io_, mine, deadline, "dealer handshake"); if (theirs.epoch != mine.epoch || theirs.lanes != mine.lanes || theirs.kind != mine.kind) throw std::runtime_error("dealer_session: party " + std::to_string(p) + " disagrees on epoch, lanes, or transport"); c.sec.peer_verified_us = (theirs.flags & detail::session_hello::verified_you) != 0; sec_[p] = c.sec; links_[p] = detail::mux_link(*io_, c, party_session::k_dealer_id, p, opt_.n_lanes, opt_.policy); const std::string role = "p" + std::to_string(p); log_link_up("accept", role, transport::mux, opt_.n_lanes, 0, epoch_[p], c.fd, opt_.policy.socket, &sec_[p]); detail::note_link(role, sec_[p]); } asio::io_context * io_ = nullptr; unsigned served_ = 0; session_options opt_; std::shared_ptr self_; #if DPF_HAS_OPENSSL std::shared_ptr tls_; #endif std::unique_ptr acceptor_; std::vector sec_; std::vector> links_; std::vector epoch_; }; } // namespace net } // namespace dpf #endif