From 0be39139f2d866ffd088476c4a0e575f19aa64dc Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Wed, 22 Nov 2023 14:47:00 -0500 Subject: [PATCH 01/21] serialization: Support for multiple parameters This commit makes a minimal change to the ParamsStream class to let it retrieve multiple parameters. Followup commits after this commit clean up code using ParamsStream and make it easier to set multiple parameters. Currently it is only possible to attach one serialization parameter to a stream at a time. For example, it is not possible to set a parameter controlling the transaction format and a parameter controlling the address format at the same time because one parameter will override the other. This limitation is inconvenient for multiprocess code since it is not possible to create just one type of stream and serialize any object to it. Instead it is necessary to create different streams for different object types, which requires extra boilerplate and makes using the new parameter fields a lot more awkward than the older version and type fields. Fix this problem by allowing an unlimited number of serialization stream parameters to be set, and allowing them to be requested by type. Later parameters will still override earlier parameters, but only if they have the same type. This change requires replacing the stream.GetParams() method with a stream.GetParams() method in order for serialization code to retreive the desired parameters. This change is more verbose, but probably a good thing for readability because previously it could be difficult to know what type the GetParams() method would return, and now it is more obvious. --- src/netaddress.h | 4 ++-- src/primitives/transaction.h | 6 +++--- src/protocol.h | 3 ++- src/serialize.h | 37 +++++++++++++++--------------------- src/test/serialize_tests.cpp | 7 ++++--- 5 files changed, 26 insertions(+), 31 deletions(-) diff --git a/src/netaddress.h b/src/netaddress.h index 08dd77c0ffab5..5e5c412586f7f 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -241,7 +241,7 @@ class CNetAddr template void Serialize(Stream& s) const { - if (s.GetParams().enc == Encoding::V2) { + if (s.template GetParams().enc == Encoding::V2) { SerializeV2Stream(s); } else { SerializeV1Stream(s); @@ -254,7 +254,7 @@ class CNetAddr template void Unserialize(Stream& s) { - if (s.GetParams().enc == Encoding::V2) { + if (s.template GetParams().enc == Encoding::V2) { UnserializeV2Stream(s); } else { UnserializeV1Stream(s); diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index ccbeb3ec49bb6..d15b8005f90e0 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -326,7 +326,7 @@ class CTransaction template inline void Serialize(Stream& s) const { - SerializeTransaction(*this, s, s.GetParams()); + SerializeTransaction(*this, s, s.template GetParams()); } /** This deserializing constructor is provided instead of an Unserialize method. @@ -386,12 +386,12 @@ struct CMutableTransaction template inline void Serialize(Stream& s) const { - SerializeTransaction(*this, s, s.GetParams()); + SerializeTransaction(*this, s, s.template GetParams()); } template inline void Unserialize(Stream& s) { - UnserializeTransaction(*this, s, s.GetParams()); + UnserializeTransaction(*this, s, s.template GetParams()); } template diff --git a/src/protocol.h b/src/protocol.h index e405253632da0..58a7287d03667 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -406,9 +406,10 @@ class CAddress : public CService static constexpr SerParams V1_DISK{{CNetAddr::Encoding::V1}, Format::Disk}; static constexpr SerParams V2_DISK{{CNetAddr::Encoding::V2}, Format::Disk}; - SERIALIZE_METHODS_PARAMS(CAddress, obj, SerParams, params) + SERIALIZE_METHODS(CAddress, obj) { bool use_v2; + auto& params = SER_PARAMS(SerParams); if (params.fmt == Format::Disk) { // In the disk serialization format, the encoding (v1 or v2) is determined by a flag version // that's part of the serialization itself. ADDRV2_FORMAT in the stream version only determines diff --git a/src/serialize.h b/src/serialize.h index 19585c630ab2a..b1d6ee211ae62 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -181,9 +181,8 @@ const Out& AsBase(const In& x) static void SerializationOps(Type& obj, Stream& s, Operation ser_action) /** - * Variant of FORMATTER_METHODS that supports a declared parameter type. - * - * If a formatter has a declared parameter type, it must be invoked directly or + * Formatter methods can retrieve parameters attached to a stream using the + * SER_PARAMS(type) macro as long as the stream is created directly or * indirectly with a parameter of that type. This permits making serialization * depend on run-time context in a type-safe way. * @@ -191,7 +190,8 @@ const Out& AsBase(const In& x) * struct BarParameter { bool fancy; ... }; * struct Bar { ... }; * struct FooFormatter { - * FORMATTER_METHODS(Bar, obj, BarParameter, param) { + * FORMATTER_METHODS(Bar, obj) { + * auto& param = SER_PARAMS(BarParameter); * if (param.fancy) { * READWRITE(VARINT(obj.value)); * } else { @@ -213,13 +213,7 @@ const Out& AsBase(const In& x) * Compilation will fail in any context where serialization is invoked but * no parameter of a type convertible to BarParameter is provided. */ -#define FORMATTER_METHODS_PARAMS(cls, obj, paramcls, paramobj) \ - template \ - static void Ser(Stream& s, const cls& obj) { SerializationOps(obj, s, ActionSerialize{}, s.GetParams()); } \ - template \ - static void Unser(Stream& s, cls& obj) { SerializationOps(obj, s, ActionUnserialize{}, s.GetParams()); } \ - template \ - static void SerializationOps(Type& obj, Stream& s, Operation ser_action, const paramcls& paramobj) +#define SER_PARAMS(type) (s.template GetParams()) #define BASE_SERIALIZE_METHODS(cls) \ template \ @@ -246,15 +240,6 @@ const Out& AsBase(const In& x) BASE_SERIALIZE_METHODS(cls) \ FORMATTER_METHODS(cls, obj) -/** - * Variant of SERIALIZE_METHODS that supports a declared parameter type. - * - * See FORMATTER_METHODS_PARAMS for more information on parameters. - */ -#define SERIALIZE_METHODS_PARAMS(cls, obj, paramcls, paramobj) \ - BASE_SERIALIZE_METHODS(cls) \ - FORMATTER_METHODS_PARAMS(cls, obj, paramcls, paramobj) - // Templates for serializing to anything that looks like a stream, // i.e. anything that supports .read(Span) and .write(Span) // @@ -1134,7 +1119,15 @@ class ParamsStream void ignore(size_t num) { m_substream.ignore(num); } bool eof() const { return m_substream.eof(); } size_t size() const { return m_substream.size(); } - const Params& GetParams() const { return m_params; } + template + const auto& GetParams() const + { + if constexpr (std::is_convertible_v) { + return m_params; + } else { + return m_substream.template GetParams

(); + } + } int GetVersion() = delete; // Deprecated with Params usage int GetType() = delete; // Deprecated with Params usage }; @@ -1176,7 +1169,7 @@ class ParamsWrapper /** \ * Return a wrapper around t that (de)serializes it with specified parameter params. \ * \ - * See FORMATTER_METHODS_PARAMS for more information on serialization parameters. \ + * See SER_PARAMS for more information on serialization parameters. \ */ \ template \ auto operator()(T&& t) const \ diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index d75eb499b4a3c..18356f2df03fb 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -289,7 +289,7 @@ class Base template void Serialize(Stream& s) const { - if (s.GetParams().m_base_format == BaseFormat::RAW) { + if (s.template GetParams().m_base_format == BaseFormat::RAW) { s << m_base_data; } else { s << Span{HexStr(Span{&m_base_data, 1})}; @@ -299,7 +299,7 @@ class Base template void Unserialize(Stream& s) { - if (s.GetParams().m_base_format == BaseFormat::RAW) { + if (s.template GetParams().m_base_format == BaseFormat::RAW) { s >> m_base_data; } else { std::string hex{"aa"}; @@ -327,8 +327,9 @@ class Derived : public Base public: std::string m_derived_data; - SERIALIZE_METHODS_PARAMS(Derived, obj, DerivedAndBaseFormat, fmt) + SERIALIZE_METHODS(Derived, obj) { + auto& fmt = SER_PARAMS(DerivedAndBaseFormat); READWRITE(fmt.m_base_format(AsBase(obj))); if (ser_action.ForRead()) { From 422bd2d7b8ef81a8594b42887265e6e9559a5b01 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Mon, 20 Nov 2023 14:54:37 -0500 Subject: [PATCH 02/21] test: add ipc test to test multiprocess type conversion code Add unit test to test IPC method calls and type conversion between bitcoin c++ types and capnproto messages. Right now there are custom type hooks in bitcoin IPC code, so the test is simple, but in upcoming commits, code will be added to convert bitcoin types to capnproto messages, and the test will be expanded. --- build_msvc/test_bitcoin/test_bitcoin.vcxproj | 2 +- src/Makefile.test.include | 36 +++++++++++++ src/test/.gitignore | 2 + src/test/ipc_test.capnp | 15 ++++++ src/test/ipc_test.cpp | 57 ++++++++++++++++++++ src/test/ipc_test.h | 18 +++++++ src/test/ipc_tests.cpp | 13 +++++ 7 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 src/test/.gitignore create mode 100644 src/test/ipc_test.capnp create mode 100644 src/test/ipc_test.cpp create mode 100644 src/test/ipc_test.h create mode 100644 src/test/ipc_tests.cpp diff --git a/build_msvc/test_bitcoin/test_bitcoin.vcxproj b/build_msvc/test_bitcoin/test_bitcoin.vcxproj index 2a78f6f2a1e93..0ae3819e50d42 100644 --- a/build_msvc/test_bitcoin/test_bitcoin.vcxproj +++ b/build_msvc/test_bitcoin/test_bitcoin.vcxproj @@ -10,7 +10,7 @@ - + diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 416a11b0c0112..b8afc47bd4e51 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -216,6 +216,39 @@ BITCOIN_TEST_SUITE += \ wallet/test/init_test_fixture.h endif # ENABLE_WALLET +if BUILD_MULTIPROCESS +# Add boost ipc_tests definition to BITCOIN_TESTS +BITCOIN_TESTS += test/ipc_tests.cpp + +# Build ipc_test code in a separate library so it can be compiled with custom +# LIBMULTIPROCESS_CFLAGS without those flags affecting other tests +LIBBITCOIN_IPC_TEST=libbitcoin_ipc_test.a +EXTRA_LIBRARIES += $(LIBBITCOIN_IPC_TEST) +libbitcoin_ipc_test_a_SOURCES = \ + test/ipc_test.cpp \ + test/ipc_test.h +libbitcoin_ipc_test_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +libbitcoin_ipc_test_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) $(LIBMULTIPROCESS_CFLAGS) + +# Generate various .c++/.h files from the ipc_test.capnp file +include $(MPGEN_PREFIX)/include/mpgen.mk +EXTRA_DIST += test/ipc_test.capnp +libbitcoin_ipc_test_mpgen_output = \ + test/ipc_test.capnp.c++ \ + test/ipc_test.capnp.h \ + test/ipc_test.capnp.proxy-client.c++ \ + test/ipc_test.capnp.proxy-server.c++ \ + test/ipc_test.capnp.proxy-types.c++ \ + test/ipc_test.capnp.proxy-types.h \ + test/ipc_test.capnp.proxy.h +nodist_libbitcoin_ipc_test_a_SOURCES = $(libbitcoin_ipc_test_mpgen_output) +CLEANFILES += $(libbitcoin_ipc_test_mpgen_output) +endif + +# Explicitly list dependencies on generated headers as described in +# https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html#Recording-Dependencies-manually +test/libbitcoin_ipc_test_a-ipc_test.$(OBJEXT): test/ipc_test.capnp.h + test_test_bitcoin_SOURCES = $(BITCOIN_TEST_SUITE) $(BITCOIN_TESTS) $(JSON_TEST_FILES) $(RAW_TEST_FILES) test_test_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(TESTDEFS) $(BOOST_CPPFLAGS) $(EVENT_CFLAGS) test_test_bitcoin_LDADD = $(LIBTEST_UTIL) @@ -223,6 +256,9 @@ if ENABLE_WALLET test_test_bitcoin_LDADD += $(LIBBITCOIN_WALLET) test_test_bitcoin_CPPFLAGS += $(BDB_CPPFLAGS) endif +if BUILD_MULTIPROCESS +test_test_bitcoin_LDADD += $(LIBBITCOIN_IPC_TEST) $(LIBMULTIPROCESS_LIBS) +endif test_test_bitcoin_LDADD += $(LIBBITCOIN_NODE) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) \ $(LIBLEVELDB) $(LIBMEMENV) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) $(MINISKETCH_LIBS) diff --git a/src/test/.gitignore b/src/test/.gitignore new file mode 100644 index 0000000000000..036df1430c561 --- /dev/null +++ b/src/test/.gitignore @@ -0,0 +1,2 @@ +# capnp generated files +*.capnp.* diff --git a/src/test/ipc_test.capnp b/src/test/ipc_test.capnp new file mode 100644 index 0000000000000..f8473a4dec349 --- /dev/null +++ b/src/test/ipc_test.capnp @@ -0,0 +1,15 @@ +# Copyright (c) 2023 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +@0xd71b0fc8727fdf83; + +using Cxx = import "/capnp/c++.capnp"; +$Cxx.namespace("gen"); + +using Proxy = import "/mp/proxy.capnp"; +$Proxy.include("test/ipc_test.h"); + +interface FooInterface $Proxy.wrap("FooImplementation") { + add @0 (a :Int32, b :Int32) -> (result :Int32); +} diff --git a/src/test/ipc_test.cpp b/src/test/ipc_test.cpp new file mode 100644 index 0000000000000..b84255f68b6fe --- /dev/null +++ b/src/test/ipc_test.cpp @@ -0,0 +1,57 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +//! Unit test that tests execution of IPC calls without actually creating a +//! separate process. This test is primarily intended to verify behavior of type +//! conversion code that converts C++ objects to Cap'n Proto messages and vice +//! versa. +//! +//! The test creates a thread which creates a FooImplementation object (defined +//! in ipc_test.h) and a two-way pipe accepting IPC requests which call methods +//! on the object through FooInterface (defined in ipc_test.capnp). +void IpcTest() +{ + // Setup: create FooImplemention object and listen for FooInterface requests + std::promise>> foo_promise; + std::function disconnect_client; + std::thread thread([&]() { + mp::EventLoop loop("IpcTest", [](bool raise, const std::string& log) { LogPrintf("LOG%i: %s\n", raise, log); }); + auto pipe = loop.m_io_context.provider->newTwoWayPipe(); + + auto connection_client = std::make_unique(loop, kj::mv(pipe.ends[0])); + auto foo_client = std::make_unique>( + connection_client->m_rpc_system.bootstrap(mp::ServerVatId().vat_id).castAs(), + connection_client.get(), /* destroy_connection= */ false); + foo_promise.set_value(std::move(foo_client)); + disconnect_client = [&] { loop.sync([&] { connection_client.reset(); }); }; + + auto connection_server = std::make_unique(loop, kj::mv(pipe.ends[1]), [&](mp::Connection& connection) { + auto foo_server = kj::heap>(std::make_shared(), connection); + return capnp::Capability::Client(kj::mv(foo_server)); + }); + connection_server->onDisconnect([&] { connection_server.reset(); }); + loop.loop(); + }); + std::unique_ptr> foo{foo_promise.get_future().get()}; + + // Test: make sure arguments were sent and return value is received + BOOST_CHECK_EQUAL(foo->add(1, 2), 3); + + // Test cleanup: disconnect pipe and join thread + disconnect_client(); + thread.join(); +} diff --git a/src/test/ipc_test.h b/src/test/ipc_test.h new file mode 100644 index 0000000000000..61c85b5a47a9d --- /dev/null +++ b/src/test/ipc_test.h @@ -0,0 +1,18 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_IPC_TEST_H +#define BITCOIN_TEST_IPC_TEST_H + +#include + +class FooImplementation +{ +public: + int add(int a, int b) { return a + b; } +}; + +void IpcTest(); + +#endif // BITCOIN_TEST_IPC_TEST_H diff --git a/src/test/ipc_tests.cpp b/src/test/ipc_tests.cpp new file mode 100644 index 0000000000000..6e144b0f418b6 --- /dev/null +++ b/src/test/ipc_tests.cpp @@ -0,0 +1,13 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include + +BOOST_AUTO_TEST_SUITE(ipc_tests) +BOOST_AUTO_TEST_CASE(ipc_tests) +{ + IpcTest(); +} +BOOST_AUTO_TEST_SUITE_END() From de13c174c809dcacc82db3f330036ec71d519dc1 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Wed, 22 Nov 2023 16:39:32 -0500 Subject: [PATCH 03/21] serialization: Drop unnecessary ParamsStream references Drop unnecessary ParamsStream references from CTransaction and CMutableTransaction constructors. This just couples these classes unnecessarily to the ParamsStream class, making the ParamsStream class harder to modify, and making the transaction classes in some cases (depending on parameter order) unable to work with stream classes that have multiple parameters set. --- src/primitives/transaction.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index d15b8005f90e0..976542cfae676 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -334,7 +334,7 @@ class CTransaction template CTransaction(deserialize_type, const TransactionSerParams& params, Stream& s) : CTransaction(CMutableTransaction(deserialize, params, s)) {} template - CTransaction(deserialize_type, ParamsStream& s) : CTransaction(CMutableTransaction(deserialize, s)) {} + CTransaction(deserialize_type, Stream& s) : CTransaction(CMutableTransaction(deserialize, s)) {} bool IsNull() const { return vin.empty() && vout.empty(); @@ -400,7 +400,7 @@ struct CMutableTransaction } template - CMutableTransaction(deserialize_type, ParamsStream& s) { + CMutableTransaction(deserialize_type, Stream& s) { Unserialize(s); } From c9dad8d3cfcaf329b025b091dcffbafe7afdb2de Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Mon, 20 Nov 2023 15:49:55 -0500 Subject: [PATCH 04/21] multiprocess: Add type conversion code for serializable types Allow any C++ object that has Serialize and Unserialize methods and can be serialized to a bitcoin CDataStream to be converted to a capnproto Data field and passed as arguments or return values to capnproto methods using the Data type. Extend IPC unit test to cover this and verify the serialization happens correctly. --- src/Makefile.am | 1 + src/ipc/capnp/common-types.h | 89 ++++++++++++++++++++++++++++++++++++ src/test/ipc_test.capnp | 2 + src/test/ipc_test.cpp | 6 ++- src/test/ipc_test.h | 1 + 5 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 src/ipc/capnp/common-types.h diff --git a/src/Makefile.am b/src/Makefile.am index b6f0daaabadbd..39c00fea61697 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1090,6 +1090,7 @@ ipc/capnp/libbitcoin_ipc_a-protocol.$(OBJEXT): $(libbitcoin_ipc_mpgen_input:=.h) if BUILD_MULTIPROCESS LIBBITCOIN_IPC=libbitcoin_ipc.a libbitcoin_ipc_a_SOURCES = \ + ipc/capnp/common-types.h \ ipc/capnp/context.h \ ipc/capnp/init-types.h \ ipc/capnp/protocol.cpp \ diff --git a/src/ipc/capnp/common-types.h b/src/ipc/capnp/common-types.h new file mode 100644 index 0000000000000..d1343c40dd1a0 --- /dev/null +++ b/src/ipc/capnp/common-types.h @@ -0,0 +1,89 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_IPC_CAPNP_COMMON_TYPES_H +#define BITCOIN_IPC_CAPNP_COMMON_TYPES_H + +#include +#include + +#include +#include +#include +#include + +namespace ipc { +namespace capnp { +//! Use SFINAE to define Serializeable trait which is true if type T has a +//! Serialize(stream) method, false otherwise. +template +struct Serializable { +private: + template + static std::true_type test(decltype(std::declval().Serialize(std::declval()))*); + template + static std::false_type test(...); + +public: + static constexpr bool value = decltype(test(nullptr))::value; +}; + +//! Use SFINAE to define Unserializeable trait which is true if type T has +//! an Unserialize(stream) method, false otherwise. +template +struct Unserializable { +private: + template + static std::true_type test(decltype(std::declval().Unserialize(std::declval()))*); + template + static std::false_type test(...); + +public: + static constexpr bool value = decltype(test(nullptr))::value; +}; +} // namespace capnp +} // namespace ipc + +//! Functions to serialize / deserialize common bitcoin types. +namespace mp { +//! Overload multiprocess library's CustomBuildField hook to allow any +//! serializable object to be stored in a capnproto Data field or passed to a +//! canproto interface. Use Priority<1> so this hook has medium priority, and +//! higher priority hooks could take precedence over this one. +template +void CustomBuildField( + TypeList, Priority<1>, InvokeContext& invoke_context, Value&& value, Output&& output, + // Enable if serializeable and if LocalType is not cv or reference + // qualified. If LocalType is cv or reference qualified, it is important to + // fall back to lower-priority Priority<0> implementation of this function + // that strips cv references, to prevent this CustomBuildField overload from + // taking precedence over more narrow overloads for specific LocalTypes. + std::enable_if_t::value && + std::is_same_v>>>* enable = nullptr) +{ + DataStream stream; + value.Serialize(stream); + auto result = output.init(stream.size()); + memcpy(result.begin(), stream.data(), stream.size()); +} + +//! Overload multiprocess library's CustomReadField hook to allow any object +//! with an Unserialize method to be read from a capnproto Data field or +//! returned from canproto interface. Use Priority<1> so this hook has medium +//! priority, and higher priority hooks could take precedence over this one. +template +decltype(auto) +CustomReadField(TypeList, Priority<1>, InvokeContext& invoke_context, Input&& input, ReadDest&& read_dest, + std::enable_if_t::value>* enable = nullptr) +{ + return read_dest.update([&](auto& value) { + if (!input.has()) return; + auto data = input.get(); + SpanReader stream({data.begin(), data.end()}); + value.Unserialize(stream); + }); +} +} // namespace mp + +#endif // BITCOIN_IPC_CAPNP_COMMON_TYPES_H diff --git a/src/test/ipc_test.capnp b/src/test/ipc_test.capnp index f8473a4dec349..7b970e2affe2e 100644 --- a/src/test/ipc_test.capnp +++ b/src/test/ipc_test.capnp @@ -9,7 +9,9 @@ $Cxx.namespace("gen"); using Proxy = import "/mp/proxy.capnp"; $Proxy.include("test/ipc_test.h"); +$Proxy.includeTypes("ipc/capnp/common-types.h"); interface FooInterface $Proxy.wrap("FooImplementation") { add @0 (a :Int32, b :Int32) -> (result :Int32); + passOutPoint @1 (arg :Data) -> (result :Data); } diff --git a/src/test/ipc_test.cpp b/src/test/ipc_test.cpp index b84255f68b6fe..f835859705d49 100644 --- a/src/test/ipc_test.cpp +++ b/src/test/ipc_test.cpp @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include #include +#include #include #include #include @@ -51,6 +51,10 @@ void IpcTest() // Test: make sure arguments were sent and return value is received BOOST_CHECK_EQUAL(foo->add(1, 2), 3); + COutPoint txout1{Txid::FromUint256(uint256{100}), 200}; + COutPoint txout2{foo->passOutPoint(txout1)}; + BOOST_CHECK(txout1 == txout2); + // Test cleanup: disconnect pipe and join thread disconnect_client(); thread.join(); diff --git a/src/test/ipc_test.h b/src/test/ipc_test.h index 61c85b5a47a9d..f100ae8c5d2a8 100644 --- a/src/test/ipc_test.h +++ b/src/test/ipc_test.h @@ -11,6 +11,7 @@ class FooImplementation { public: int add(int a, int b) { return a + b; } + COutPoint passOutPoint(COutPoint o) { return o; } }; void IpcTest(); From 9e9dfce3d090ecce422ab788ec714c5d0b090df8 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Wed, 22 Nov 2023 16:39:32 -0500 Subject: [PATCH 05/21] serialization: Reverse ParamsStream constructor order Move parameter argument after stream argument so will be possible to accept multiple variadic parameter arguments in the following commit. Also reverse template parameter order for consistency. --- src/addrman.cpp | 4 ++-- src/net.cpp | 2 +- src/serialize.h | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/addrman.cpp b/src/addrman.cpp index a8206de6ee291..b649950eced3a 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -171,7 +171,7 @@ void AddrManImpl::Serialize(Stream& s_) const */ // Always serialize in the latest version (FILE_FORMAT). - ParamsStream s{CAddress::V2_DISK, s_}; + ParamsStream s{s_, CAddress::V2_DISK}; s << static_cast(FILE_FORMAT); @@ -236,7 +236,7 @@ void AddrManImpl::Unserialize(Stream& s_) s_ >> Using>(format); const auto ser_params = (format >= Format::V3_BIP155 ? CAddress::V2_DISK : CAddress::V1_DISK); - ParamsStream s{ser_params, s_}; + ParamsStream s{s_, ser_params}; uint8_t compat; s >> compat; diff --git a/src/net.cpp b/src/net.cpp index 0cc794c2c3fcf..ba764e28e195c 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -199,7 +199,7 @@ static std::vector ConvertSeeds(const std::vector &vSeedsIn) std::vector vSeedsOut; FastRandomContext rng; DataStream underlying_stream{vSeedsIn}; - ParamsStream s{CAddress::V2_NETWORK, underlying_stream}; + ParamsStream s{underlying_stream, CAddress::V2_NETWORK}; while (!s.eof()) { CService endpoint; s >> endpoint; diff --git a/src/serialize.h b/src/serialize.h index b1d6ee211ae62..60819899439ed 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -1104,14 +1104,14 @@ size_t GetSerializeSize(const T& t) } /** Wrapper that overrides the GetParams() function of a stream (and hides GetVersion/GetType). */ -template +template class ParamsStream { const Params& m_params; SubStream& m_substream; // private to avoid leaking version/type into serialization code that shouldn't see it public: - ParamsStream(const Params& params LIFETIMEBOUND, SubStream& substream LIFETIMEBOUND) : m_params{params}, m_substream{substream} {} + ParamsStream(SubStream& substream LIFETIMEBOUND, const Params& params LIFETIMEBOUND) : m_params{params}, m_substream{substream} {} template ParamsStream& operator<<(const U& obj) { ::Serialize(*this, obj); return *this; } template ParamsStream& operator>>(U&& obj) { ::Unserialize(*this, obj); return *this; } void write(Span src) { m_substream.write(src); } @@ -1145,13 +1145,13 @@ class ParamsWrapper template void Serialize(Stream& s) const { - ParamsStream ss{m_params, s}; + ParamsStream ss{s, m_params}; ::Serialize(ss, m_object); } template void Unserialize(Stream& s) { - ParamsStream ss{m_params, s}; + ParamsStream ss{s, m_params}; ::Unserialize(ss, m_object); } }; From afce1829195cdcad5cd7864207b7a9468df12784 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Mon, 20 Nov 2023 15:49:55 -0500 Subject: [PATCH 06/21] multiprocess: Add type conversion code for UniValue types Extend IPC unit test to cover this and verify the serialization happens correctly. --- src/ipc/capnp/common-types.h | 19 +++++++++++++++++++ src/test/ipc_test.capnp | 1 + src/test/ipc_test.cpp | 6 ++++++ src/test/ipc_test.h | 2 ++ 4 files changed, 28 insertions(+) diff --git a/src/ipc/capnp/common-types.h b/src/ipc/capnp/common-types.h index d1343c40dd1a0..39e368491b4d0 100644 --- a/src/ipc/capnp/common-types.h +++ b/src/ipc/capnp/common-types.h @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -84,6 +85,24 @@ CustomReadField(TypeList, Priority<1>, InvokeContext& invoke_context, value.Unserialize(stream); }); } + +template +void CustomBuildField(TypeList, Priority<1>, InvokeContext& invoke_context, Value&& value, Output&& output) +{ + std::string str = value.write(); + auto result = output.init(str.size()); + memcpy(result.begin(), str.data(), str.size()); +} + +template +decltype(auto) CustomReadField(TypeList, Priority<1>, InvokeContext& invoke_context, Input&& input, + ReadDest&& read_dest) +{ + return read_dest.update([&](auto& value) { + auto data = input.get(); + value.read(std::string_view{data.begin(), data.size()}); + }); +} } // namespace mp #endif // BITCOIN_IPC_CAPNP_COMMON_TYPES_H diff --git a/src/test/ipc_test.capnp b/src/test/ipc_test.capnp index 7b970e2affe2e..55a3dc2683933 100644 --- a/src/test/ipc_test.capnp +++ b/src/test/ipc_test.capnp @@ -14,4 +14,5 @@ $Proxy.includeTypes("ipc/capnp/common-types.h"); interface FooInterface $Proxy.wrap("FooImplementation") { add @0 (a :Int32, b :Int32) -> (result :Int32); passOutPoint @1 (arg :Data) -> (result :Data); + passUniValue @2 (arg :Text) -> (result :Text); } diff --git a/src/test/ipc_test.cpp b/src/test/ipc_test.cpp index f835859705d49..ce4edaceb0821 100644 --- a/src/test/ipc_test.cpp +++ b/src/test/ipc_test.cpp @@ -55,6 +55,12 @@ void IpcTest() COutPoint txout2{foo->passOutPoint(txout1)}; BOOST_CHECK(txout1 == txout2); + UniValue uni1{UniValue::VOBJ}; + uni1.pushKV("i", 1); + uni1.pushKV("s", "two"); + UniValue uni2{foo->passUniValue(uni1)}; + BOOST_CHECK_EQUAL(uni1.write(), uni2.write()); + // Test cleanup: disconnect pipe and join thread disconnect_client(); thread.join(); diff --git a/src/test/ipc_test.h b/src/test/ipc_test.h index f100ae8c5d2a8..bcfcc2125c68b 100644 --- a/src/test/ipc_test.h +++ b/src/test/ipc_test.h @@ -6,12 +6,14 @@ #define BITCOIN_TEST_IPC_TEST_H #include +#include class FooImplementation { public: int add(int a, int b) { return a + b; } COutPoint passOutPoint(COutPoint o) { return o; } + UniValue passUniValue(UniValue v) { return v; } }; void IpcTest(); From ffc1e9fd8cf53cfcf4d7926ca2963958eacf73d7 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Wed, 22 Nov 2023 17:56:04 -0500 Subject: [PATCH 07/21] serialization: Accept multiple parameters in ParamsStream constructor Before this change it was possible but awkward to create ParamStream streams with multiple parameter objects. After this change it is straightforward. --- src/serialize.h | 27 +++++++++++++++++-- src/test/serialize_tests.cpp | 52 ++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/src/serialize.h b/src/serialize.h index 60819899439ed..051ff155ad9e3 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -1108,10 +1108,15 @@ template class ParamsStream { const Params& m_params; - SubStream& m_substream; // private to avoid leaking version/type into serialization code that shouldn't see it + SubStream m_substream; // private to avoid leaking version/type into serialization code that shouldn't see it public: - ParamsStream(SubStream& substream LIFETIMEBOUND, const Params& params LIFETIMEBOUND) : m_params{params}, m_substream{substream} {} + ParamsStream(SubStream substream LIFETIMEBOUND, const Params& params LIFETIMEBOUND) : m_params{params}, m_substream{substream} {} + + template + ParamsStream(NestedSubstream& s LIFETIMEBOUND, const Params1& params1 LIFETIMEBOUND, const Params2& params2 LIFETIMEBOUND, const NestedParams&... params LIFETIMEBOUND) + : ParamsStream{::ParamsStream{s, params2, params...}, params1} {} + template ParamsStream& operator<<(const U& obj) { ::Serialize(*this, obj); return *this; } template ParamsStream& operator>>(U&& obj) { ::Unserialize(*this, obj); return *this; } void write(Span src) { m_substream.write(src); } @@ -1132,6 +1137,24 @@ class ParamsStream int GetType() = delete; // Deprecated with Params usage }; +/** + * Template deduction guide for a single params argument that's slightly + * different from the default generated deduction guide because it stores a + * reference to the substream inside ParamsStream instead of a copy. (Storing a + * copy instead of a reference is still possible by specifying template + * arguments explicitly and bypassing this deduction guide.) + */ +template +ParamsStream(Substream&, const Params&) -> ParamsStream; + +/** + * Template deduction guide for multiple params arguments that creates a nested + * ParamStream. + */ +template +ParamsStream(Substream& s LIFETIMEBOUND, const Params1& params1 LIFETIMEBOUND, const Params2& params2 LIFETIMEBOUND, const Params&... params LIFETIMEBOUND) -> + ParamsStream; + /** Wrapper that serializes objects with the specified parameters. */ template class ParamsWrapper diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index 18356f2df03fb..bc49ca24fcd56 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -344,6 +344,58 @@ class Derived : public Base } }; +struct OtherParam { + uint8_t param; + SER_PARAMS_OPFUNC +}; + +class Other +{ +public: + template + void Serialize(Stream& s) const + { + const uint8_t param = s.template GetParams().param; + s << param; + } + + template + void Unserialize(Stream& s) + { + const uint8_t param = s.template GetParams().param; + uint8_t value; + s >> value; + BOOST_CHECK_EQUAL(value, param); + } +}; + +//! Test creating a stream with multiple parameters and making sure +//! serialization code requiring different parameters can retrieve them. Also +//! test that earlier parameters take precedence if the same parameter type is +//! specified twice. (Choice of whether earlier or later values take precedence +//! or multiple values of the same type are allowed was arbitrary, and just +//! decided based on what would require smallest amount of ugly C++ template +//! code. Intent of the test is to just ensure there is no unexpected behavior.) +BOOST_AUTO_TEST_CASE(with_params_multi) +{ + const OtherParam other_param_used{.param = 0x07}; + const OtherParam other_param_ignored{.param = 0x09}; + DataStream stream; + ParamsStream pstream{stream, RAW, other_param_used, other_param_ignored}; + + Base b1{0x08}; + Other ou1; + Other oi1; + pstream << b1 << ou1 << other_param_ignored(oi1); + BOOST_CHECK_EQUAL(stream.str(), "\x08\x07\x09"); + + Base b2; + Other ou2; + Other oi2; + pstream >> b2 >> ou2 >> other_param_ignored(oi1); + BOOST_CHECK_EQUAL(b2.m_base_data, 0x08); +} + BOOST_AUTO_TEST_CASE(with_params_base) { Base b{0x0F}; From ee4b9138c837f6dc6b8f063b0df27573736d6578 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Thu, 11 Jan 2024 18:07:20 -0500 Subject: [PATCH 08/21] doc: multiprocess documentation improvements All suggested by stickies-v https://github.com/bitcoin/bitcoin/pull/28978#pullrequestreview-1800375604 Co-authored-by: stickies-v --- doc/design/multiprocess.md | 8 ++++---- doc/multiprocess.md | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/doc/design/multiprocess.md b/doc/design/multiprocess.md index 636d78d905c45..304b86387b4b6 100644 --- a/doc/design/multiprocess.md +++ b/doc/design/multiprocess.md @@ -81,7 +81,7 @@ This section describes the major components of the Inter-Process Communication ( - In the generated code, we have C++ client subclasses that inherit from the abstract classes in [`src/interfaces/`](../../src/interfaces/). These subclasses are the workhorses of the IPC mechanism. - They implement all the methods of the interface, marshalling arguments into a structured format, sending them as requests to the IPC server via a UNIX socket, and handling the responses. - These subclasses effectively mask the complexity of IPC, presenting a familiar C++ interface to developers. -- Internally, the client subclasses generated by the `mpgen` tool wrap [client classes generated by Cap'n Proto](https://capnproto.org/cxxrpc.html#clients), and use them to send IPC requests. +- Internally, the client subclasses generated by the `mpgen` tool wrap [client classes generated by Cap'n Proto](https://capnproto.org/cxxrpc.html#clients), and use them to send IPC requests. The Cap'n Proto client classes are low-level, with non-blocking methods that use asynchronous I/O and pass request and response objects, while mpgen client subclasses provide normal C++ methods that block while executing and convert between request/response objects and arguments/return values. ### C++ Server Classes in Generated Code - On the server side, corresponding generated C++ classes receive IPC requests. These server classes are responsible for unmarshalling method arguments, invoking the corresponding methods in the local [`src/interfaces/`](../../src/interfaces/) objects, and creating the IPC response. @@ -142,7 +142,7 @@ In the current design, class names, method names, and parameter names are duplic An alternate approach could use custom [C++ Attributes](https://en.cppreference.com/w/cpp/language/attributes) embedded in interface declarations to automatically generate `.capnp` files from C++ headers. This has not been pursued because parsing C++ headers is more complicated than parsing Cap’n Proto interface definitions, especially portably on multiple platforms. -In the meantime, the developer guide [Internal interface guidelines](developer-notes.md#internal-interface-guidelines) can provide guidance on keeping interfaces consistent and functional and avoiding compile errors. +In the meantime, the developer guide [Internal interface guidelines](../developer-notes.md#internal-interface-guidelines) can provide guidance on keeping interfaces consistent and functional and avoiding compile errors. ### Interface Stability @@ -197,7 +197,7 @@ sequenceDiagram - Upon receiving the request, the Cap'n Proto dispatching code in the `bitcoin-node` process calls the `getBlockHash` method of the `Chain` [server class](#c-server-classes-in-generated-code). - The server class is automatically generated by the `mpgen` tool from the [`chain.capnp`](https://github.com/ryanofsky/bitcoin/blob/pr/ipc/src/ipc/capnp/chain.capnp) file in [`src/ipc/capnp/`](../../src/ipc/capnp/). - The `getBlockHash` method of the generated `Chain` server subclass in `bitcoin-wallet` receives a Cap’n Proto request object with the `height` parameter, and calls the `getBlockHash` method on its local `Chain` object with the provided `height`. - - When the call returns, it encapsulates the return value in a Cap’n Proto response, which it sends back to the `bitcoin-wallet` process, + - When the call returns, it encapsulates the return value in a Cap’n Proto response, which it sends back to the `bitcoin-wallet` process. 5. **Response and Return** - The `getBlockHash` method of the generated `Chain` client subclass in `bitcoin-wallet` which sent the request now receives the response. @@ -232,7 +232,7 @@ This modularization represents an advancement in Bitcoin Core's architecture, of - **Cap’n Proto struct**: A structured data format used in Cap’n Proto, similar to structs in C++, for organizing and transporting data across different processes. -- **client class (in generated code)**: A C++ class generated from a Cap’n Proto interface which inherits from a Bitcoin core abstract class, and implements each virtual method to send IPC requests to another process. (see also [components section](#c-client-subclasses-in-generated-code)) +- **client class (in generated code)**: A C++ class generated from a Cap’n Proto interface which inherits from a Bitcoin Core abstract class, and implements each virtual method to send IPC requests to another process. (see also [components section](#c-client-subclasses-in-generated-code)) - **IPC (inter-process communication)**: Mechanisms that enable processes to exchange requests and data. diff --git a/doc/multiprocess.md b/doc/multiprocess.md index 7ba89b3ff56a2..6186b31712433 100644 --- a/doc/multiprocess.md +++ b/doc/multiprocess.md @@ -17,7 +17,9 @@ The multiprocess feature requires [Cap'n Proto](https://capnproto.org/) and [lib ``` cd make -C depends NO_QT=1 MULTIPROCESS=1 -CONFIG_SITE=$PWD/depends/x86_64-pc-linux-gnu/share/config.site ./configure +# Set host platform to output of gcc -dumpmachine or clang -dumpmachine or check the depends/ directory for the generated subdirectory name +HOST_PLATFORM="x86_64-pc-linux-gnu" +CONFIG_SITE="$PWD/depends/$HOST_PLATFORM/share/config.site" ./configure make src/bitcoin-node -regtest -printtoconsole -debug=ipc BITCOIND=bitcoin-node test/functional/test_runner.py From df27e301e603d460cbd187056aeff20087cefe3e Mon Sep 17 00:00:00 2001 From: Russell Yanofsky Date: Tue, 5 Dec 2017 15:57:12 -0500 Subject: [PATCH 09/21] Increase feature_block.py and feature_taproot.py timeouts Needed because BlockConnected notifications are a lot slower with the wallet running in separate process. --- test/functional/feature_block.py | 12 ++++++++++-- test/functional/feature_taproot.py | 1 + test/functional/wallet_transactiontime_rescan.py | 1 + 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index 58ef1e761d830..974cc95eb5e0a 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -90,6 +90,7 @@ def set_test_params(self): '-acceptnonstdtxn=1', # This is a consensus block test, we don't care about tx policy '-testactivationheight=bip34@2', ]] + self.rpc_timeout = 1920 def run_test(self): node = self.nodes[0] # convenience reference to the node @@ -1290,7 +1291,7 @@ def run_test(self): blocks2 = [] for i in range(89, LARGE_REORG_SIZE + 89): blocks2.append(self.next_block("alt" + str(i))) - self.send_blocks(blocks2, False, force_send=False) + self.send_blocks(blocks2, False, force_send=False, timeout=1920) # extend alt chain to trigger re-org block = self.next_block("alt" + str(chain1_tip + 1)) @@ -1299,7 +1300,7 @@ def run_test(self): # ... and re-org back to the first chain self.move_tip(chain1_tip) block = self.next_block(chain1_tip + 1) - self.send_blocks([block], False, force_send=True) + self.send_blocks([block], False, force_send=True, timeout=1920) block = self.next_block(chain1_tip + 2) self.send_blocks([block], True, timeout=2440) @@ -1315,6 +1316,13 @@ def run_test(self): b_cb34.solve() self.send_blocks([b_cb34], success=False, reject_reason='bad-cb-height', reconnect=True) + # Flush the notification queue before shutting down, so the + # FlushBackgroundCallbacks call made during shutdown won't exceed the + # test framework's 60 second shutdown timeout on slow systems, due to + # all the BlockConnected notifications generated during the test. + self.log.info("Wait for BlockConnected notifications to be processed before shutdown") + self.nodes[0].syncwithvalidationinterfacequeue() + # Helper methods ################ diff --git a/test/functional/feature_taproot.py b/test/functional/feature_taproot.py index e85541d0ec258..0500fb44e906f 100755 --- a/test/functional/feature_taproot.py +++ b/test/functional/feature_taproot.py @@ -1286,6 +1286,7 @@ def set_test_params(self): self.num_nodes = 1 self.setup_clean_chain = True self.extra_args = [["-par=1"]] + self.rpc_timeout = 120 def block_submit(self, node, txs, msg, err_msg, cb_pubkey=None, fees=0, sigops_weight=0, witness=False, accept=False): diff --git a/test/functional/wallet_transactiontime_rescan.py b/test/functional/wallet_transactiontime_rescan.py index ea99992084865..10dde5e1e17ef 100755 --- a/test/functional/wallet_transactiontime_rescan.py +++ b/test/functional/wallet_transactiontime_rescan.py @@ -32,6 +32,7 @@ def set_test_params(self): ["-keypool=400"], [] ] + self.rpc_timeout = 120 def skip_test_if_missing_module(self): self.skip_if_no_wallet() From cbc5e366a21660601bee8270b5f1d183c148d092 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Tue, 23 Aug 2022 22:02:24 -0400 Subject: [PATCH 10/21] Add util::Result workaround to be compatible with libmultiprocess Make default constructor more generic so it doesn't only work with void types. --- src/Makefile.am | 2 +- src/util/result.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Makefile.am b/src/Makefile.am index 39c00fea61697..13825dbff1770 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1101,7 +1101,7 @@ libbitcoin_ipc_a_SOURCES = \ ipc/process.cpp \ ipc/process.h \ ipc/protocol.h -libbitcoin_ipc_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +libbitcoin_ipc_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DLIBMULTIPROCESS_IPC libbitcoin_ipc_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) $(LIBMULTIPROCESS_CFLAGS) include $(MPGEN_PREFIX)/include/mpgen.mk diff --git a/src/util/result.h b/src/util/result.h index b99995c7e5620..5f61c3e0055a8 100644 --- a/src/util/result.h +++ b/src/util/result.h @@ -43,7 +43,7 @@ class Result friend bilingual_str ErrorString(const Result& result); public: - Result() : m_variant{std::in_place_index_t<1>{}, std::monostate{}} {} // constructor for void + Result() : m_variant{std::in_place_index_t<1>{}, T{}} {} Result(T obj) : m_variant{std::in_place_index_t<1>{}, std::move(obj)} {} Result(Error error) : m_variant{std::in_place_index_t<0>{}, std::move(error.message)} {} From 3427e7e0a96502b9c8c6a9d74e5f21a8d84de928 Mon Sep 17 00:00:00 2001 From: Russell Yanofsky Date: Tue, 5 Dec 2017 15:57:12 -0500 Subject: [PATCH 11/21] Add capnp serialization code for bitcoin types --- contrib/devtools/circular-dependencies.py | 2 +- src/Makefile.am | 4 + src/ipc/capnp/common-types.h | 344 +++++++++++++++++++++- src/ipc/capnp/common.capnp | 49 +++ src/ipc/capnp/common.cpp | 49 +++ src/ipc/capnp/common.h | 29 ++ 6 files changed, 473 insertions(+), 4 deletions(-) create mode 100644 src/ipc/capnp/common.capnp create mode 100644 src/ipc/capnp/common.cpp create mode 100644 src/ipc/capnp/common.h diff --git a/contrib/devtools/circular-dependencies.py b/contrib/devtools/circular-dependencies.py index b742a8cea6718..1f69bf4b21b01 100755 --- a/contrib/devtools/circular-dependencies.py +++ b/contrib/devtools/circular-dependencies.py @@ -15,7 +15,7 @@ # define functions and variables declared in corresponding .h files is # incorrect. HEADER_MODULE_PATHS = [ - 'interfaces/' + 'ipc/' ] def module_name(path): diff --git a/src/Makefile.am b/src/Makefile.am index 13825dbff1770..cc429807e35d9 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1078,6 +1078,7 @@ if HARDEN endif libbitcoin_ipc_mpgen_input = \ + ipc/capnp/common.capnp \ ipc/capnp/echo.capnp \ ipc/capnp/init.capnp EXTRA_DIST += $(libbitcoin_ipc_mpgen_input) @@ -1085,11 +1086,14 @@ EXTRA_DIST += $(libbitcoin_ipc_mpgen_input) # Explicitly list dependencies on generated headers as described in # https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html#Recording-Dependencies-manually +ipc/capnp/libbitcoin_ipc_a-common.$(OBJEXT): $(libbitcoin_ipc_mpgen_input:=.h) ipc/capnp/libbitcoin_ipc_a-protocol.$(OBJEXT): $(libbitcoin_ipc_mpgen_input:=.h) if BUILD_MULTIPROCESS LIBBITCOIN_IPC=libbitcoin_ipc.a libbitcoin_ipc_a_SOURCES = \ + ipc/capnp/common.cpp \ + ipc/capnp/common.h \ ipc/capnp/common-types.h \ ipc/capnp/context.h \ ipc/capnp/init-types.h \ diff --git a/src/ipc/capnp/common-types.h b/src/ipc/capnp/common-types.h index 39e368491b4d0..d7414a1f3eba4 100644 --- a/src/ipc/capnp/common-types.h +++ b/src/ipc/capnp/common-types.h @@ -5,9 +5,19 @@ #ifndef BITCOIN_IPC_CAPNP_COMMON_TYPES_H #define BITCOIN_IPC_CAPNP_COMMON_TYPES_H +#include #include +#include +#include +#include +#include +#include #include #include +#include +#include +#include +#include #include #include @@ -16,6 +26,56 @@ namespace ipc { namespace capnp { +//! Convert kj::StringPtr to std::string. +inline std::string ToString(const kj::StringPtr& str) { return {str.cStr(), str.size()}; } + +//! Convert kj::ArrayPtr to std::string. +inline std::string ToString(const kj::ArrayPtr& array) +{ + return {reinterpret_cast(array.begin()), array.size()}; +} + +//! Convert kj::ArrayPtr to base_blob. +template +inline T ToBlob(const kj::ArrayPtr& array) +{ + return T({array.begin(), array.begin() + array.size()}); +} + +//! Convert base_blob to kj::ArrayPtr. +template +inline kj::ArrayPtr ToArray(const T& blob) +{ + return {reinterpret_cast(blob.data()), blob.size()}; +} + +template +auto Wrap(S& s) +{ + return ParamsStream{s, TX_WITH_WITNESS, CAddress::V2_NETWORK}; +} + +//! Serialize bitcoin value. +template +DataStream Serialize(const T& value) +{ + DataStream stream; + auto wrapper{Wrap(stream)}; + value.Serialize(wrapper); + return stream; +} + +//! Deserialize bitcoin value. +template +T Unserialize(const kj::ArrayPtr& data) +{ + SpanReader stream{{data.begin(), data.end()}}; + T value; + auto wrapper{Wrap(stream)}; + value.Unserialize(wrapper); + return value; +} + //! Use SFINAE to define Serializeable trait which is true if type T has a //! Serialize(stream) method, false otherwise. template @@ -43,6 +103,9 @@ struct Unserializable { public: static constexpr bool value = decltype(test(nullptr))::value; }; + +template +using Deserializable = std::is_constructible; } // namespace capnp } // namespace ipc @@ -64,7 +127,8 @@ void CustomBuildField( std::is_same_v>>>* enable = nullptr) { DataStream stream; - value.Serialize(stream); + auto wrapper{ipc::capnp::Wrap(stream)}; + value.Serialize(wrapper); auto result = output.init(stream.size()); memcpy(result.begin(), stream.data(), stream.size()); } @@ -76,16 +140,97 @@ void CustomBuildField( template decltype(auto) CustomReadField(TypeList, Priority<1>, InvokeContext& invoke_context, Input&& input, ReadDest&& read_dest, - std::enable_if_t::value>* enable = nullptr) + std::enable_if_t::value && + !ipc::capnp::Deserializable::value>* enable = nullptr) { return read_dest.update([&](auto& value) { if (!input.has()) return; auto data = input.get(); SpanReader stream({data.begin(), data.end()}); - value.Unserialize(stream); + auto wrapper{ipc::capnp::Wrap(stream)}; + value.Unserialize(wrapper); }); } +template +decltype(auto) +CustomReadField(TypeList, Priority<1>, InvokeContext& invoke_context, Input&& input, ReadDest&& read_dest, + std::enable_if_t::value>* enable = nullptr) +{ + assert(input.has()); + auto data = input.get(); + SpanReader stream({data.begin(), data.end()}); + // TODO: instead of always preferring Deserialize implementation over + // Unserialize should prefer Deserializing when emplacing, unserialize when + // updating. Can implement by adding read_dest.alreadyConstructed() + // constexpr bool method in libmultiprocess. + auto wrapper{ipc::capnp::Wrap(stream)}; + return read_dest.construct(deserialize, wrapper); +} + +template +void CustomBuildField(TypeList, Priority<1>, InvokeContext& invoke_context, Value&& value, + Output&& output) +{ + output.set(value.count()); +} + +template +decltype(auto) CustomReadField(TypeList, Priority<1>, InvokeContext& invoke_context, + Input&& input, ReadDest&& read_dest) +{ + return read_dest.construct(input.get()); +} + +template +void CustomBuildField(TypeList, Priority<1>, InvokeContext& invoke_context, Value&& value, + Output&& output) +{ + output.set(value.count()); +} + +template +decltype(auto) CustomReadField(TypeList, Priority<1>, InvokeContext& invoke_context, + Input&& input, ReadDest&& read_dest) +{ + return read_dest.construct(input.get()); +} + +template +void CustomBuildField(TypeList, Priority<1>, InvokeContext& invoke_context, Value&& path, Output&& output) +{ + std::string str = fs::PathToString(path); + auto result = output.init(str.size()); + memcpy(result.begin(), str.data(), str.size()); +} + +template +decltype(auto) CustomReadField(TypeList, Priority<1>, InvokeContext& invoke_context, Input&& input, + ReadDest&& read_dest) +{ + auto data = input.get(); + return read_dest.construct(fs::PathFromString({CharCast(data.begin()), data.size()})); +} + +template +void CustomBuildField(TypeList, Priority<1>, InvokeContext& invoke_context, Value&& str, Output&& output) +{ + auto result = output.init(str.size()); + // Copy SecureString into output. Caller needs to be responsible for calling + // memory_cleanse later on the output after it is sent. + memcpy(result.begin(), str.data(), str.size()); +} + +template +decltype(auto) CustomReadField(TypeList, Priority<1>, InvokeContext& invoke_context, Input&& input, + ReadDest&& read_dest) +{ + auto data = input.get(); + // Copy input into SecureString. Caller needs to be responsible for calling + // memory_cleanse on the input. + return read_dest.construct(CharCast(data.begin()), data.size()); +} + template void CustomBuildField(TypeList, Priority<1>, InvokeContext& invoke_context, Value&& value, Output&& output) { @@ -103,6 +248,199 @@ decltype(auto) CustomReadField(TypeList, Priority<1>, InvokeContext& i value.read(std::string_view{data.begin(), data.size()}); }); } + +template +void CustomBuildField(TypeList, Priority<1>, InvokeContext& invoke_context, + Value&& value, Output&& output) +{ + BuildField(TypeList(), invoke_context, output, std::string(value.what())); +} + +template +decltype(auto) CustomReadField(TypeList, Priority<1>, InvokeContext& invoke_context, + Input&& input, ReadDest&& read_dest) +{ + read_dest.construct(ReadField(TypeList(), invoke_context, input, mp::ReadDestTemp())); +} + +template +void CustomBuildField( + TypeList<>, Priority<1>, InvokeContext& invoke_context, Output&& output, + typename std::enable_if::value>::type* enable = nullptr) +{ + ipc::capnp::BuildGlobalArgs(invoke_context, output.init()); +} + +template +auto CustomPassField(TypeList<>, ServerContext& server_context, const Fn& fn, Args&&... args) -> + typename std::enable_if::value>::type +{ + ipc::capnp::ReadGlobalArgs(server_context, Accessor::get(server_context.call_context.getParams())); + return fn.invoke(server_context, std::forward(args)...); +} + +//! util::Result builder. +template +void CustomBuildField(TypeList>, Priority<1>, InvokeContext& invoke_context, Value&& value, + Output&& output) +{ + auto result = output.init(); + if (value) { + if constexpr (!std::is_same_v) { + using ValueAccessor = typename ProxyStruct::ValueAccessor; + BuildField(TypeList(), invoke_context, Make(result), *value); + } + } else { + BuildField(TypeList(), invoke_context, Make(result.initError()), + util::ErrorString(value)); + } +} + +//! util::Result reader. +template +decltype(auto) CustomReadField(TypeList>, Priority<1>, InvokeContext& invoke_context, + Input&& input, ReadDest&& read_dest) +{ + auto result = input.get(); + return read_dest.update([&](auto& value) { + if (result.hasError()) { + bilingual_str error; + ReadField(mp::TypeList(), invoke_context, mp::Make(result.getError()), + mp::ReadDestValue(error)); + value = util::Error{std::move(error)}; + } else { + if constexpr (!std::is_same_v) { + assert (result.hasValue()); + ReadField(mp::TypeList(), invoke_context, mp::Make(result.getValue()), + mp::ReadDestEmplace( + mp::TypeList(), [&](auto&&... args) -> auto& { + value = LocalType{std::forward(args)...}; + return *value; + })); + } + } + }); +} + +//! Forwarder to CustomBuildMessage. +template +void CustomBuildField(TypeList, Priority<2>, InvokeContext& invoke_context, Value&& value, Output&& output, + decltype(CustomBuildMessage(invoke_context, value, std::move(output.get())))* enable = nullptr) +{ + CustomBuildMessage(invoke_context, value, std::move(output.init())); +} + +//! Forwarder to CustomReadMessage. +template +decltype(auto) CustomReadField(TypeList, Priority<2>, InvokeContext& invoke_context, Reader&& reader, + ReadDest&& read_dest, + decltype(CustomReadMessage(invoke_context, reader.get(), + std::declval()))* enable = nullptr) +{ + return read_dest.update([&](auto& value) { CustomReadMessage(invoke_context, reader.get(), value); }); +} + +template +decltype(auto) MaybeGet(Message&& message, decltype(Accessor::get(message))* enable = nullptr) +{ + return Accessor::get(message); +} + +template +::capnp::Void MaybeGet(...) +{ + return {}; +} + +template +decltype(auto) MaybeInit(Message&& message, decltype(Accessor::get(message))* enable = nullptr) +{ + return Accessor::init(message); +} + +template +::capnp::Void MaybeInit(...) +{ + return {}; +} + +//! Forwarder to CustomPassMessage. +template +auto CustomPassField(TypeList, ServerContext& server_context, Fn&& fn, Args&&... args) + -> decltype(CustomPassMessage(server_context, MaybeGet(server_context.call_context.getParams()), + MaybeGet(server_context.call_context.getResults()), nullptr)) +{ + CustomPassMessage(server_context, MaybeGet(server_context.call_context.getParams()), + MaybeInit(server_context.call_context.getResults()), + [&](LocalTypes... param) { fn.invoke(server_context, std::forward(args)..., param...); }); +} + +//! Generic ::capnp::Data field builder for any class that a Span can be +//! constructed from, particularly BaseHash and base_blob classes and +//! subclasses. It's also used to serialize vector +//! GCSFilter::ElementSet set elements. +//! +//! There is currently no corresponding ::capnp::Data CustomReadField function +//! that works using Spans, because the bitcoin classes in the codebase like +//! BaseHash and blob_blob that can converted /to/ Span don't currently have +//! Span constructors that allow them to be constructed /from/ Span. For example +//! CustomReadField function could be written that would allow dropping +//! specialized CustomReadField functions for types like PKHash. +//! +//! For the LocalType = vector case, it's also not necessary to +//! have ::capnp::Data CustomReadField function corresponding to this +//! CustomBuildField function because ::capnp::Data inherits from +//! ::capnp::ArrayPtr, and libmultiprocess already provides a generic +//! CustomReadField function that can read from ::capnp::ArrayPtr into +//! std::vector. +template +void CustomBuildField( + TypeList, Priority<2>, InvokeContext& invoke_context, Value&& value, Output&& output, + typename std::enable_if_t::type>::type>::value>* enable_not_serializable = nullptr, + typename std::enable_if_t>* enable_output = + nullptr, + decltype(Span{value})* enable_value = nullptr) +{ + auto data = Span{value}; + auto result = output.init(data.size()); + memcpy(result.begin(), data.data(), data.size()); +} + +// libmultiprocess only provides read/build functions for std::set, not +// std::unordered_set, so copy and paste those functions here. +// TODO: Move these to libmultiprocess and dedup std::set, std::unordered_set, +// and std::vector implementations. +template +decltype(auto) CustomReadField(TypeList>, Priority<1>, + InvokeContext& invoke_context, Input&& input, ReadDest&& read_dest) +{ + return read_dest.update([&](auto& value) { + auto data = input.get(); + value.clear(); + for (auto item : data) { + ReadField(TypeList(), invoke_context, Make(item), + ReadDestEmplace( + TypeList(), [&](auto&&... args) -> auto& { + return *value.emplace(std::forward(args)...).first; + })); + } + }); +} + +template +void CustomBuildField(TypeList>, Priority<1>, InvokeContext& invoke_context, + Value&& value, Output&& output) +{ + auto list = output.init(value.size()); + size_t i = 0; + for (const auto& elem : value) { + BuildField(TypeList(), invoke_context, ListOutput(list, i), elem); + ++i; + } +} } // namespace mp #endif // BITCOIN_IPC_CAPNP_COMMON_TYPES_H diff --git a/src/ipc/capnp/common.capnp b/src/ipc/capnp/common.capnp new file mode 100644 index 0000000000000..8218ec5778918 --- /dev/null +++ b/src/ipc/capnp/common.capnp @@ -0,0 +1,49 @@ +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +@0xcd2c6232cb484a28; + +using Cxx = import "/capnp/c++.capnp"; +$Cxx.namespace("ipc::capnp::messages"); + +using Proxy = import "/mp/proxy.capnp"; +$Proxy.include("ipc/capnp/common.h"); +$Proxy.includeTypes("ipc/capnp/common-types.h"); + +struct Settings $Proxy.wrap("common::Settings") { + forcedSettings @0 :List(Pair(Text, Text)) $Proxy.name("forced_settings"); + commandLineOptions @1 :List(Pair(Text, List(Text))) $Proxy.name("command_line_options"); + rwSettings @2 :List(Pair(Text, Text)) $Proxy.name("rw_settings"); + roConfig @3 :List(Pair(Text, List(Pair(Text, List(Text))))) $Proxy.name("ro_config"); +} + +struct GlobalArgs $Proxy.count(0) { + settings @0 :Settings; + configPath @1 : Text; +} + +struct BilingualStr $Proxy.wrap("bilingual_str") { + original @0 :Text; + translated @1 :Text; +} + +struct Result(Value) { + value @0 :Value; + error @1: BilingualStr; +} + +# Wrapper for util::Result +struct ResultVoid(Value) { + error @0: BilingualStr; +} + +struct Pair(Key, Value) { + key @0 :Key; + value @1 :Value; +} + +struct PairUInt64(Key) { + key @0 :Key; + value @1 :UInt64; +} diff --git a/src/ipc/capnp/common.cpp b/src/ipc/capnp/common.cpp new file mode 100644 index 0000000000000..0c6a612425443 --- /dev/null +++ b/src/ipc/capnp/common.cpp @@ -0,0 +1,49 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace ipc { +namespace capnp { +void BuildGlobalArgs(mp::InvokeContext& invoke_context, messages::GlobalArgs::Builder&& builder) +{ + gArgs.LockSettings([&](const common::Settings& settings) { + mp::BuildField(mp::TypeList(), invoke_context, + mp::Make(builder.initSettings()), settings); + }); + builder.setConfigPath(fs::PathToString(gArgs.GetConfigFilePath())); +} + +void ReadGlobalArgs(mp::InvokeContext& invoke_context, const messages::GlobalArgs::Reader& reader) +{ + gArgs.LockSettings([&](common::Settings& settings) { + mp::ReadField(mp::TypeList(), invoke_context, mp::Make(reader.getSettings()), + mp::ReadDestValue(settings)); + }); + gArgs.SetConfigFilePath(fs::PathFromString(ipc::capnp::ToString(reader.getConfigPath()))); + SelectParams(gArgs.GetChainType()); +} +} // namespace capnp +} // namespace ipc diff --git a/src/ipc/capnp/common.h b/src/ipc/capnp/common.h new file mode 100644 index 0000000000000..6cdbc18ac434d --- /dev/null +++ b/src/ipc/capnp/common.h @@ -0,0 +1,29 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_IPC_CAPNP_COMMON_H +#define BITCOIN_IPC_CAPNP_COMMON_H + +#include +#include + +#include + +class RPCTimerInterface; + +namespace mp { +struct InvokeContext; +} // namespace mp + +namespace ipc { +namespace capnp { +//! GlobalArgs client-side argument handling. Builds message from ::gArgs variable. +void BuildGlobalArgs(mp::InvokeContext& invoke_context, messages::GlobalArgs::Builder&& builder); + +//! GlobalArgs server-side argument handling. Reads message into ::gArgs variable. +void ReadGlobalArgs(mp::InvokeContext& invoke_context, const messages::GlobalArgs::Reader& reader); +} // namespace capnp +} // namespace ipc + +#endif // BITCOIN_IPC_CAPNP_COMMON_H From b6516c454f70d8ef3091ee2ef0bdaede3b0f5893 Mon Sep 17 00:00:00 2001 From: Russell Yanofsky Date: Tue, 5 Dec 2017 15:57:12 -0500 Subject: [PATCH 12/21] Add capnp wrapper for Handler interface --- src/Makefile.am | 1 + src/ipc/capnp/handler.capnp | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 src/ipc/capnp/handler.capnp diff --git a/src/Makefile.am b/src/Makefile.am index cc429807e35d9..bcf54a1b7dc4a 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1080,6 +1080,7 @@ endif libbitcoin_ipc_mpgen_input = \ ipc/capnp/common.capnp \ ipc/capnp/echo.capnp \ + ipc/capnp/handler.capnp \ ipc/capnp/init.capnp EXTRA_DIST += $(libbitcoin_ipc_mpgen_input) %.capnp: diff --git a/src/ipc/capnp/handler.capnp b/src/ipc/capnp/handler.capnp new file mode 100644 index 0000000000000..3c1fadbac894e --- /dev/null +++ b/src/ipc/capnp/handler.capnp @@ -0,0 +1,16 @@ +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +@0xebd8f46e2f369076; + +using Cxx = import "/capnp/c++.capnp"; +$Cxx.namespace("ipc::capnp::messages"); + +using Proxy = import "/mp/proxy.capnp"; +$Proxy.include("interfaces/handler.h"); + +interface Handler $Proxy.wrap("interfaces::Handler") { + destroy @0 (context :Proxy.Context) -> (); + disconnect @1 (context :Proxy.Context) -> (); +} From 5cf30e5357fcc3d0ffb871b237e47b79fedd317a Mon Sep 17 00:00:00 2001 From: Russell Yanofsky Date: Tue, 5 Dec 2017 15:57:12 -0500 Subject: [PATCH 13/21] Add capnp wrapper for Chain interface --- src/Makefile.am | 4 + src/interfaces/chain.h | 3 +- src/ipc/capnp/chain-types.h | 80 +++++++++++++++ src/ipc/capnp/chain.capnp | 184 +++++++++++++++++++++++++++++++++ src/ipc/capnp/chain.cpp | 200 ++++++++++++++++++++++++++++++++++++ src/ipc/capnp/common.capnp | 5 + 6 files changed, 475 insertions(+), 1 deletion(-) create mode 100644 src/ipc/capnp/chain-types.h create mode 100644 src/ipc/capnp/chain.capnp create mode 100644 src/ipc/capnp/chain.cpp diff --git a/src/Makefile.am b/src/Makefile.am index bcf54a1b7dc4a..07e96479b85c8 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1078,6 +1078,7 @@ if HARDEN endif libbitcoin_ipc_mpgen_input = \ + ipc/capnp/chain.capnp \ ipc/capnp/common.capnp \ ipc/capnp/echo.capnp \ ipc/capnp/handler.capnp \ @@ -1087,12 +1088,15 @@ EXTRA_DIST += $(libbitcoin_ipc_mpgen_input) # Explicitly list dependencies on generated headers as described in # https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html#Recording-Dependencies-manually +ipc/capnp/libbitcoin_ipc_a-chain.$(OBJEXT): $(libbitcoin_ipc_mpgen_input:=.h) ipc/capnp/libbitcoin_ipc_a-common.$(OBJEXT): $(libbitcoin_ipc_mpgen_input:=.h) ipc/capnp/libbitcoin_ipc_a-protocol.$(OBJEXT): $(libbitcoin_ipc_mpgen_input:=.h) if BUILD_MULTIPROCESS LIBBITCOIN_IPC=libbitcoin_ipc.a libbitcoin_ipc_a_SOURCES = \ + ipc/capnp/chain.cpp \ + ipc/capnp/chain-types.h \ ipc/capnp/common.cpp \ ipc/capnp/common.h \ ipc/capnp/common-types.h \ diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index 9da5cb96373f3..10609af2601c3 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -382,7 +382,8 @@ class ChainClient //! Load saved state. virtual bool load() = 0; - //! Start client execution and provide a scheduler. + //! Start client execution and provide a scheduler. (Scheduler is + //! ignored if client is out-of-process). virtual void start(CScheduler& scheduler) = 0; //! Save state to disk. diff --git a/src/ipc/capnp/chain-types.h b/src/ipc/capnp/chain-types.h new file mode 100644 index 0000000000000..66f87e1f6927e --- /dev/null +++ b/src/ipc/capnp/chain-types.h @@ -0,0 +1,80 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_IPC_CAPNP_CHAIN_TYPES_H +#define BITCOIN_IPC_CAPNP_CHAIN_TYPES_H + +#include +#include +#include +#include + +#include + +//! Specialization of handleRpc needed because it takes a CRPCCommand& reference +//! argument, so a manual cleanup callback is needed to free the passed +//! CRPCCommand struct and proxy ActorCallback object. +template <> +struct mp::ProxyServerMethodTraits +{ + using Context = ServerContext; + static ::capnp::Void invoke(Context& context); +}; + +//! Specialization of start method needed to provide CScheduler& reference +//! argument. +template <> +struct mp::ProxyServerMethodTraits +{ + using ChainContext = ServerContext; + static void invoke(ChainContext& context); +}; + +namespace mp { +void CustomBuildMessage(InvokeContext& invoke_context, + const interfaces::FoundBlock& dest, + ipc::capnp::messages::FoundBlockParam::Builder&& builder); +void CustomPassMessage(InvokeContext& invoke_context, + const ipc::capnp::messages::FoundBlockParam::Reader& reader, + ipc::capnp::messages::FoundBlockResult::Builder&& builder, + std::function&& fn); +void CustomReadMessage(InvokeContext& invoke_context, + const ipc::capnp::messages::FoundBlockResult::Reader& reader, + const interfaces::FoundBlock& dest); + +void CustomBuildMessage(InvokeContext& invoke_context, + const interfaces::BlockInfo& block, + ipc::capnp::messages::BlockInfo::Builder&& builder); +void CustomPassMessage(InvokeContext& invoke_context, + const ipc::capnp::messages::BlockInfo::Reader& reader, + ::capnp::Void builder, + std::function&& fn); + +//! CScheduler& server-side argument handling. Skips argument so it can +//! be handled by ProxyServerCustom code. +template +void CustomPassField(TypeList, ServerContext& server_context, const Fn& fn, Args&&... args) +{ + fn.invoke(server_context, std::forward(args)...); +} + +//! CRPCCommand& server-side argument handling. Skips argument so it can +//! be handled by ProxyServerCustom code. +template +void CustomPassField(TypeList, ServerContext& server_context, const Fn& fn, Args&&... args) +{ + fn.invoke(server_context, std::forward(args)...); +} + +//! Override to avoid assert failures that would happen trying to serialize +//! spent coins. Probably it would be best for Coin serialization code not +//! to assert, but avoiding serialization in this case is harmless. +bool CustomHasValue(InvokeContext& invoke_context, const Coin& coin); +} // namespace mp + +#endif // BITCOIN_IPC_CAPNP_CHAIN_TYPES_H diff --git a/src/ipc/capnp/chain.capnp b/src/ipc/capnp/chain.capnp new file mode 100644 index 0000000000000..d557934f97fb2 --- /dev/null +++ b/src/ipc/capnp/chain.capnp @@ -0,0 +1,184 @@ +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +@0x94f21a4864bd2c65; + +using Cxx = import "/capnp/c++.capnp"; +$Cxx.namespace("ipc::capnp::messages"); + +using Proxy = import "/mp/proxy.capnp"; +$Proxy.include("interfaces/chain.h"); +$Proxy.include("rpc/server.h"); +$Proxy.includeTypes("ipc/capnp/chain-types.h"); + +using Common = import "common.capnp"; +using Handler = import "handler.capnp"; + +interface Chain $Proxy.wrap("interfaces::Chain") { + destroy @0 (context :Proxy.Context) -> (); + getHeight @1 (context :Proxy.Context) -> (result :Int32, hasResult :Bool); + getBlockHash @2 (context :Proxy.Context, height :Int32) -> (result :Data); + haveBlockOnDisk @3 (context :Proxy.Context, height :Int32) -> (result :Bool); + getTipLocator @4 (context :Proxy.Context) -> (result :Data); + getActiveChainLocator @5 (context :Proxy.Context, blockHash :Data) -> (result :Data); + findLocatorFork @6 (context :Proxy.Context, locator :Data) -> (result :Int32, hasResult :Bool); + hasBlockFilterIndex @7 (context :Proxy.Context, filterType :UInt8) -> (result :Bool); + blockFilterMatchesAny @8 (context :Proxy.Context, filterType :UInt8, blockHash :Data, filterSet :List(Data)) -> (result :Bool, hasResult :Bool); + findBlock @9 (context :Proxy.Context, hash :Data, block :FoundBlockParam) -> (block :FoundBlockResult, result :Bool); + findFirstBlockWithTimeAndHeight @10 (context :Proxy.Context, minTime :Int64, minHeight :Int32, block :FoundBlockParam) -> (block :FoundBlockResult, result :Bool); + findAncestorByHeight @11 (context :Proxy.Context, blockHash :Data, ancestorHeight :Int32, ancestor :FoundBlockParam) -> (ancestor :FoundBlockResult, result :Bool); + findAncestorByHash @12 (context :Proxy.Context, blockHash :Data, ancestorHash :Data, ancestor :FoundBlockParam) -> (ancestor :FoundBlockResult, result :Bool); + findCommonAncestor @13 (context :Proxy.Context, blockHash1 :Data, blockHash2 :Data, ancestor :FoundBlockParam, block1 :FoundBlockParam, block2 :FoundBlockParam) -> (ancestor :FoundBlockResult, block1 :FoundBlockResult, block2 :FoundBlockResult, result :Bool); + findCoins @14 (context :Proxy.Context, coins :List(Common.Pair(Data, Data))) -> (coins :List(Common.Pair(Data, Data))); + guessVerificationProgress @15 (context :Proxy.Context, blockHash :Data) -> (result :Float64); + hasBlocks @16 (context :Proxy.Context, blockHash :Data, minHeight :Int32, maxHeight: Int32, hasMaxHeight :Bool) -> (result :Bool); + isRBFOptIn @17 (context :Proxy.Context, tx :Data) -> (result :Int32); + isInMempool @18 (context :Proxy.Context, tx :Data) -> (result :Bool); + hasDescendantsInMempool @19 (context :Proxy.Context, txid :Data) -> (result :Bool); + broadcastTransaction @20 (context :Proxy.Context, tx: Data, maxTxFee :Int64, relay :Bool) -> (error: Text, result :Bool); + getTransactionAncestry @21 (context :Proxy.Context, txid :Data) -> (ancestors :UInt64, descendants :UInt64, ancestorsize :UInt64, ancestorfees :Int64); + calculateIndividualBumpFees @22 (context :Proxy.Context, outpoints :List(Data), targetFeerate :Data) -> (result: List(Common.PairInt64(Data))); + calculateCombinedBumpFee @23 (context :Proxy.Context, outpoints :List(Data), targetFeerate :Data) -> (result :Int64, hasResult :Bool); + getPackageLimits @24 (context :Proxy.Context) -> (ancestors :UInt64, descendants :UInt64); + checkChainLimits @25 (context :Proxy.Context, tx :Data) -> (result :Common.ResultVoid); + estimateSmartFee @26 (context :Proxy.Context, numBlocks :Int32, conservative :Bool, wantCalc :Bool) -> (calc :FeeCalculation, result :Data); + estimateMaxBlocks @27 (context :Proxy.Context) -> (result :UInt32); + mempoolMinFee @28 (context :Proxy.Context) -> (result :Data); + relayMinFee @29 (context :Proxy.Context) -> (result :Data); + relayIncrementalFee @30 (context :Proxy.Context) -> (result :Data); + relayDustFee @31 (context :Proxy.Context) -> (result :Data); + havePruned @32 (context :Proxy.Context) -> (result :Bool); + isReadyToBroadcast @33 (context :Proxy.Context) -> (result :Bool); + isInitialBlockDownload @34 (context :Proxy.Context) -> (result :Bool); + shutdownRequested @35 (context :Proxy.Context) -> (result :Bool); + initMessage @36 (context :Proxy.Context, message :Text) -> (); + initWarning @37 (context :Proxy.Context, message :Common.BilingualStr) -> (); + initError @38 (context :Proxy.Context, message :Common.BilingualStr) -> (); + showProgress @39 (context :Proxy.Context, title :Text, progress :Int32, resumePossible :Bool) -> (); + handleNotifications @40 (context :Proxy.Context, notifications :ChainNotifications) -> (result :Handler.Handler); + waitForNotificationsIfTipChanged @41 (context :Proxy.Context, oldTip :Data) -> (); + handleRpc @42 (context :Proxy.Context, command :RPCCommand) -> (result :Handler.Handler); + rpcEnableDeprecated @43 (context :Proxy.Context, method :Text) -> (result :Bool); + rpcRunLater @44 (context :Proxy.Context, name :Text, fn: RunLaterCallback, seconds: Int64) -> (); + getSetting @45 (context :Proxy.Context, name :Text) -> (result :Text); + getSettingsList @46 (context :Proxy.Context, name :Text) -> (result :List(Text)); + getRwSetting @47 (context :Proxy.Context, name :Text) -> (result :Text); + updateRwSetting @48 (context :Proxy.Context, name :Text, value :Text, write :Bool) -> (result :Bool); + requestMempoolTransactions @49 (context :Proxy.Context, notifications :ChainNotifications) -> (); + hasAssumedValidChain @50 (context :Proxy.Context) -> (result :Bool); +} + +interface ChainNotifications $Proxy.wrap("interfaces::Chain::Notifications") { + destroy @0 (context :Proxy.Context) -> (); + transactionAddedToMempool @1 (context :Proxy.Context, tx :Data) -> (); + transactionRemovedFromMempool @2 (context :Proxy.Context, tx :Data, reason :Int32) -> (); + blockConnected @3 (context :Proxy.Context, role: UInt32, block :BlockInfo) -> (); + blockDisconnected @4 (context :Proxy.Context, block :BlockInfo) -> (); + updatedBlockTip @5 (context :Proxy.Context) -> (); + chainStateFlushed @6 (context :Proxy.Context, role: UInt32, locator :Data) -> (); +} + +interface ChainClient $Proxy.wrap("interfaces::ChainClient") { + destroy @0 (context :Proxy.Context) -> (); + registerRpcs @1 (context :Proxy.Context) -> (); + verify @2 (context :Proxy.Context) -> (result :Bool); + load @3 (context :Proxy.Context) -> (result :Bool); + start @4 (context :Proxy.Context, scheduler :Void) -> (); + flush @5 (context :Proxy.Context) -> (); + stop @6 (context :Proxy.Context) -> (); + setMockTime @7 (context :Proxy.Context, time :Int64) -> (); + schedulerMockForward @8 (context :Proxy.Context, time :Int64) -> (); +} + +struct FeeCalculation $Proxy.wrap("FeeCalculation") { + est @0 :EstimationResult; + reason @1 :Int32; + desiredTarget @2 :Int32; + returnedTarget @3 :Int32; +} + +struct EstimationResult $Proxy.wrap("EstimationResult") +{ + pass @0 :EstimatorBucket; + fail @1 :EstimatorBucket; + decay @2 :Float64; + scale @3 :UInt32; +} + +struct EstimatorBucket $Proxy.wrap("EstimatorBucket") +{ + start @0 :Float64; + end @1 :Float64; + withinTarget @2 :Float64; + totalConfirmed @3 :Float64; + inMempool @4 :Float64; + leftMempool @5 :Float64; +} + +struct RPCCommand $Proxy.wrap("CRPCCommand") { + category @0 :Text; + name @1 :Text; + actor @2 :ActorCallback; + argNames @3 :List(RPCArg); + uniqueId @4 :Int64 $Proxy.name("unique_id"); +} + +struct RPCArg { + name @0 :Text; + namedOnly @1: Bool; +} + +interface ActorCallback $Proxy.wrap("ProxyCallback") { + call @0 (context :Proxy.Context, request :JSONRPCRequest, response :Text, lastCallback :Bool) -> (error :Text $Proxy.exception("std::exception"), rpcError :Text $Proxy.exception("UniValue"), typeError :Text $Proxy.exception("UniValue::type_error"), response :Text, result: Bool); +} + +struct JSONRPCRequest $Proxy.wrap("JSONRPCRequest") { + id @0 :Text; + method @1 :Text $Proxy.name("strMethod"); + params @2 :Text; + mode @3 :Int32; + uri @4 :Text $Proxy.name("URI"); + authUser @5 :Text; +} + +interface RunLaterCallback $Proxy.wrap("ProxyCallback>") { + destroy @0 (context :Proxy.Context) -> (); + call @1 (context :Proxy.Context) -> (); +} + +struct FoundBlockParam { + wantHash @0 :Bool; + wantHeight @1 :Bool; + wantTime @2 :Bool; + wantMaxTime @3 :Bool; + wantMtpTime @4 :Bool; + wantInActiveChain @5 :Bool; + nextBlock @6: FoundBlockParam; + wantData @7 :Bool; +} + +struct FoundBlockResult { + hash @0 :Data; + height @1 :Int32; + time @2 :Int64; + maxTime @3 :Int64; + mtpTime @4 :Int64; + inActiveChain @5 :Int64; + nextBlock @6: FoundBlockResult; + data @7 :Data; + found @8 :Bool; +} + +struct BlockInfo $Proxy.wrap("interfaces::BlockInfo") { + # Fields skipped below with Proxy.skip are pointer fields manually handled + # by CustomBuildMessage / CustomPassMessage overloads. + hash @0 :Data $Proxy.skip; + prevHash @1 :Data $Proxy.skip; + height @2 :Int32 = -1; + fileNumber @3 :Int32 = -1 $Proxy.name("file_number"); + dataPos @4 :UInt32 = 0 $Proxy.name("data_pos"); + data @5 :Data $Proxy.skip; + undoData @6 :Data $Proxy.skip; + chainTimeMax @7 :UInt32 = 0 $Proxy.name("chain_time_max"); +} diff --git a/src/ipc/capnp/chain.cpp b/src/ipc/capnp/chain.cpp new file mode 100644 index 0000000000000..fb1ba5a0720d0 --- /dev/null +++ b/src/ipc/capnp/chain.cpp @@ -0,0 +1,200 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace mp { +void CustomBuildMessage(InvokeContext& invoke_context, + const interfaces::FoundBlock& dest, + ipc::capnp::messages::FoundBlockParam::Builder&& builder) +{ + if (dest.m_hash) builder.setWantHash(true); + if (dest.m_height) builder.setWantHeight(true); + if (dest.m_time) builder.setWantTime(true); + if (dest.m_max_time) builder.setWantMaxTime(true); + if (dest.m_mtp_time) builder.setWantMtpTime(true); + if (dest.m_in_active_chain) builder.setWantInActiveChain(true); + if (dest.m_next_block) CustomBuildMessage(invoke_context, *dest.m_next_block, builder.initNextBlock()); + if (dest.m_data) builder.setWantData(true); +} + +void FindBlock(const std::function& find, + const ipc::capnp::messages::FoundBlockParam::Reader& reader, + ipc::capnp::messages::FoundBlockResult::Builder&& builder, + interfaces::FoundBlock& found_block) +{ + uint256 hash; + int height = -1; + int64_t time = -1; + int64_t max_time = -1; + int64_t mtp_time = -1; + bool in_active_chain = -1; + CBlock data; + if (reader.getWantHash()) found_block.hash(hash); + if (reader.getWantHeight()) found_block.height(height); + if (reader.getWantTime()) found_block.time(time); + if (reader.getWantMaxTime()) found_block.maxTime(max_time); + if (reader.getWantMtpTime()) found_block.mtpTime(mtp_time); + if (reader.getWantInActiveChain()) found_block.inActiveChain(in_active_chain); + if (reader.getWantData()) found_block.data(data); + if (reader.hasNextBlock()) { + interfaces::FoundBlock next_block; + found_block.nextBlock(next_block); + FindBlock(find, reader.getNextBlock(), builder.initNextBlock(), next_block); + } else { + find(); + } + if (!found_block.found) return; + if (reader.getWantHash()) builder.setHash(ipc::capnp::ToArray(ipc::capnp::Serialize(hash))); + if (reader.getWantHeight()) builder.setHeight(height); + if (reader.getWantTime()) builder.setTime(time); + if (reader.getWantMaxTime()) builder.setMaxTime(max_time); + if (reader.getWantMtpTime()) builder.setMtpTime(mtp_time); + if (reader.getWantInActiveChain()) builder.setInActiveChain(in_active_chain); + if (reader.getWantData()) builder.setData(ipc::capnp::ToArray(ipc::capnp::Serialize(data))); + builder.setFound(true); +} + +void CustomPassMessage(InvokeContext& invoke_context, + const ipc::capnp::messages::FoundBlockParam::Reader& reader, + ipc::capnp::messages::FoundBlockResult::Builder&& builder, + std::function&& fn) +{ + interfaces::FoundBlock found_block; + FindBlock([&] { fn(found_block); }, reader, std::move(builder), found_block); +} + +void CustomReadMessage(InvokeContext& invoke_context, + const ipc::capnp::messages::FoundBlockResult::Reader& reader, + const interfaces::FoundBlock& dest) +{ + if (!reader.getFound()) return; + if (dest.m_hash) *dest.m_hash = ipc::capnp::Unserialize(reader.getHash()); + if (dest.m_height) *dest.m_height = reader.getHeight(); + if (dest.m_time) *dest.m_time = reader.getTime(); + if (dest.m_max_time) *dest.m_max_time = reader.getMaxTime(); + if (dest.m_mtp_time) *dest.m_mtp_time = reader.getMtpTime(); + if (dest.m_in_active_chain) *dest.m_in_active_chain = reader.getInActiveChain(); + if (dest.m_next_block) CustomReadMessage(invoke_context, reader.getNextBlock(), *dest.m_next_block); + if (dest.m_data) *dest.m_data = ipc::capnp::Unserialize(reader.getData()); + dest.found = true; +} + +void CustomBuildMessage(InvokeContext& invoke_context, + const interfaces::BlockInfo& block, + ipc::capnp::messages::BlockInfo::Builder&& builder) +{ + // Pointer fields are annotated with Proxy.skip so need to be filled + // manually. Generated code would actually work correctly, though, so this + // could be dropped if there were a Proxy.skip(read_only=True) half-skip + // option. + builder.setHash(ipc::capnp::ToArray(block.hash)); + if (block.prev_hash) builder.setPrevHash(ipc::capnp::ToArray(*block.prev_hash)); + if (block.data) builder.setData(ipc::capnp::ToArray(ipc::capnp::Serialize(*block.data))); + if (block.undo_data) builder.setUndoData(ipc::capnp::ToArray(ipc::capnp::Serialize(*block.undo_data))); + // Copy the remaining fields using the code generated by Proxy.wrap. + mp::BuildOne<0>(mp::TypeList(), invoke_context, builder, block); +} + +void CustomPassMessage(InvokeContext& invoke_context, + const ipc::capnp::messages::BlockInfo::Reader& reader, + ::capnp::Void builder, + std::function&& fn) +{ + // Pointer fields are annotated with Proxy.skip because code generator can't + // figure out pointer lifetimes to be able to implementat a ReadField + // implementation for the BlockInfo struct, and the default PassField + // function calls readfield. In theory though, code generator could create a + // PassField specialization for the struct which could allocate pointers for + // the lifetime of the call, like the code below is doing manually. This + // would take care of all pointer fields, though the BlockInfo.hash + // reference field would still need to be handled specially. Or even + // BlockInfo.hash field could be handled automatically if the generated code + // used C++20 designated member initialization. + const uint256 hash = ipc::capnp::ToBlob(reader.getHash()); + std::optional prev_hash; + std::optional data; + std::optional undo_data; + interfaces::BlockInfo block{hash}; + if (reader.hasPrevHash()) { + prev_hash.emplace(ipc::capnp::ToBlob(reader.getPrevHash())); + block.prev_hash = &*prev_hash; + } + if (reader.hasData()) { + data.emplace(ipc::capnp::Unserialize(reader.getData())); + block.data = &*data; + } + if (reader.hasUndoData()) { + undo_data.emplace(ipc::capnp::Unserialize(reader.getUndoData())); + block.undo_data = &*undo_data; + } + mp::ReadField(mp::TypeList(), invoke_context, + mp::Make(reader), mp::ReadDestValue(block)); + fn(block); +} + +::capnp::Void ProxyServerMethodTraits::invoke( + Context& context) +{ + auto params = context.call_context.getParams(); + auto command = params.getCommand(); + + CRPCCommand::Actor actor; + ReadField(TypeList(), context, Make(command.getActor()), ReadDestValue(actor)); + std::vector> args; + ReadField(TypeList(), context, Make(command.getArgNames()), ReadDestValue(args)); + + auto rpc_command = std::make_unique(command.getCategory(), command.getName(), std::move(actor), + std::move(args), command.getUniqueId()); + auto handler = context.proxy_server.m_impl->handleRpc(*rpc_command); + auto results = context.call_context.getResults(); + auto result = kj::heap>(std::shared_ptr(handler.release()), *context.proxy_server.m_context.connection); + result->m_context.cleanup.emplace_back([rpc_command = rpc_command.release()] { delete rpc_command; }); + results.setResult(kj::mv(result)); + return {}; +} + +void ProxyServerMethodTraits::invoke(ChainContext& context) +{ + // This method is never called because ChainClient::Start is overridden by + // WalletLoader::Start. The custom implementation is needed just because + // the CScheduler& argument this is supposed to pass is not serializable. + assert(0); +} + +bool CustomHasValue(InvokeContext& invoke_context, const Coin& coin) +{ + // Spent coins cannot be serialized due to an assert in Coin::Serialize. + return !coin.IsSpent(); +} +} // namespace mp diff --git a/src/ipc/capnp/common.capnp b/src/ipc/capnp/common.capnp index 8218ec5778918..22934c8e3c6d0 100644 --- a/src/ipc/capnp/common.capnp +++ b/src/ipc/capnp/common.capnp @@ -43,6 +43,11 @@ struct Pair(Key, Value) { value @1 :Value; } +struct PairInt64(Key) { + key @0 :Key; + value @1 :Int64; +} + struct PairUInt64(Key) { key @0 :Key; value @1 :UInt64; From b5fd425e075a3b21bc5e5b4f25209d8abacbeb58 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Tue, 28 Nov 2023 13:52:32 -0500 Subject: [PATCH 14/21] Add capnp wrapper for Wallet interface --- src/Makefile.am | 6 +- src/interfaces/wallet.h | 2 + src/ipc/capnp/chain-types.h | 5 + src/ipc/capnp/wallet-types.h | 55 ++++++++ src/ipc/capnp/wallet.capnp | 257 +++++++++++++++++++++++++++++++++++ src/ipc/capnp/wallet.cpp | 222 ++++++++++++++++++++++++++++++ src/ipc/capnp/wallet.h | 41 ++++++ src/serialize.h | 1 + src/wallet/coincontrol.h | 3 + src/wallet/wallet.h | 2 + 10 files changed, 593 insertions(+), 1 deletion(-) create mode 100644 src/ipc/capnp/wallet-types.h create mode 100644 src/ipc/capnp/wallet.capnp create mode 100644 src/ipc/capnp/wallet.cpp create mode 100644 src/ipc/capnp/wallet.h diff --git a/src/Makefile.am b/src/Makefile.am index 07e96479b85c8..128f6bb8e27d3 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1082,7 +1082,8 @@ libbitcoin_ipc_mpgen_input = \ ipc/capnp/common.capnp \ ipc/capnp/echo.capnp \ ipc/capnp/handler.capnp \ - ipc/capnp/init.capnp + ipc/capnp/init.capnp \ + ipc/capnp/wallet.capnp EXTRA_DIST += $(libbitcoin_ipc_mpgen_input) %.capnp: @@ -1104,6 +1105,9 @@ libbitcoin_ipc_a_SOURCES = \ ipc/capnp/init-types.h \ ipc/capnp/protocol.cpp \ ipc/capnp/protocol.h \ + ipc/capnp/wallet-types.h \ + ipc/capnp/wallet.cpp \ + ipc/capnp/wallet.h \ ipc/context.h \ ipc/exception.h \ ipc/interfaces.cpp \ diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index 61142366234cc..15ae587b87a3f 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -361,6 +361,8 @@ struct WalletAddress wallet::AddressPurpose purpose; std::string name; + WalletAddress() = default; + WalletAddress(CTxDestination dest, wallet::isminetype is_mine, wallet::AddressPurpose purpose, std::string name) : dest(std::move(dest)), is_mine(is_mine), purpose(std::move(purpose)), name(std::move(name)) { diff --git a/src/ipc/capnp/chain-types.h b/src/ipc/capnp/chain-types.h index 66f87e1f6927e..dff52d1e3dea4 100644 --- a/src/ipc/capnp/chain-types.h +++ b/src/ipc/capnp/chain-types.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -33,6 +34,10 @@ struct mp::ProxyServerMethodTraits; static void invoke(ChainContext& context); + using WalletContext = ServerContext; + static void invoke(WalletContext& context); }; namespace mp { diff --git a/src/ipc/capnp/wallet-types.h b/src/ipc/capnp/wallet-types.h new file mode 100644 index 0000000000000..bbda504eb85dd --- /dev/null +++ b/src/ipc/capnp/wallet-types.h @@ -0,0 +1,55 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_IPC_CAPNP_WALLET_TYPES_H +#define BITCOIN_IPC_CAPNP_WALLET_TYPES_H + +#include +#include +#include +#include +#include + +class CKey; +namespace wallet { +class CCoinControl; +} // namespace wallet + +namespace mp { +void CustomBuildMessage(InvokeContext& invoke_context, + const CTxDestination& dest, + ipc::capnp::messages::TxDestination::Builder&& builder); +void CustomReadMessage(InvokeContext& invoke_context, + const ipc::capnp::messages::TxDestination::Reader& reader, + CTxDestination& dest); +void CustomBuildMessage(InvokeContext& invoke_context, + const WitnessUnknown& dest, + ipc::capnp::messages::WitnessUnknown::Builder&& builder); +void CustomReadMessage(InvokeContext& invoke_context, + const ipc::capnp::messages::WitnessUnknown::Reader& reader, + WitnessUnknown& dest); +void CustomBuildMessage(InvokeContext& invoke_context, const CKey& key, ipc::capnp::messages::Key::Builder&& builder); +void CustomReadMessage(InvokeContext& invoke_context, const ipc::capnp::messages::Key::Reader& reader, CKey& key); +void CustomBuildMessage(InvokeContext& invoke_context, + const wallet::CCoinControl& coin_control, + ipc::capnp::messages::CoinControl::Builder&& builder); +void CustomReadMessage(InvokeContext& invoke_context, + const ipc::capnp::messages::CoinControl::Reader& reader, + wallet::CCoinControl& coin_control); + +//! CustomReadField implementation needed for converting ::capnp::Data messages +//! to PKHash objects because PKHash doesn't currently have a Span constructor. +//! This could be dropped and replaced with a more generic ::capnp::Data +//! CustomReadField function as described in common-types.h near the generic +//! CustomBuildField function which is already used to build ::capnp::Data +//! messages from PKHash objects. +template +decltype(auto) CustomReadField( + TypeList, Priority<1>, InvokeContext& invoke_context, Reader&& reader, ReadDest&& read_dest) +{ + return read_dest.construct(ipc::capnp::ToBlob(reader.get())); +} +} // namespace mp + +#endif // BITCOIN_IPC_CAPNP_WALLET_TYPES_H diff --git a/src/ipc/capnp/wallet.capnp b/src/ipc/capnp/wallet.capnp new file mode 100644 index 0000000000000..55b50353ef6e4 --- /dev/null +++ b/src/ipc/capnp/wallet.capnp @@ -0,0 +1,257 @@ +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +@0xe234cce74feea00c; + +using Cxx = import "/capnp/c++.capnp"; +$Cxx.namespace("ipc::capnp::messages"); + +using Proxy = import "/mp/proxy.capnp"; +$Proxy.include("ipc/capnp/wallet.h"); +$Proxy.includeTypes("ipc/capnp/wallet-types.h"); + +using Chain = import "chain.capnp"; +using Common = import "common.capnp"; +using Handler = import "handler.capnp"; + +interface Wallet $Proxy.wrap("interfaces::Wallet") { + destroy @0 (context :Proxy.Context) -> (); + encryptWallet @1 (context :Proxy.Context, walletPassphrase :Data) -> (result :Bool); + isCrypted @2 (context :Proxy.Context) -> (result :Bool); + lock @3 (context :Proxy.Context) -> (result :Bool); + unlock @4 (context :Proxy.Context, walletPassphrase :Data) -> (result :Bool); + isLocked @5 (context :Proxy.Context) -> (result :Bool); + changeWalletPassphrase @6 (context :Proxy.Context, oldWalletPassphrase :Data, newWalletPassphrase :Data) -> (result :Bool); + abortRescan @7 (context :Proxy.Context) -> (); + backupWallet @8 (context :Proxy.Context, filename :Text) -> (result :Bool); + getWalletName @9 (context :Proxy.Context) -> (result :Text); + getNewDestination @10 (context :Proxy.Context, outputType :Int32, label :Text) -> (result :Common.Result(TxDestination)); + getPubKey @11 (context :Proxy.Context, script :Data, address :Data) -> (pubKey :Data, result :Bool); + signMessage @12 (context :Proxy.Context, message :Text, pkhash :Data) -> (signature :Text, result :Int32); + isSpendable @13 (context :Proxy.Context, dest :TxDestination) -> (result :Bool); + haveWatchOnly @14 (context :Proxy.Context) -> (result :Bool); + setAddressBook @15 (context :Proxy.Context, dest :TxDestination, name :Text, purpose :Int32) -> (result :Bool); + delAddressBook @16 (context :Proxy.Context, dest :TxDestination) -> (result :Bool); + getAddress @17 (context :Proxy.Context, dest :TxDestination, wantName :Bool, wantIsMine :Bool, wantPurpose :Bool) -> (name :Text, isMine :Int32, purpose :Int32, result :Bool); + getAddresses @18 (context :Proxy.Context) -> (result :List(WalletAddress)); + getAddressReceiveRequests @19 (context :Proxy.Context) -> (result :List(Data)); + setAddressReceiveRequest @20 (context :Proxy.Context, dest :TxDestination, id :Data, value :Data) -> (result :Bool); + displayAddress @21 (context :Proxy.Context, dest :TxDestination) -> (result :Bool); + lockCoin @22 (context :Proxy.Context, output :Data, writeToDb :Bool) -> (result :Bool); + unlockCoin @23 (context :Proxy.Context, output :Data) -> (result :Bool); + isLockedCoin @24 (context :Proxy.Context, output :Data) -> (result :Bool); + listLockedCoins @25 (context :Proxy.Context) -> (outputs :List(Data)); + createTransaction @26 (context :Proxy.Context, recipients :List(Recipient), coinControl :CoinControl, sign :Bool, changePos :Int32) -> (changePos :Int32, fee :Int64, result :Common.Result(Data)); + commitTransaction @27 (context :Proxy.Context, tx :Data, valueMap :List(Common.Pair(Text, Text)), orderForm :List(Common.Pair(Text, Text))) -> (); + transactionCanBeAbandoned @28 (context :Proxy.Context, txid :Data) -> (result :Bool); + abandonTransaction @29 (context :Proxy.Context, txid :Data) -> (result :Bool); + transactionCanBeBumped @30 (context :Proxy.Context, txid :Data) -> (result :Bool); + createBumpTransaction @31 (context :Proxy.Context, txid :Data, coinControl :CoinControl) -> (errors :List(Common.BilingualStr), oldFee :Int64, newFee :Int64, mtx :Data, result :Bool); + signBumpTransaction @32 (context :Proxy.Context, mtx :Data) -> (mtx :Data, result :Bool); + commitBumpTransaction @33 (context :Proxy.Context, txid :Data, mtx :Data) -> (errors :List(Common.BilingualStr), bumpedTxid :Data, result :Bool); + getTx @34 (context :Proxy.Context, txid :Data) -> (result :Data); + getWalletTx @35 (context :Proxy.Context, txid :Data) -> (result :WalletTx); + getWalletTxs @36 (context :Proxy.Context) -> (result :List(WalletTx)); + tryGetTxStatus @37 (context :Proxy.Context, txid :Data) -> (txStatus :WalletTxStatus, numBlocks :Int32, blockTime :Int64, result :Bool); + getWalletTxDetails @38 (context :Proxy.Context, txid :Data) -> (txStatus :WalletTxStatus, orderForm :List(Common.Pair(Text, Text)), inMempool :Bool, numBlocks :Int32, result :WalletTx); + getBalances @39 (context :Proxy.Context) -> (result :WalletBalances); + fillPSBT @40 (context :Proxy.Context, sighashType :Int32, sign :Bool, bip32derivs :Bool, wantNSigned :Bool) -> (nSigned: UInt64, psbt :Data, complete :Bool, result :Int32); + tryGetBalances @41 (context :Proxy.Context) -> (balances :WalletBalances, blockHash :Data, result :Bool); + getBalance @42 (context :Proxy.Context) -> (result :Int64); + getAvailableBalance @43 (context :Proxy.Context, coinControl :CoinControl) -> (result :Int64); + txinIsMine @44 (context :Proxy.Context, txin :Data) -> (result :Int32); + txoutIsMine @45 (context :Proxy.Context, txout :Data) -> (result :Int32); + getDebit @46 (context :Proxy.Context, txin :Data, filter :UInt32) -> (result :Int64); + getCredit @47 (context :Proxy.Context, txout :Data, filter :UInt32) -> (result :Int64); + listCoins @48 (context :Proxy.Context) -> (result :List(Common.Pair(TxDestination, List(Common.Pair(Data, WalletTxOut))))); + getCoins @49 (context :Proxy.Context, outputs :List(Data)) -> (result :List(WalletTxOut)); + getRequiredFee @50 (context :Proxy.Context, txBytes :UInt32) -> (result :Int64); + getMinimumFee @51 (context :Proxy.Context, txBytes :UInt32, coinControl :CoinControl, wantReturnedTarget :Bool, wantReason :Bool) -> (returnedTarget :Int32, reason :Int32, result :Int64); + getConfirmTarget @52 (context :Proxy.Context) -> (result :UInt32); + hdEnabled @53 (context :Proxy.Context) -> (result :Bool); + canGetAddresses @54 (context :Proxy.Context) -> (result :Bool); + privateKeysDisabled @55 (context :Proxy.Context) -> (result :Bool); + taprootEnabled @56 (context :Proxy.Context) -> (result :Bool); + hasExternalSigner @57 (context :Proxy.Context) -> (result :Bool); + getDefaultAddressType @58 (context :Proxy.Context) -> (result :Int32); + getDefaultMaxTxFee @59 (context :Proxy.Context) -> (result :Int64); + remove @60 (context :Proxy.Context) -> (); + isLegacy @61 (context :Proxy.Context) -> (result :Bool); + handleUnload @62 (context :Proxy.Context, callback :UnloadWalletCallback) -> (result :Handler.Handler); + handleShowProgress @63 (context :Proxy.Context, callback :ShowWalletProgressCallback) -> (result :Handler.Handler); + handleStatusChanged @64 (context :Proxy.Context, callback :StatusChangedCallback) -> (result :Handler.Handler); + handleAddressBookChanged @65 (context :Proxy.Context, callback :AddressBookChangedCallback) -> (result :Handler.Handler); + handleTransactionChanged @66 (context :Proxy.Context, callback :TransactionChangedCallback) -> (result :Handler.Handler); + handleWatchOnlyChanged @67 (context :Proxy.Context, callback :WatchOnlyChangedCallback) -> (result :Handler.Handler); + handleCanGetAddressesChanged @68 (context :Proxy.Context, callback :CanGetAddressesChangedCallback) -> (result :Handler.Handler); +} + +interface UnloadWalletCallback $Proxy.wrap("ProxyCallback") { + destroy @0 (context :Proxy.Context) -> (); + call @1 (context :Proxy.Context) -> (); +} + +interface ShowWalletProgressCallback $Proxy.wrap("ProxyCallback") { + destroy @0 (context :Proxy.Context) -> (); + call @1 (context :Proxy.Context, title :Text, progress :Int32) -> (); +} + +interface StatusChangedCallback $Proxy.wrap("ProxyCallback") { + destroy @0 (context :Proxy.Context) -> (); + call @1 (context :Proxy.Context) -> (); +} + +interface AddressBookChangedCallback $Proxy.wrap("ProxyCallback") { + destroy @0 (context :Proxy.Context) -> (); + call @1 (context :Proxy.Context, address :TxDestination, label :Text, isMine :Bool, purpose :Int32, status :Int32) -> (); +} + +interface TransactionChangedCallback $Proxy.wrap("ProxyCallback") { + destroy @0 (context :Proxy.Context) -> (); + call @1 (context :Proxy.Context, txid :Data, status :Int32) -> (); +} + +interface WatchOnlyChangedCallback $Proxy.wrap("ProxyCallback") { + destroy @0 (context :Proxy.Context) -> (); + call @1 (context :Proxy.Context, haveWatchOnly :Bool) -> (); +} + +interface CanGetAddressesChangedCallback $Proxy.wrap("ProxyCallback") { + destroy @0 (context :Proxy.Context) -> (); + call @1 (context :Proxy.Context) -> (); +} + +interface WalletLoader extends(Chain.ChainClient) $Proxy.wrap("interfaces::WalletLoader") { + createWallet @0 (context :Proxy.Context, name :Text, passphrase :Text, flags :UInt64) -> (warning :List(Common.BilingualStr), result :Common.Result(Wallet)); + loadWallet @1 (context :Proxy.Context, name :Text) -> (warning :List(Common.BilingualStr), result :Common.Result(Wallet)); + getWalletDir @2 (context :Proxy.Context) -> (result :Text); + restoreWallet @3 (context :Proxy.Context, backupFile :Data, name :Text) -> (warning :List(Common.BilingualStr), result :Common.Result(Wallet)); + migrateWallet @4 (context :Proxy.Context, name :Text, passphrase :Text) -> (result :Common.Result(WalletMigrationResult)); + listWalletDir @5 (context :Proxy.Context) -> (result :List(Text)); + getWallets @6 (context :Proxy.Context) -> (result :List(Wallet)); + handleLoadWallet @7 (context :Proxy.Context, callback :LoadWalletCallback) -> (result :Handler.Handler); +} + +interface LoadWalletCallback $Proxy.wrap("ProxyCallback") { + destroy @0 (context :Proxy.Context) -> (); + call @1 (context :Proxy.Context, wallet :Wallet) -> (); +} + +struct Key { + secret @0 :Data; + isCompressed @1 :Bool; +} + +struct TxDestination { + pubKey @0 :Data; + pkHash @1 :Data; + scriptHash @2 :Data; + witnessV0ScriptHash @3 :Data; + witnessV0KeyHash @4 :Data; + witnessV1Taproot @5 :Data; + witnessUnknown @6 :WitnessUnknown; +} + +struct WitnessUnknown +{ + version @0 :UInt32; + program @1 :Data; +} + +struct WalletAddress $Proxy.wrap("interfaces::WalletAddress") { + dest @0 :TxDestination; + isMine @1 :Int32 $Proxy.name("is_mine"); + name @2 :Text; + purpose @3 :Int32; +} + +struct Recipient $Proxy.wrap("wallet::CRecipient") { + dest @0 :TxDestination; + amount @1 :Int64 $Proxy.name("nAmount"); + subtractFeeFromAmount @2 :Bool $Proxy.name("fSubtractFeeFromAmount"); +} + +struct CoinControl { + destChange @0 :TxDestination; + hasChangeType @1 :Bool; + changeType @2 :Int32; + includeUnsafeInputs @3 :Bool; + allowOtherInputs @4 :Bool; + allowWatchOnly @5 :Bool; + overrideFeeRate @6 :Bool; + hasFeeRate @7 :Bool; + feeRate @8 :Data; + hasConfirmTarget @9 :Bool; + confirmTarget @10 :Int32; + hasSignalRbf @11 :Bool; + signalRbf @12 :Bool; + avoidPartialSpends @13 :Bool; + avoidAddressReuse @14 :Bool; + feeMode @15 :Int32; + minDepth @16 :Int32; + maxDepth @17 :Int32; + # Note: The corresponding CCoinControl class has a FlatSigningProvider + # m_external_provider member here, but it is intentionally not included in + # this capnp struct because it is not needed by the GUI. Probably the + # CCoinControl class should be split up to separate options which are + # intended to be set by GUI and RPC interfaces from members containing + # internal wallet data that just happens to be stored in the CCoinControl + # object because it is convenient to pass around. + hasLockTime @18 :UInt32; + lockTime @19 :UInt32; + hasVersion @20 :UInt32; + version @21 :UInt32; + setSelected @22 :List(Data); +} + +struct WalletTx $Proxy.wrap("interfaces::WalletTx") { + tx @0 :Data; + txinIsMine @1 :List(Int32) $Proxy.name("txin_is_mine"); + txoutIsMine @2 :List(Int32) $Proxy.name("txout_is_mine"); + txoutIsChange @3 :List(Bool) $Proxy.name("txout_is_change"); + txoutAddress @4 :List(TxDestination) $Proxy.name("txout_address"); + txoutAddressIsMine @5 :List(Int32) $Proxy.name("txout_address_is_mine"); + credit @6 :Int64; + debit @7 :Int64; + change @8 :Int64; + time @9 :Int64; + valueMap @10 :List(Common.Pair(Text, Text)) $Proxy.name("value_map"); + isCoinbase @11 :Bool $Proxy.name("is_coinbase"); +} + +struct WalletTxOut $Proxy.wrap("interfaces::WalletTxOut") { + txout @0 :Data; + time @1 :Int64; + depthInMainChain @2 :Int32 $Proxy.name("depth_in_main_chain"); + isSpent @3 :Bool $Proxy.name("is_spent"); +} + +struct WalletTxStatus $Proxy.wrap("interfaces::WalletTxStatus") { + blockHeight @0 :Int32 $Proxy.name("block_height"); + blocksToMaturity @1 :Int32 $Proxy.name("blocks_to_maturity"); + depthInMainChain @2 :Int32 $Proxy.name("depth_in_main_chain"); + timeReceived @3 :UInt32 $Proxy.name("time_received"); + lockTime @4 :UInt32 $Proxy.name("lock_time"); + isTrusted @5 :Bool $Proxy.name("is_trusted"); + isAbandoned @6 :Bool $Proxy.name("is_abandoned"); + isCoinbase @7 :Bool $Proxy.name("is_coinbase"); + isInMainChain @8 :Bool $Proxy.name("is_in_main_chain"); +} + +struct WalletBalances $Proxy.wrap("interfaces::WalletBalances") { + balance @0 :Int64; + unconfirmedBalance @1 :Int64 $Proxy.name("unconfirmed_balance"); + immatureBalance @2 :Int64 $Proxy.name("immature_balance"); + haveWatchOnly @3 :Bool $Proxy.name("have_watch_only"); + watchOnlyBalance @4 :Int64 $Proxy.name("watch_only_balance"); + unconfirmedWatchOnlyBalance @5 :Int64 $Proxy.name("unconfirmed_watch_only_balance"); + immatureWatchOnlyBalance @6 :Int64 $Proxy.name("immature_watch_only_balance"); +} + +struct WalletMigrationResult $Proxy.wrap("interfaces::WalletMigrationResult") { + wallet @0 :Wallet; + watchonlyWalletName @1 :Text $Proxy.name("watchonly_wallet_name"); + solvablesWalletName @2 :Text $Proxy.name("solvables_wallet_name"); + backupPath @3 :Text $Proxy.name("backup_path"); +} diff --git a/src/ipc/capnp/wallet.cpp b/src/ipc/capnp/wallet.cpp new file mode 100644 index 0000000000000..40556da36650e --- /dev/null +++ b/src/ipc/capnp/wallet.cpp @@ -0,0 +1,222 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include