Add support for Light MLS (#436)

* Move tree storage from vector to map.

* Add tree slice logic

* Add a flags extension

* Add the ability for clients to join as light

* Implement LightCommit more or less along the lines in the draft

* Fix build errors after rebase

* Add the ability for a light client to upgrade to being a full client

* Add the ability to handle an AnnotatedCommit

* Add the ability to generate annotated commits

* Add an AnnotatedCommit test

* Handle GroupContextExtensions and PSKs via proposals, not annotations

* Optionally run passive client tests as a light client

* Add DS utilities library

* Use TreeFollower in light passive client tests

* Update AnnotatedCommit to work with new commit structure

* Enable light clients to process external joins

* Properly ignore sender membership proofs in the external commit case

* Revert in-Welcome sending of membership proofs

* Add a test case that shows client fast-join behavior

* Fix external commit test

* Clean up AnnotatedWelcome API

* Allow light clients to validate parent hashes

* clang-tidy

* clang-format

* Revert changes to passive client test vector processing

* Fix shadowing warnings on Windows

* clang-format

* Add missing mls_ds dependency

* Fix windows build

* Remove unneeded flags extension

* Don't replicate proposals in the AnnotatedCommit

* clang-format

.clang-tidy

@@ -19,6 +19,7 @@ Checks:            '*,
                     -google-runtime-references,
                     -hicpp-no-assembler,
                     -hicpp-special-member-functions,-warnings-as-errors,
+                    -hicpp-signed-bitwise,
                     -llvm-include-order,
                     -llvmlibc-callee-namespace,
                     -llvmlibc-implementation-in-namespace,

include/mls/common.h

@@ -216,11 +216,21 @@ std::vector<Value>
 transform(const Container& c, const UnaryOperation& op)
 {
   auto out = std::vector<Value>{};
-  auto ins = std::inserter(out, out.begin());
+  auto ins = std::back_inserter(out);
   std::transform(c.begin(), c.end(), ins, op);
   return out;
 }
 
+template<typename Value, typename Container, typename UnaryOperation>
+std::vector<Value>
+filter(const Container& c, const UnaryOperation& op)
+{
+  auto out = std::vector<Value>{};
+  auto ins = std::back_inserter(out);
+  std::copy_if(c.begin(), c.end(), ins, op);
+  return out;
+}
+
 template<typename Container, typename UnaryPredicate>
 bool
 any_of(const Container& c, const UnaryPredicate& pred)

include/mls/messages.h

@@ -691,6 +691,47 @@ external_proposal(CipherSuite suite,
                   uint32_t signer_index,
                   const SignaturePrivateKey& sig_priv);
 
+struct AnnotatedWelcome
+{
+  Welcome welcome;
+
+  TreeSlice sender_membership_proof;
+  TreeSlice receiver_membership_proof;
+
+  static AnnotatedWelcome from(Welcome welcome,
+                               const TreeKEMPublicKey& tree,
+                               LeafIndex sender,
+                               LeafIndex joiner);
+
+  TreeKEMPublicKey tree() const;
+
+  TLS_SERIALIZABLE(welcome, sender_membership_proof, receiver_membership_proof);
+};
+
+struct AnnotatedCommit
+{
+  MLSMessage commit_message;
+  std::optional<TreeSlice> sender_membership_proof_before;
+  std::optional<uint32_t> resolution_index;
+
+  bytes tree_hash_after;
+  TreeSlice sender_membership_proof_after;
+  TreeSlice receiver_membership_proof_after;
+
+  static AnnotatedCommit from(LeafIndex receiver,
+                              const std::vector<MLSMessage>& proposals,
+                              const MLSMessage& commit_message,
+                              const TreeKEMPublicKey& tree_before,
+                              const TreeKEMPublicKey& tree_after);
+
+  TLS_SERIALIZABLE(commit_message,
+                   sender_membership_proof_before,
+                   resolution_index,
+                   tree_hash_after,
+                   sender_membership_proof_after,
+                   receiver_membership_proof_after);
+};
+
 } // namespace MLS_NAMESPACE
 
 namespace MLS_NAMESPACE::tls {

include/mls/state.h

@@ -19,9 +19,16 @@ struct RosterIndex : public UInt32
 
 struct CommitOpts
 {
+  // Include these proposals in the commit by value
   std::vector<Proposal> extra_proposals;
-  bool inline_tree;
-  bool force_path;
+
+  // Send a ratchet_tree extension in the Welcome
+  bool inline_tree = false;
+
+  // Send an UpdatePath even if none is required
+  bool force_path = false;
+
+  // Update the committer's LeafNode in the following way
   LeafNodeOptions leaf_node_opts;
 };
 
@@ -127,6 +134,14 @@ class State
   std::optional<State> handle(const ValidatedContent& content_auth,
                               std::optional<State> cached_state);
 
+  ///
+  /// Light MLS
+  ///
+  void implant_tree_slice(const TreeSlice& slice);
+  State handle(const AnnotatedCommit& annotated_commit);
+  bool is_full_client() const { return _tree.is_complete(); }
+  void upgrade_to_full_client(TreeKEMPublicKey tree);
+
   ///
   /// PSK management
   ///
@@ -327,6 +342,7 @@ class State
   CommitMaterials prepare_commit(const bytes& leaf_secret,
                                  const std::optional<CommitOpts>& opts,
                                  const CommitParams& params) const;
+  GroupInfo group_info(bool external_pub, bool inline_tree) const;
   Welcome welcome(bool inline_tree,
                   const std::vector<PSKWithSecret>& psks,
                   const std::vector<KeyPackage>& joiners,

include/mls/tree_math.h

@@ -99,8 +99,8 @@ struct NodeIndex : public UInt32
   // of `ancestor` that is not in the direct path of this node.
   NodeIndex sibling(NodeIndex ancestor) const;
 
-  std::vector<NodeIndex> dirpath(LeafCount n);
-  std::vector<NodeIndex> copath(LeafCount n);
+  std::vector<NodeIndex> dirpath(LeafCount n) const;
+  std::vector<NodeIndex> copath(LeafCount n) const;
 
   uint32_t level() const;
 };

include/mls/treekem.h

@@ -59,6 +59,18 @@ struct OptionalNode
   TLS_SERIALIZABLE(node)
 };
 
+struct TreeSlice
+{
+  LeafIndex leaf_index;
+  LeafCount n_leaves;
+  std::vector<OptionalNode> direct_path_nodes;
+  std::vector<bytes> copath_hashes;
+
+  bytes tree_hash(CipherSuite suite) const;
+
+  TLS_SERIALIZABLE(leaf_index, n_leaves, direct_path_nodes, copath_hashes);
+};
+
 struct TreeKEMPublicKey;
 
 struct TreeKEMPrivateKey
@@ -113,15 +125,19 @@ struct TreeKEMPrivateKey
   void implant(const TreeKEMPublicKey& pub,
                NodeIndex start,
                const bytes& path_secret);
+  void implant_matching(const TreeKEMPublicKey& pub,
+                        NodeIndex start,
+                        const bytes& path_secret);
 };
 
 struct TreeKEMPublicKey
 {
   CipherSuite suite;
   LeafCount size{ 0 };
-  std::vector<OptionalNode> nodes;
+  std::map<NodeIndex, OptionalNode> nodes;
 
   explicit TreeKEMPublicKey(CipherSuite suite);
+  TreeKEMPublicKey(CipherSuite suite, const TreeSlice& slice);
 
   TreeKEMPublicKey() = default;
   TreeKEMPublicKey(const TreeKEMPublicKey& other) = default;
@@ -148,14 +164,29 @@ struct TreeKEMPublicKey
   const bytes& get_hash(NodeIndex index);
   bytes root_hash() const;
 
+  bool parent_hash_valid(LeafIndex from) const;
   bool parent_hash_valid(LeafIndex from, const UpdatePath& path) const;
   bool parent_hash_valid() const;
+  bool is_complete() const;
 
   bool has_leaf(LeafIndex index) const;
   std::optional<LeafIndex> find(const LeafNode& leaf) const;
   std::optional<LeafNode> leaf_node(LeafIndex index) const;
   std::vector<NodeIndex> resolve(NodeIndex index) const;
 
+  TreeSlice extract_slice(LeafIndex leaf) const;
+  void implant_slice(const TreeSlice& slice);
+  std::tuple<HPKECiphertext, NodeIndex> slice_path(UpdatePath path,
+                                                   LeafIndex from,
+                                                   LeafIndex to) const;
+
+  struct AncestorIndex
+  {
+    size_t ancestor_node_index;
+    NodeIndex resolution_node;
+  };
+  AncestorIndex ancestor_index(LeafIndex to, LeafIndex from) const;
+
   struct DecapCoords
   {
     size_t ancestor_node_index;
@@ -171,6 +202,13 @@ struct TreeKEMPublicKey
   bool all_leaves(const UnaryPredicate& pred) const
   {
     for (LeafIndex i{ 0 }; i < size; i.val++) {
+      // Only test known nodes
+      // XXX(RLB) This could be dangerous, since it allows for nodes to fail the
+      // predicate as long as they are unknown.
+      if (nodes.count(NodeIndex(i)) == 0) {
+        continue;
+      }
+
       const auto& node = node_at(i);
       if (node.blank()) {
         continue;
@@ -201,8 +239,8 @@ struct TreeKEMPublicKey
     return false;
   }
 
-  using FilteredDirectPath =
-    std::vector<std::tuple<NodeIndex, std::vector<NodeIndex>>>;
+  using FilteredDirectPathEntry = std::tuple<NodeIndex, std::vector<NodeIndex>>;
+  using FilteredDirectPath = std::vector<FilteredDirectPathEntry>;
   FilteredDirectPath filtered_direct_path(NodeIndex index) const;
 
   void truncate();
@@ -225,6 +263,9 @@ struct TreeKEMPublicKey
   void clear_hash_path(LeafIndex index);
 
   bool has_parent_hash(NodeIndex child, const bytes& target_ph) const;
+  bool parent_hash_valid(LeafIndex from,
+                         const UpdatePath& path,
+                         const FilteredDirectPath& fdp) const;
 
   bytes parent_hash(const ParentNode& parent, NodeIndex copath_child) const;
   std::vector<bytes> parent_hashes(
@@ -245,6 +286,8 @@ struct TreeKEMPublicKey
   bool exists_in_tree(const SignaturePublicKey& key,
                       std::optional<LeafIndex> except) const;
 
+  void implant_slice_unchecked(const TreeSlice& slice);
+
   OptionalNode blank_node;
 
   friend struct TreeKEMPrivateKey;

lib/CMakeLists.txt

@@ -1,4 +1,5 @@
 add_subdirectory(bytes)
 add_subdirectory(hpke)
 add_subdirectory(tls_syntax)
+add_subdirectory(mls_ds)
 add_subdirectory(mls_vectors)

lib/bytes/src/bytes.cpp

@@ -30,8 +30,6 @@ bytes::operator==(const std::vector<uint8_t>& other) const
 
   unsigned char diff = 0;
   for (size_t i = 0; i < size; ++i) {
-    // Not sure why the linter thinks `diff` is signed
-    // NOLINTNEXTLINE(hicpp-signed-bitwise)
     diff |= (_data.at(i) ^ other.at(i));
   }
   return (diff == 0);

lib/hpke/src/certificate.cpp

@@ -404,7 +404,6 @@ Certificate::parse_pem(const bytes& pem)
     auto x509 = make_typed_unique<X509>(
       PEM_read_bio_X509(bio.get(), nullptr, nullptr, nullptr));
     if (!x509) {
-      // NOLINTNEXTLINE(hicpp-signed-bitwise)
       auto err = ERR_GET_REASON(ERR_peek_last_error());
       if (err == PEM_R_NO_START_LINE) {
         // No more objects to read

lib/hpke/src/rsa.cpp

@@ -31,7 +31,6 @@ RSASignature::generate_key_pair(size_t bits)
     throw openssl_error();
   }
 
-  // NOLINTNEXTLINE(hicpp-signed-bitwise)
   if (EVP_PKEY_CTX_set_rsa_keygen_bits(ctx.get(), static_cast<int>(bits)) <=
       0) {
     throw openssl_error();

lib/mls_ds/CMakeLists.txt

@@ -0,0 +1,38 @@
+set(CURRENT_LIB_NAME mls_ds)
+
+###
+### Library Config
+###
+
+file(GLOB_RECURSE LIB_HEADERS CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/include/*.h")
+file(GLOB_RECURSE LIB_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp")
+
+add_library(${CURRENT_LIB_NAME} ${LIB_HEADERS} ${LIB_SOURCES})
+add_dependencies(${CURRENT_LIB_NAME} mlspp)
+target_link_libraries(${CURRENT_LIB_NAME} mlspp bytes tls_syntax)
+target_include_directories(${CURRENT_LIB_NAME}
+  PUBLIC
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+    $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>
+    $<INSTALL_INTERFACE:include/${PROJECT_NAME}>
+)
+
+###
+### Install
+###
+
+install(TARGETS ${CURRENT_LIB_NAME} EXPORT mlspp-targets)
+install(
+  DIRECTORY
+    include/
+  DESTINATION
+    ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}
+)
+
+###
+### Tests
+###
+
+if (TESTING)
+  add_subdirectory(test)
+endif()

lib/mls_ds/README.md

@@ -0,0 +1,11 @@
+# MLS Delivery Service Tools
+
+This library provides tools that can be convenient for an MLS Delivery Service
+(DS). We do not cover the actual delivery mechanics, but instead on more
+advanced functions where the DS needs to be aware of the internals of the MLS
+protocol.
+
+For example, it is sometimes useful for the DS to maintain a view of a group's
+ratchet tree based on seeing the group's Commits (sent as PublicMessage).  To do
+this, the DS needs to parse commits and know how to apply them to the tree.
+The `TreeFollower` class provided in this library implements this functionality.

lib/mls_ds/include/mls_ds/tree_follower.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include <mls/messages.h>
+#include <mls/treekem.h>
+#include <vector>
+
+namespace MLS_NAMESPACE::mls_ds {
+
+using namespace MLS_NAMESPACE;
+
+class TreeFollower
+{
+public:
+  // Construct a one-member tree
+  TreeFollower(const KeyPackage& key_package);
+
+  // Import a tree as a starting point for future updates
+  TreeFollower(TreeKEMPublicKey tree);
+
+  // Update the tree with a set of proposals applied by a commit
+  void update(const MLSMessage& commit_message,
+              const std::vector<MLSMessage>& extra_proposals);
+
+  // Accessors
+  CipherSuite cipher_suite() const { return _suite; }
+  const TreeKEMPublicKey& tree() const { return _tree; }
+
+private:
+  CipherSuite _suite;
+  TreeKEMPublicKey _tree;
+};
+
+} // namespace MLS_NAMESPACE::mls_ds