Merge pull request #1 from basiliscos/0.2.0-dev

0.2.0-dev

CMakeLists.txt

@@ -24,7 +24,7 @@ find_package(OpenSSL REQUIRED)
 find_package(Protobuf REQUIRED)
 find_package(ZLIB REQUIRED)
 
-set(SYNCSPIRIT_VERSION "v0.1.0")
+set(SYNCSPIRIT_VERSION "v0.2.0")
 
 configure_file(misc/syncspirit-config.h.in include/syncspirit-config.h @ONLY)
 set(Protobuf_IMPORT_DIRS ${syncspirit_SOURCE_DIR}/src)
@@ -97,6 +97,7 @@ add_library(syncspirit_lib
     src/model/diff/modify/local_update.cpp
     src/model/diff/modify/lock_file.cpp
     src/model/diff/modify/new_file.cpp
+    src/model/diff/modify/relay_connect_request.cpp
     src/model/diff/modify/share_folder.cpp
     src/model/diff/modify/update_contact.cpp
     src/model/diff/modify/update_peer.cpp
@@ -128,18 +129,21 @@ add_library(syncspirit_lib
     src/net/dialer_actor.cpp
     src/net/global_discovery_actor.cpp
     src/net/http_actor.cpp
+    src/net/initiator_actor.cpp
     src/net/local_discovery_actor.cpp
     src/net/messages.cpp
     src/net/net_supervisor.cpp
     src/net/peer_actor.cpp
     src/net/peer_supervisor.cpp
+    src/net/relay_actor.cpp
     src/net/resolver_actor.cpp
     src/net/names.cpp
     src/net/ssdp_actor.cpp
     src/net/upnp_actor.cpp
     src/proto/bep_support.cpp
     src/proto/discovery_support.cpp
     src/proto/luhn32.cpp
+    src/proto/relay_support.cpp
     src/proto/upnp_support.cpp
     src/transport/stream.cpp
     src/transport/http.cpp
@@ -164,6 +168,8 @@ set(BUILD_SHARED_LIBS false CACHE BOOL "BUILD_SHARED_LIBS")
 set(LZ4_BUILD_CLI false CACHE BOOL "LZ4_BUILD_CLI")
 set(LZ4_BUILD_LEGACY_LZ4C false CACHE BOOL "LZ4_BUILD_LEGACY_LZ4C")
 set(MDBX_BUILD_TOOLS false CACHE BOOL "MDBX_BUILD_TOOLS")
+set(MDBX_ENABLE_TESTS false CACHE BOOL "MDBX_ENABLE_TESTS")
+set(MDBX_BUILD_CXX false CACHE BOOL "MDBX_BUILD_CXX")
 
 cmake_policy(SET CMP0077 NEW)
 set(PUGIXML_NO_EXCEPTIONS on)

README.md

@@ -1,51 +1,52 @@
 # syncspirit
 
-sites: [github](https://github.com/basiliscos/syncspirit), [abf](https://github.com/basiliscos/syncspirit)
+sites: [github](https://github.com/basiliscos/syncspirit), [abf](https://github.com/basiliscos/syncspirit),
+[gitflic](https://gitflic.ru/project/basiliscos/syncspirit)
 
 `syncspirit` is a continuous file synchronization program, which synchronizes files between devices.
 It is build using C++ [rotor](github.com/basiliscos/cpp-rotor) actor framework. It implements
 [BEP-protocol](https://docs.syncthing.net/specs/bep-v1.html) for files syncrhonization, or, 
 simplistically speaking, it is [syncthing](https://syncthing.net)-compatible syncrhonization
-program.
+program, which uses [syncthing](https://syncthing.net) infrastructure (for global discovery
+and relaying)
 
 Despite of being functional `syncspirit` is much less feature-rich then [syncthing](https://syncthing.net)
 and still is in heavy development.
 
 
 # status
 
- [x] downloading files from peer devices (aka all folders are receive only)
+- [x] downloading files from peer devices (aka all folders are receive only)
 
- [x] [global peer discovery](https://docs.syncthing.net/specs/globaldisco-v3.html)
+- [x] [global peer discovery](https://docs.syncthing.net/specs/globaldisco-v3.html)
 
- [x] [local (LAN) peer discovery](https://docs.syncthing.net/specs/localdisco-v4.html)
+- [x] [local (LAN) peer discovery](https://docs.syncthing.net/specs/localdisco-v4.html)
 
- [x] upnp & nat passthough
+- [x] upnp & nat passthough
 
- [x] certificates generation
+- [x] certificates generation
 
+- [x] relay transport
 
 # missing features
 
 This list is probably incomplete, here are the most important changes
 
- [ ] relay transport
+- [ ] full-powered files synchronization (aka send and receive)
 
- [ ] full-powered files synchronization (aka send and receive)
+- [ ] conflict resolution
 
- [ ] conflict resolution
+- [ ] ignoring files
 
- [ ] ingoring files
+- [ ] [QUIC transport](https://en.wikipedia.org/wiki/QUIC)
 
- [ ] [QUIC transport](https://en.wikipedia.org/wiki/QUIC)
+- [ ] introducer support
 
- [ ] introducer support
+- [ ] outgoing messages compression
 
- [ ] outgoing messages compression
+- [ ] [untrusted devices encryption](https://docs.syncthing.net/specs/untrusted.html)
 
- [ ] [untrusted devices encryption](https://docs.syncthing.net/specs/untrusted.html)
-
- [ ] ...
+- [ ] ...
 
 # run
 
@@ -63,7 +64,7 @@ the output should be like
 [![asciicast](https://asciinema.org/a/474217.svg)](https://asciinema.org/a/474217)
 
 i.e. it records some peer, adds a folder, then shares the folder with the peer device, connects to
-the peer and downloads all files into `/tmp/my_dir/data` . The peer device Currently can be
+the peer and downloads all files into `/tmp/my_dir/data` . The peer device currently can be
 only [syncthing](https://syncthing.net). Then `syncspirit` either exits after 2 minutes of inactivity
 or when you press `ctrl+c`. The output is successful, because I previousy authorized this device
 with [syncthing](https://syncthing.net) web interface, and shared the folder with this device
@@ -92,7 +93,7 @@ classical desktop and client-server application models. I think,
 "core" into multiple different user interfaces (GUIs). 
 
 Currently, there syncspirit has only "daemon-ui", i.e. a simple non-interactive application,
-which shows synchronization log, and the only possible just stop it. However, as soon
+which shows synchronization log, and the only possibility is just to stop it. However, as soon
 as the "core" will be complete, there are plans to develop multiple `syncspirit` UIs:
 [wx-widgets](https://www.wxwidgets.org/), qt, gtk, may be native, may be even native mobile UIs...
 
@@ -117,6 +118,14 @@ after the core completion.
 
 # changes
 
+## 0.2.0 (22-May-2022)
+- [feature] implement [relay transport](https://docs.syncthing.net/specs/relay-v1.html),
+the relay is randombly chosen from the public relays [pool](https://relays.syncthing.net/endpoint)
+- [feature] output binary is compressed via [upx](https://upx.github.io)
+- [feature] small optimization, use thread less in overall program
+- [bugfix] sometimes fs::scan_actor request timeout ocurrs, which is fatal
+- [bugfix] global discovery sometimes skipped announcements
+
 ## 0.1.0 (18-Arp-2022)
  - initial release
 

docs/config.md

@@ -70,6 +70,12 @@ timeout = 5000
 # the amount of hasher threads
 hasher_threads = 3
 
+[relay]
+enabled = true
+# where pick the list of relay servers pool
+discovery_url = 'https://relays.syncthing.net/endpoint'
+rx_buff_size = 1048576
+
 [upnp]
 enabled = true
 # do output of upnp requests

misc/ubuntu14.04.toolchain

@@ -0,0 +1,8 @@
+set(TOOLCHAIN_HOME "/home/b/x-tools/x86_64-ubuntu14.04-linux-gnu")
+set(CMAKE_C_COMPILER   ${TOOLCHAIN_HOME}/bin/x86_64-ubuntu14.04-linux-gnu-gcc)
+set(CMAKE_CXX_COMPILER ${TOOLCHAIN_HOME}/bin/x86_64-ubuntu14.04-linux-gnu-g++)
+set(CMAKE_EXE_LINKER_FLAGS "-static-libgcc -static-libstdc++")
+set(CMAKE_FIND_ROOT_PATH
+    ${TOOLCHAIN_HOME}/x86_64-ubuntu14.04-linux-gnu/sysroot/
+    /home/b/development/cpp/syncspirit-cross/sysroot
+)

src/config/main.h

@@ -11,6 +11,7 @@
 #include "global_announce.h"
 #include "local_announce.h"
 #include "log.h"
+#include "relay.h"
 #include "upnp.h"
 #include <boost/filesystem.hpp>
 
@@ -31,6 +32,7 @@ struct main_t {
     dialer_config_t dialer_config;
     fs_config_t fs_config;
     db_config_t db_config;
+    relay_config_t relay_config;
 
     std::uint32_t timeout;
     std::string device_name;

src/config/relay.h

@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+// SPDX-FileCopyrightText: 2022 Ivan Baidakou
+
+#pragma once
+
+#include <string>
+
+namespace syncspirit::config {
+
+struct relay_config_t {
+    bool enabled;
+    std::string discovery_url;
+    std::uint32_t rx_buff_size;
+};
+
+} // namespace syncspirit::config

src/config/utils.cpp

@@ -22,51 +22,6 @@ static const std::string home_path = "~/.config/syncspirit";
 
 namespace syncspirit::config {
 
-bool operator==(const bep_config_t &lhs, const bep_config_t &rhs) noexcept {
-    return lhs.rx_buff_size == rhs.rx_buff_size && lhs.connect_timeout == rhs.connect_timeout &&
-           lhs.request_timeout == rhs.request_timeout && lhs.tx_timeout == rhs.tx_timeout &&
-           lhs.rx_timeout == rhs.rx_timeout && lhs.blocks_max_requested == rhs.blocks_max_requested;
-}
-
-bool operator==(const dialer_config_t &lhs, const dialer_config_t &rhs) noexcept {
-    return lhs.enabled == rhs.enabled && lhs.redial_timeout == rhs.redial_timeout;
-}
-
-bool operator==(const fs_config_t &lhs, const fs_config_t &rhs) noexcept {
-    return lhs.temporally_timeout == rhs.temporally_timeout && lhs.mru_size == rhs.mru_size;
-}
-
-bool operator==(const db_config_t &lhs, const db_config_t &rhs) noexcept {
-    return lhs.upper_limit == rhs.upper_limit && lhs.uncommited_threshold == rhs.uncommited_threshold;
-}
-
-bool operator==(const global_announce_config_t &lhs, const global_announce_config_t &rhs) noexcept {
-    return lhs.enabled == rhs.enabled && lhs.announce_url == rhs.announce_url && lhs.device_id == rhs.device_id &&
-           lhs.cert_file == rhs.cert_file && lhs.key_file == rhs.key_file && lhs.rx_buff_size == rhs.rx_buff_size &&
-           lhs.timeout == rhs.timeout && lhs.reannounce_after == rhs.reannounce_after;
-}
-
-bool operator==(const local_announce_config_t &lhs, const local_announce_config_t &rhs) noexcept {
-    return lhs.enabled == rhs.enabled && lhs.port == rhs.port && lhs.frequency == rhs.frequency;
-}
-
-bool operator==(const log_config_t &lhs, const log_config_t &rhs) noexcept {
-    return lhs.name == rhs.name && lhs.level == rhs.level && lhs.sinks == rhs.sinks;
-}
-
-bool operator==(const main_t &lhs, const main_t &rhs) noexcept {
-    return lhs.local_announce_config == rhs.local_announce_config && lhs.upnp_config == rhs.upnp_config &&
-           lhs.global_announce_config == rhs.global_announce_config && lhs.bep_config == rhs.bep_config &&
-           lhs.db_config == rhs.db_config && lhs.timeout == rhs.timeout && lhs.device_name == rhs.device_name &&
-           lhs.config_path == rhs.config_path && lhs.log_configs == rhs.log_configs &&
-           lhs.hasher_threads == rhs.hasher_threads;
-}
-
-bool operator==(const upnp_config_t &lhs, const upnp_config_t &rhs) noexcept {
-    return lhs.enabled == rhs.enabled && lhs.max_wait == rhs.max_wait && lhs.external_port == rhs.external_port &&
-           lhs.rx_buff_size == rhs.rx_buff_size && lhs.debug == rhs.debug;
-}
-
 using device_name_t = outcome::result<std::string>;
 
 static std::string expand_home(const std::string &path, const char *home) {
@@ -282,6 +237,30 @@ config_result_t get_config(std::istream &config, const boost::filesystem::path &
         c.debug = debug.value();
     };
 
+    // relay
+    {
+        auto t = root_tbl["relay"];
+        auto &c = cfg.relay_config;
+
+        auto enabled = t["enabled"].value<bool>();
+        if (!enabled) {
+            return "relay/enabled is incorrect or missing";
+        }
+        c.enabled = enabled.value();
+
+        auto discovery_url = t["discovery_url"].value<std::string>();
+        if (!discovery_url) {
+            return "upnp/discovery_url is incorrect or missing";
+        }
+        c.discovery_url = discovery_url.value();
+
+        auto rx_buff_size = t["rx_buff_size"].value<std::uint32_t>();
+        if (!rx_buff_size) {
+            return "relay/rx_buff_size is incorrect or missing";
+        }
+        c.rx_buff_size = rx_buff_size.value();
+    };
+
     // bep
     {
         auto t = root_tbl["bep"];
@@ -467,6 +446,11 @@ outcome::result<void> serialize(const main_t cfg, std::ostream &out) noexcept {
                    {"upper_limit", cfg.db_config.upper_limit},
                    {"uncommited_threshold", cfg.db_config.uncommited_threshold},
                }}},
+        {"relay", toml::table{{
+                      {"enabled", cfg.relay_config.enabled},
+                      {"discovery_url", cfg.relay_config.discovery_url},
+                      {"rx_buff_size", cfg.relay_config.rx_buff_size},
+                  }}},
     }};
     // clang-format on
     out << tbl;
@@ -559,8 +543,12 @@ outcome::result<main_t> generate_config(const boost::filesystem::path &config_pa
         0x400000000,   /* upper_limit, 16Gb */
         150,           /* uncommited_threshold */
     };
+    cfg.relay_config = relay_config_t {
+        true,                                       /* enabled */
+        "https://relays.syncthing.net/endpoint",    /* discovery url */
+        1024 * 1024,                                /* rx buff size */
+    };
     return cfg;
 }
 
-
 }

src/config/utils.h

@@ -13,17 +13,6 @@ namespace outcome = boost::outcome_v2;
 
 // comparators
 
-SYNCSPIRIT_API bool operator==(const bep_config_t &lhs, const bep_config_t &rhs) noexcept;
-SYNCSPIRIT_API bool operator==(const dialer_config_t &lhs, const dialer_config_t &rhs) noexcept;
-SYNCSPIRIT_API bool operator==(const fs_config_t &lhs, const fs_config_t &rhs) noexcept;
-SYNCSPIRIT_API bool operator==(const db_config_t &lhs, const db_config_t &rhs) noexcept;
-SYNCSPIRIT_API bool operator==(const global_announce_config_t &lhs, const global_announce_config_t &rhs) noexcept;
-SYNCSPIRIT_API bool operator==(const local_announce_config_t &lhs, const local_announce_config_t &rhs) noexcept;
-SYNCSPIRIT_API bool operator==(const main_t &lhs, const main_t &rhs) noexcept;
-SYNCSPIRIT_API bool operator==(const upnp_config_t &lhs, const upnp_config_t &rhs) noexcept;
-
-// misc
-
 using config_result_t = outcome::outcome<main_t, std::string>;
 
 SYNCSPIRIT_API config_result_t get_config(std::istream &config, const boost::filesystem::path &config_path);

src/constants.cpp

@@ -7,5 +7,6 @@ namespace syncspirit::constants {
 const char *client_name = "syncspirit";
 const char *issuer_name = "syncthing";
 const char *protocol_name = "bep/1.0";
+const char *relay_protocol_name = "bep-relay";
 const char *client_version = "v0.01";
 } // namespace syncspirit::constants

src/constants.h

@@ -11,5 +11,6 @@ static const constexpr std::uint32_t rescan_interval = 3600;
 extern const char *client_name;
 extern const char *issuer_name;
 extern const char *protocol_name;
+extern const char *relay_protocol_name;
 
 } // namespace syncspirit::constants

src/hasher/hasher_actor.cpp

@@ -2,7 +2,6 @@
 // SPDX-FileCopyrightText: 2019-2022 Ivan Baidakou
 
 #include "hasher_actor.h"
-#include "../net/names.h"
 #include "../utils/tls.h"
 #include <fmt/fmt.h>
 #include <zlib.h>

src/hasher/hasher_proxy_actor.cpp

@@ -2,7 +2,6 @@
 // SPDX-FileCopyrightText: 2019-2022 Ivan Baidakou
 
 #include "hasher_proxy_actor.h"
-#include "../net/names.h"
 #include "../utils/error_code.h"
 #include <fmt/fmt.h>
 #include <numeric>

src/model/device.cpp

@@ -50,7 +50,8 @@ outcome::result<device_ptr_t> device_t::create(const device_id_t &device_id, std
 
 device_t::device_t(const device_id_t &device_id_, std::string_view name_, std::string_view cert_name_) noexcept
     : id(std::move(device_id_)), name{name_}, compression{proto::Compression::METADATA}, cert_name{cert_name_},
-      introducer{false}, auto_accept{false}, paused{false}, skip_introduction_removals{false}, online{false} {}
+      introducer{false}, auto_accept{false}, paused{false},
+      skip_introduction_removals{false}, state{device_state_t::offline} {}
 
 void device_t::update(const db::Device &source) noexcept { assign(source); }
 
@@ -82,14 +83,16 @@ std::string device_t::serialize() noexcept {
     return r.SerializeAsString();
 }
 
-void device_t::mark_online(bool value) noexcept { online = value; }
+void device_t::update_state(device_state_t new_state) { state = new_state; }
 
 std::string_view device_t::get_key() const noexcept { return id.get_key(); }
 
 void device_t::assing_uris(const uris_t &uris_) noexcept { uris = uris_; }
 
 local_device_t::local_device_t(const device_id_t &device_id, std::string_view name, std::string_view cert_name) noexcept
-    : device_t(device_id, name, cert_name) {}
+    : device_t(device_id, name, cert_name) {
+    state = device_state_t::online;
+}
 
 std::string_view local_device_t::get_key() const noexcept { return local_device_id.get_key(); }