fix(skymp5-server): fix OpenSSL conflicts by rewriting in libsodium (skyrim-multiplayer#2608)

Pospelove · web-flow · commit 842cfef583df · 2026-01-13T22:38:58.000+05:00
diff --git a/skymp5-server/cpp/CMakeLists.txt b/skymp5-server/cpp/CMakeLists.txt
@@ -108,7 +108,7 @@ if(NOT EMSCRIPTEN)
   find_package(mongoc-1.0 CONFIG REQUIRED)
 endif()
 
-find_package(OpenSSL REQUIRED)
+find_package(unofficial-sodium CONFIG REQUIRED)
 
 # MacOS not supported by rsm-bsa
 if(NOT APPLE AND NOT EMSCRIPTEN)
@@ -137,7 +137,7 @@ foreach(target ${VCPKG_DEPENDENT})
     target_compile_definitions(${target} PUBLIC NO_MONGO=1)
   endif()
 
-  target_link_libraries(${target} PUBLIC OpenSSL::SSL OpenSSL::Crypto)
+  target_link_libraries(${target} PUBLIC unofficial-sodium::sodium)
 
   if(TARGET bsa::bsa)
     target_link_libraries(${target} PUBLIC bsa::bsa)
diff --git a/skymp5-server/cpp/server_guest_lib/OpenSSLSigner.cpp b/skymp5-server/cpp/server_guest_lib/OpenSSLSigner.cpp
@@ -1,91 +1,123 @@
 #include "OpenSSLSigner.h"
 
-#include <memory>
+#include <algorithm>
+#include <cstring>
+#include <sodium.h>
 #include <stdexcept>
 #include <string_view>
+#include <vector>
 
-#include <antigo/Context.h>
-#include <openssl/bio.h>
-#include <openssl/err.h>
-#include <openssl/evp.h>
-#include <openssl/pem.h>
-#include <spdlog/spdlog.h>
-
-using namespace std::string_view_literals;
+using namespace std::string_literals;
 
 namespace {
-void OpenSSLThrow(std::string_view msgUser)
+
+std::vector<unsigned char> Base64Decode(std::string_view input)
 {
-  ANTIGO_CONTEXT_INIT(ctx);
-  std::string msgSsl;
-  auto callback = [](const char* str, size_t len, void* u) -> int {
-    auto& msgSsl = *reinterpret_cast<std::string*>(u);
-    msgSsl.insert(msgSsl.end(), str, str + len);
-    return len;
-  };
-  ERR_print_errors_cb(callback, &msgSsl);
-  while (!msgSsl.empty() && msgSsl.ends_with('\n')) {
-    msgSsl.pop_back();
+  // Remove newlines if any
+  std::string clean;
+  clean.reserve(input.size());
+  for (char c : input) {
+    if (!isspace(c))
+      clean.push_back(c);
   }
-  throw std::runtime_error(fmt::format("{}: openssl: {}", msgUser, msgSsl));
-}
 
-template <class T, class F>
-auto OpenSSLPtrWrap(T* ptr, F cb,
-                    std::string_view msg = "init returned null pointer"sv)
-{
-  ANTIGO_CONTEXT_INIT(ctx);
-  if (ptr == nullptr) {
-    OpenSSLThrow(msg);
+  size_t required_len = clean.size() / 4 * 3; // Estimate
+  std::vector<unsigned char> out(required_len + 10);
+  size_t bin_len;
+
+  if (sodium_base642bin(out.data(), out.size(), clean.c_str(), clean.size(),
+                        nullptr, &bin_len, nullptr,
+                        sodium_base64_VARIANT_ORIGINAL) != 0) {
+    throw std::runtime_error("Base64 decode failed");
   }
-  return OpenSSLSignerImpl::UniquePtrWithDeleter<T>(
-    ptr, OpenSSLSignerImpl::Deleter<T>{ cb });
+  out.resize(bin_len);
+  return out;
 }
 
-std::string Base64Encode(const unsigned char* data, size_t dataLen)
+std::vector<unsigned char> ParseEd25519Pem(const std::string& pkeyPem)
 {
-  size_t encLen = EVP_ENCODE_LENGTH(dataLen);
-  std::string buf(encLen, '\0');
-  int realLen = EVP_EncodeBlock(reinterpret_cast<unsigned char*>(buf.data()),
-                                data, dataLen);
-  if (realLen <= 0) {
-    OpenSSLThrow("base64 encode failed");
+  // Basic PEM parser for "PRIVATE KEY" (PKCS#8 for Ed25519)
+  // We expect header/footer and Base64 content.
+
+  std::string_view header = "-----BEGIN PRIVATE KEY-----";
+  std::string_view footer = "-----END PRIVATE KEY-----";
+
+  auto start = pkeyPem.find(header);
+  if (start == std::string::npos)
+    throw std::runtime_error("Missing PEM header");
+  start += header.size();
+
+  auto end = pkeyPem.find(footer, start);
+  if (end == std::string::npos)
+    throw std::runtime_error("Missing PEM footer");
+
+  auto b64 = std::string_view(pkeyPem).substr(start, end - start);
+  auto der = Base64Decode(b64);
+
+  // Parse PKCS#8 structure for Ed25519.
+  // The structure typically involves an algorithm identifier and the key octet
+  // string. For Ed25519, the private key seed is the last 32 bytes of the
+  // 48-byte structure.
+
+  if (der.size() == 48) {
+    return std::vector<unsigned char>(der.end() - 32, der.end());
+  } else if (der.size() == 32) {
+    return der;
+  } else {
+    throw std::runtime_error(
+      "Unsupported PEM key size or format. Expected 48-byte PKCS#8 Ed25519.");
   }
-  buf.resize(static_cast<size_t>(realLen));
-  return buf;
 }
+
 } // namespace
 
 OpenSSLPrivateKey::OpenSSLPrivateKey(const std::string& pkeyPem)
 {
-  auto bio = OpenSSLPtrWrap(BIO_new_mem_buf(pkeyPem.c_str(), pkeyPem.length()),
-                            BIO_free, "could not allocate io buffer for pkey");
-  pkey = OpenSSLPtrWrap(
-    PEM_read_bio_PrivateKey(bio.get(), nullptr, nullptr, nullptr),
-    EVP_PKEY_free, "could not parse pkey");
+  static bool g_initStatus = (sodium_init() >= 0);
+  if (!g_initStatus) {
+    throw std::runtime_error("libsodium initialization failed");
+  }
+
+  auto seed = ParseEd25519Pem(pkeyPem);
+  if (seed.size() != crypto_sign_SEEDBYTES) {
+    throw std::runtime_error("Invalid Ed25519 seed size");
+  }
+
+  secretKey.resize(crypto_sign_SECRETKEYBYTES);
+  std::vector<unsigned char> pk(crypto_sign_PUBLICKEYBYTES);
+
+  crypto_sign_seed_keypair(pk.data(), secretKey.data(), seed.data());
 }
 
 OpenSSLSigner::OpenSSLSigner(std::shared_ptr<OpenSSLPrivateKey> pkey_)
   : pkey(pkey_)
-  , sslMdCtx(OpenSSLPtrWrap(EVP_MD_CTX_new(), EVP_MD_CTX_free,
-                            "could not create MD_CTX"))
 {
-  if (EVP_DigestSignInit(sslMdCtx.get(), nullptr, nullptr, nullptr,
-                         pkey->GetWrapped()) <= 0) {
-    OpenSSLThrow("failed to init sign");
-  }
+  if (!pkey)
+    throw std::runtime_error("pkey is null");
 }
 
 std::string OpenSSLSigner::SignB64(const unsigned char* data, size_t len)
 {
-  ANTIGO_CONTEXT_INIT(ctx);
-  size_t sigLen;
-  if (EVP_DigestSign(sslMdCtx.get(), nullptr, &sigLen, data, len) <= 0) {
-    OpenSSLThrow("failed to get signature len");
-  }
-  std::vector<unsigned char> sig(sigLen);
-  if (EVP_DigestSign(sslMdCtx.get(), sig.data(), &sigLen, data, len) <= 0) {
-    OpenSSLThrow("failed to get signature");
+  if (!pkey)
+    throw std::runtime_error("pkey is null");
+
+  std::vector<unsigned char> sig(crypto_sign_BYTES);
+  unsigned long long sigLen_out;
+
+  crypto_sign_detached(sig.data(), &sigLen_out, data, len,
+                       pkey->GetSecretKey().data());
+
+  // Base64 encode signature
+  size_t b64maxlen = sodium_base64_ENCODED_LEN(crypto_sign_BYTES,
+                                               sodium_base64_VARIANT_ORIGINAL);
+  std::string b64(b64maxlen, '\0');
+
+  if (sodium_bin2base64(b64.data(), b64maxlen, sig.data(), sigLen_out,
+                        sodium_base64_VARIANT_ORIGINAL) == nullptr) {
+    throw std::runtime_error("Base64 encode failed");
   }
-  return Base64Encode(sig.data(), sig.size());
+
+  b64.resize(strlen(b64.c_str()));
+
+  return b64;
 }
diff --git a/skymp5-server/cpp/server_guest_lib/OpenSSLSigner.h b/skymp5-server/cpp/server_guest_lib/OpenSSLSigner.h
@@ -1,52 +1,18 @@
 #pragma once
 
 #include <memory>
-#include <stdexcept>
 #include <string>
-#include <type_traits>
-#include <variant>
-
-// openssl
-typedef struct evp_pkey_st EVP_PKEY;
-typedef struct evp_md_ctx_st EVP_MD_CTX;
-
-namespace OpenSSLSignerImpl {
-template <class T>
-struct Deleter
-{
-  using FreeCbIntT = int (*)(T*);
-  using FreeCbVoidT = void (*)(T*);
-  std::variant<FreeCbIntT, FreeCbVoidT> freeCb;
-
-  void operator()(T* ptr) const
-  {
-    std::visit(
-      [ptr](auto cb) {
-        if constexpr (std::is_same_v<decltype(cb(ptr)), int>) {
-          if (cb(ptr) <= 0) {
-            throw std::runtime_error("dealloc failed");
-          }
-        } else {
-          cb(ptr);
-        }
-      },
-      freeCb);
-  }
-};
-
-template <class T>
-using UniquePtrWithDeleter = std::unique_ptr<T, Deleter<T>>;
-} // namespace OpenSSLSignerImpl
+#include <vector>
 
 class OpenSSLPrivateKey
 {
 public:
   explicit OpenSSLPrivateKey(const std::string& pkeyPem);
 
-  EVP_PKEY* GetWrapped() { return pkey.get(); }
+  const std::vector<unsigned char>& GetSecretKey() const { return secretKey; }
 
 private:
-  OpenSSLSignerImpl::UniquePtrWithDeleter<EVP_PKEY> pkey;
+  std::vector<unsigned char> secretKey;
 };
 
 class OpenSSLSigner
@@ -58,5 +24,4 @@ class OpenSSLSigner
 
 private:
   std::shared_ptr<OpenSSLPrivateKey> pkey;
-  OpenSSLSignerImpl::UniquePtrWithDeleter<EVP_MD_CTX> sslMdCtx;
 };
diff --git a/skymp5-server/cpp/server_guest_lib/database_drivers/MongoDatabase.cpp b/skymp5-server/cpp/server_guest_lib/database_drivers/MongoDatabase.cpp
@@ -18,7 +18,7 @@
 #include <atomic>
 #include <map>
 #include <mutex>
-#include <openssl/sha.h>
+#include <sodium.h>
 #include <spdlog/spdlog.h>
 #include <thread>
 #include <vector>
@@ -321,9 +321,9 @@ std::string MongoDatabase::BytesToHexString(const uint8_t* bytes,
 
 std::string MongoDatabase::Sha256(const std::string& str)
 {
-  uint8_t hash[SHA256_DIGEST_LENGTH];
-  SHA256(reinterpret_cast<const uint8_t*>(str.data()), str.size(), hash);
-  return BytesToHexString(hash, SHA256_DIGEST_LENGTH);
+  unsigned char hash[crypto_hash_sha256_BYTES];
+  crypto_hash_sha256(hash, reinterpret_cast<const unsigned char*>(str.data()), str.size());
+  return BytesToHexString(hash, crypto_hash_sha256_BYTES);
 }
 
 #endif // #ifndef NO_MONGO
diff --git a/unit/MongoDatabaseTest.cpp b/unit/MongoDatabaseTest.cpp
@@ -2,7 +2,7 @@
 #include "TestUtils.hpp"
 #include <catch2/catch_test_macros.hpp>
 #include <nlohmann/json.hpp>
-#include <openssl/sha.h>
+#include <sodium.h>
 #include <simdjson.h>
 
 namespace MongoDatabaseTestUtils {
@@ -21,9 +21,9 @@ std::string BytesToHexString(const uint8_t* bytes, size_t length)
 
 std::string Sha256(const std::string& str)
 {
-  uint8_t hash[SHA256_DIGEST_LENGTH];
-  SHA256(reinterpret_cast<const uint8_t*>(str.data()), str.size(), hash);
-  return BytesToHexString(hash, SHA256_DIGEST_LENGTH);
+  unsigned char hash[crypto_hash_sha256_BYTES];
+  crypto_hash_sha256(hash, reinterpret_cast<const unsigned char*>(str.data()), str.size());
+  return BytesToHexString(hash, crypto_hash_sha256_BYTES);
 }
 }
 
diff --git a/unit/OpenSSLSignerTest.cpp b/unit/OpenSSLSignerTest.cpp
@@ -28,3 +28,14 @@ MC4CAQAwBQYDK2VwBCIEIILh6zzNPCOMUWMvW4QXXXPPWbyJoNL8ggkqiD+2mZdp
             "2Walov9ocK6JYClLhF3DhJ49yBQ==");
   }
 }
+
+TEST_CASE("OpenSSLSigner Invalid Key Size")
+{
+  std::string badPem = "-----BEGIN PRIVATE KEY-----\n"
+                       "AAAA\n"
+                       "-----END PRIVATE KEY-----";
+
+  REQUIRE_THROWS_WITH(
+    std::make_shared<OpenSSLPrivateKey>(badPem),
+    "Unsupported PEM key size or format. Expected 48-byte PKCS#8 Ed25519.");
+}

Original file line number	Diff line number	Diff line change
`@@ -2,7 +2,7 @@`
`2`	`2`	`#include "TestUtils.hpp"`
`3`	`3`	`#include <catch2/catch_test_macros.hpp>`
`4`	`4`	`#include <nlohmann/json.hpp>`
`5`		`-#include <openssl/sha.h>`
	`5`	`+#include <sodium.h>`
`6`	`6`	`#include <simdjson.h>`
`7`	`7`
`8`	`8`	`namespace MongoDatabaseTestUtils {`
`@@ -21,9 +21,9 @@ std::string BytesToHexString(const uint8_t* bytes, size_t length)`
`21`	`21`
`22`	`22`	`std::string Sha256(const std::string& str)`
`23`	`23`	`{`
`24`		`- uint8_t hash[SHA256_DIGEST_LENGTH];`
`25`		`- SHA256(reinterpret_cast<const uint8_t*>(str.data()), str.size(), hash);`
`26`		`- return BytesToHexString(hash, SHA256_DIGEST_LENGTH);`
	`24`	`+ unsigned char hash[crypto_hash_sha256_BYTES];`
	`25`	`+ crypto_hash_sha256(hash, reinterpret_cast<const unsigned char*>(str.data()), str.size());`
	`26`	`+ return BytesToHexString(hash, crypto_hash_sha256_BYTES);`
`27`	`27`	`}`
`28`	`28`	`}`
`29`	`29`