cmake: Add OpenSSL as an alternative to MbedTLS

CMakeLists.txt

@@ -16,9 +16,33 @@ include(defaults)
 include(helpers)
 
 option(ENABLE_FRONTEND "Enable building with UI (requires Qt)" ON)
+option(ENABLE_PLUGINS "Enable building OBS plugins" ON)
 option(ENABLE_SCRIPTING "Enable scripting support" ON)
 option(ENABLE_HEVC "Enable HEVC encoders" ON)
 
+set(CRYPTO_BACKEND
+    "mbedtls"
+    CACHE STRING "Crypto backend for OBS (mbedtls or openssl)"
+)
+set_property(CACHE CRYPTO_BACKEND PROPERTY STRINGS mbedtls openssl)
+
+string(TOLOWER "${CRYPTO_BACKEND}" CRYPTO_BACKEND_U)
+
+if(ENABLE_PLUGINS OR (OS_WINDOWS AND ENABLE_FRONTEND))
+  if(CRYPTO_BACKEND_U STREQUAL "mbedtls")
+    set(CMAKE_FIND_PACKAGE_PREFER_CONFIG TRUE)
+    find_package(MbedTLS REQUIRED)
+    set(CMAKE_FIND_PACKAGE_PREFER_CONFIG FALSE)
+  elseif(CRYPTO_BACKEND_U STREQUAL "openssl")
+    if(NOT OS_LINUX)
+      message(FATAL_ERROR "OpenSSL is only supported as a crypto backend on Linux.")
+    endif()
+    find_package(OpenSSL 1.1.0 REQUIRED)
+  else()
+    message(FATAL_ERROR "Unsupported CRYPTO_BACKEND value: ${CRYPTO_BACKEND}. Expected 'mbedtls' or 'openssl'.")
+  endif()
+endif()
+
 add_subdirectory(libobs)
 if(OS_WINDOWS)
   add_subdirectory(libobs-d3d11)

frontend/cmake/feature-whatsnew.cmake

@@ -6,16 +6,21 @@ if(ENABLE_WHATSNEW AND TARGET OBS::browser-panels)
   if(OS_MACOS)
     include(cmake/feature-macos-update.cmake)
   elseif(OS_LINUX)
-    set(CMAKE_FIND_PACKAGE_PREFER_CONFIG TRUE)
-    find_package(MbedTLS REQUIRED)
-    set(CMAKE_FIND_PACKAGE_PREFER_CONFIG FALSE)
     find_package(nlohmann_json 3.11 REQUIRED)
 
+    if(CRYPTO_BACKEND_U STREQUAL "mbedtls")
+      set(crypto_helpers_source utility/crypto-helpers-mbedtls.cpp)
+      set(crypto_helpers_library MbedTLS::mbedtls)
+    elseif(CRYPTO_BACKEND_U STREQUAL "openssl")
+      set(crypto_helpers_source utility/crypto-helpers-openssl.cpp)
+      set(crypto_helpers_library OpenSSL::Crypto)
+    endif()
+
     if(NOT TARGET OBS::blake2)
       add_subdirectory("${CMAKE_SOURCE_DIR}/deps/blake2" "${CMAKE_BINARY_DIR}/deps/blake2")
     endif()
 
-    target_link_libraries(obs-studio PRIVATE MbedTLS::mbedtls nlohmann_json::nlohmann_json OBS::blake2)
+    target_link_libraries(obs-studio PRIVATE ${crypto_helpers_library} nlohmann_json::nlohmann_json OBS::blake2)
 
     target_sources(
       obs-studio
@@ -24,7 +29,7 @@ if(ENABLE_WHATSNEW AND TARGET OBS::browser-panels)
         utility/WhatsNewBrowserInitThread.hpp
         utility/WhatsNewInfoThread.cpp
         utility/WhatsNewInfoThread.hpp
-        utility/crypto-helpers-mbedtls.cpp
+        ${crypto_helpers_source}
         utility/crypto-helpers.hpp
         utility/models/whatsnew.hpp
         utility/update-helpers.cpp

frontend/cmake/os-windows.cmake

@@ -6,9 +6,6 @@ if(NOT TARGET OBS::w32-pthreads)
   add_subdirectory("${CMAKE_SOURCE_DIR}/deps/w32-pthreads" "${CMAKE_BINARY_DIR}/deps/w32-pthreads")
 endif()
 
-set(CMAKE_FIND_PACKAGE_PREFER_CONFIG TRUE)
-find_package(MbedTLS REQUIRED)
-set(CMAKE_FIND_PACKAGE_PREFER_CONFIG FALSE)
 find_package(Detours REQUIRED)
 find_package(nlohmann_json 3.11 REQUIRED)
 

frontend/utility/crypto-helpers-openssl.cpp

@@ -0,0 +1,52 @@
+#include "crypto-helpers.hpp"
+
+#include <limits>
+
+#include <openssl/bio.h>
+#include <openssl/evp.h>
+#include <openssl/pem.h>
+
+bool VerifySignature(const uint8_t *pubKey, const size_t pubKeyLen, const uint8_t *buf, const size_t len,
+		     const uint8_t *sig, const size_t sigLen)
+{
+	bool result = false;
+	BIO *bio = nullptr;
+	EVP_PKEY *key = nullptr;
+	EVP_MD_CTX *context = nullptr;
+
+	if (pubKeyLen > static_cast<size_t>(std::numeric_limits<int>::max())) {
+		goto exit;
+	}
+
+	bio = BIO_new_mem_buf(pubKey, static_cast<int>(pubKeyLen));
+	if (!bio) {
+		goto exit;
+	}
+
+	key = PEM_read_bio_PUBKEY(bio, nullptr, nullptr, nullptr);
+	if (!key) {
+		goto exit;
+	}
+
+	context = EVP_MD_CTX_new();
+	if (!context) {
+		goto exit;
+	}
+
+	if (EVP_DigestVerifyInit(context, nullptr, EVP_sha512(), nullptr, key) != 1) {
+		goto exit;
+	}
+
+	if (EVP_DigestVerifyUpdate(context, buf, len) != 1) {
+		goto exit;
+	}
+
+	result = EVP_DigestVerifyFinal(context, sig, sigLen) == 1;
+
+exit:
+	EVP_MD_CTX_free(context);
+	EVP_PKEY_free(key);
+	BIO_free(bio);
+
+	return result;
+}

plugins/CMakeLists.txt

@@ -1,7 +1,5 @@
 cmake_minimum_required(VERSION 3.28...3.30)
 
-option(ENABLE_PLUGINS "Enable building OBS plugins" ON)
-
 if(NOT ENABLE_PLUGINS)
   set_property(GLOBAL APPEND PROPERTY OBS_FEATURES_DISABLED "Plugin Support")
   return()

plugins/obs-outputs/CMakeLists.txt

@@ -1,8 +1,5 @@
 cmake_minimum_required(VERSION 3.28...3.30)
 
-set(CMAKE_FIND_PACKAGE_PREFER_CONFIG TRUE)
-find_package(MbedTLS REQUIRED)
-set(CMAKE_FIND_PACKAGE_PREFER_CONFIG FALSE)
 find_package(ZLIB REQUIRED)
 find_package(jansson REQUIRED)
 
@@ -63,7 +60,15 @@ target_sources(
     utils.h
 )
 
-target_compile_definitions(obs-outputs PRIVATE USE_MBEDTLS CRYPTO)
+target_compile_definitions(obs-outputs PRIVATE CRYPTO)
+
+if(CRYPTO_BACKEND_U STREQUAL "mbedtls")
+  target_compile_definitions(obs-outputs PRIVATE USE_MBEDTLS)
+  set(librtmp_crypto_libraries MbedTLS::mbedtls)
+elseif(CRYPTO_BACKEND_U STREQUAL "openssl")
+  target_compile_definitions(obs-outputs PRIVATE USE_OPENSSL)
+  set(librtmp_crypto_libraries OpenSSL::SSL OpenSSL::Crypto)
+endif()
 
 target_compile_options(
   obs-outputs
@@ -79,7 +84,7 @@ target_link_libraries(
     OBS::happy-eyeballs
     OBS::opts-parser
     OBS::bpm
-    MbedTLS::mbedtls
+    ${librtmp_crypto_libraries}
     ZLIB::ZLIB
     jansson::jansson
     $<$<PLATFORM_ID:Windows>:OBS::w32-pthreads>

plugins/obs-outputs/librtmp/handshake.h

@@ -24,11 +24,12 @@
 
 /* This file is #included in rtmp.c, it is not meant to be compiled alone */
 
-#if defined(USE_MBEDTLS)
-#include <mbedtls/md.h>
 #ifndef SHA256_DIGEST_LENGTH
 #define SHA256_DIGEST_LENGTH	32
 #endif
+
+#if defined(USE_MBEDTLS)
+#include <mbedtls/md.h>
 typedef mbedtls_md_context_t *HMAC_CTX;
 #define HMAC_setup(ctx, key, len)	ctx = malloc(sizeof(mbedtls_md_context_t)); mbedtls_md_init(ctx); \
   mbedtls_md_setup(ctx, mbedtls_md_info_from_type(MBEDTLS_MD_SHA256), 1); \
@@ -39,9 +40,6 @@ typedef mbedtls_md_context_t *HMAC_CTX;
 
 #elif defined(USE_POLARSSL)
 #include <polarssl/sha2.h>
-#ifndef SHA256_DIGEST_LENGTH
-#define SHA256_DIGEST_LENGTH	32
-#endif
 #define HMAC_CTX	sha2_context
 #define HMAC_setup(ctx, key, len)	sha2_hmac_starts(&ctx, (unsigned char *)key, len, 0)
 #define HMAC_crunch(ctx, buf, len)	sha2_hmac_update(&ctx, buf, len)
@@ -50,25 +48,74 @@ typedef mbedtls_md_context_t *HMAC_CTX;
 #elif defined(USE_GNUTLS)
 #include <nettle/hmac.h>
 #include <nettle/arcfour.h>
-#ifndef SHA256_DIGEST_LENGTH
-#define SHA256_DIGEST_LENGTH	32
-#endif
 #undef HMAC_CTX
 #define HMAC_CTX	struct hmac_sha256_ctx
 #define HMAC_setup(ctx, key, len)	hmac_sha256_set_key(&ctx, len, key)
 #define HMAC_crunch(ctx, buf, len)	hmac_sha256_update(&ctx, len, buf)
 #define HMAC_finish(ctx, dig)		hmac_sha256_digest(&ctx, SHA256_DIGEST_LENGTH, dig)
 #define HMAC_close(ctx)
 
-#else	/* USE_OPENSSL */
+#elif defined(USE_OPENSSL)
+#include <openssl/evp.h>
 #include <openssl/sha.h>
-#include <openssl/hmac.h>
-#if OPENSSL_VERSION_NUMBER < 0x0090800 || !defined(SHA256_DIGEST_LENGTH)
-#error Your OpenSSL is too old, need 0.9.8 or newer with SHA256
-#endif
-#define HMAC_setup(ctx, key, len)	HMAC_CTX_init(&ctx); HMAC_Init_ex(&ctx, key, len, EVP_sha256(), 0)
-#define HMAC_crunch(ctx, buf, len)	HMAC_Update(&ctx, buf, len)
-#define HMAC_finish(ctx, dig, len)	HMAC_Final(&ctx, dig, &len); HMAC_CTX_cleanup(&ctx)
+typedef struct librtmp_hmac_ctx {
+    EVP_MD_CTX *md_ctx;
+    EVP_PKEY *key;
+} RTMP_HMAC_CTX;
+#define HMAC_CTX RTMP_HMAC_CTX *
+static inline int
+HMAC_setup_openssl(HMAC_CTX *ctx, const uint8_t *key_data, size_t len)
+{
+    *ctx = calloc(1, sizeof(**ctx));
+    if (!*ctx) {
+        RTMP_Log(RTMP_LOGERROR, "%s: failed to allocate HMAC context", __FUNCTION__);
+        return FALSE;
+    }
+
+    (*ctx)->md_ctx = EVP_MD_CTX_new();
+    if (!(*ctx)->md_ctx) {
+        RTMP_Log(RTMP_LOGERROR, "%s: failed to allocate EVP message digest context",
+                 __FUNCTION__);
+        goto fail;
+    }
+
+    (*ctx)->key = EVP_PKEY_new_mac_key(EVP_PKEY_HMAC, NULL, key_data, (int)len);
+    if (!(*ctx)->key) {
+        RTMP_Log(RTMP_LOGERROR, "%s: failed to allocate EVP HMAC key", __FUNCTION__);
+        goto fail;
+    }
+
+    if (EVP_DigestSignInit((*ctx)->md_ctx, NULL, EVP_sha256(), NULL, (*ctx)->key) != 1) {
+        RTMP_Log(RTMP_LOGERROR, "%s: failed to initialize EVP HMAC context", __FUNCTION__);
+        goto fail;
+    }
+
+    return TRUE;
+
+fail:
+    EVP_MD_CTX_free((*ctx)->md_ctx);
+    EVP_PKEY_free((*ctx)->key);
+    free(*ctx);
+    *ctx = NULL;
+    return FALSE;
+}
+#define HMAC_setup(ctx, key_data, len)  HMAC_setup_openssl(&ctx, key_data, len)
+#define HMAC_crunch(ctx, buf, len)      EVP_DigestSignUpdate((ctx)->md_ctx, buf, len)
+#define HMAC_close(ctx) do { \
+    EVP_MD_CTX_free((ctx)->md_ctx); \
+    EVP_PKEY_free((ctx)->key); \
+    free(ctx); \
+    ctx = NULL; \
+} while (0)
+static inline void
+HMAC_finish_openssl(HMAC_CTX *ctx, uint8_t *dig, unsigned int *len)
+{
+    size_t hmac_len = SHA256_DIGEST_LENGTH;
+    EVP_DigestSignFinal((*ctx)->md_ctx, dig, &hmac_len);
+    *len = (unsigned int)hmac_len;
+    HMAC_close(*ctx);
+}
+#define HMAC_finish(ctx, dig, len)      HMAC_finish_openssl(&ctx, dig, &len)
 #endif
 
 #define FP10
@@ -180,7 +227,7 @@ HMACsha256(const uint8_t *message, size_t messageLen, const uint8_t *key,
 #if defined(USE_MBEDTLS) || defined(USE_POLARSSL) || defined(USE_GNUTLS)
     digestLen = SHA256_DIGEST_LENGTH;
     HMAC_finish(ctx, digest);
-#else
+#elif defined(USE_OPENSSL)
     HMAC_finish(ctx, digest, digestLen);
 #endif