From 096bdb25ebd2c3066e61c29539aa361a4a914798 Mon Sep 17 00:00:00 2001 From: Connor Cihula Date: Thu, 2 Jul 2026 11:23:13 -0500 Subject: [PATCH] cmake: Add OpenSSL as an alternative to MbedTLS Give Linux builds the option to choose OpenSSL as the crypto backend with `-DCRYPTO_BACKEND=openssl`. MbedTLS remains the default, but can be explicitly set with `-DCRYPTO_BACKEND=mbedtls`. MbedTLS updates have introduced breaking changes in minor releases, leading to issues like #11020. OpenSSL tends not to have this problem. Upstream librtmp added OpenSSL RTMPS support in commit 8251927. This is a backport from upstream, along with modifications to use non-deprecated OpenSSL APIs, per-RTMP TLS state, certificate verification, and frontend signature verification. --- CMakeLists.txt | 24 +++++ frontend/cmake/feature-whatsnew.cmake | 15 ++-- frontend/cmake/os-windows.cmake | 3 - frontend/utility/crypto-helpers-openssl.cpp | 52 +++++++++++ plugins/CMakeLists.txt | 2 - plugins/obs-outputs/CMakeLists.txt | 15 ++-- plugins/obs-outputs/librtmp/handshake.h | 81 +++++++++++++---- plugins/obs-outputs/librtmp/hashswf.c | 82 +++++++++++++---- plugins/obs-outputs/librtmp/rtmp.c | 98 +++++++++++++++++---- plugins/obs-outputs/librtmp/rtmp.h | 7 +- 10 files changed, 311 insertions(+), 68 deletions(-) create mode 100644 frontend/utility/crypto-helpers-openssl.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 7ae56113192815..bdd101cb8bbac9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,9 +16,33 @@ include(defaults) include(helpers) option(ENABLE_FRONTEND "Enable building with UI (requires Qt)" ON) +option(ENABLE_PLUGINS "Enable building OBS plugins" ON) option(ENABLE_SCRIPTING "Enable scripting support" ON) option(ENABLE_HEVC "Enable HEVC encoders" ON) +set(CRYPTO_BACKEND + "mbedtls" + CACHE STRING "Crypto backend for OBS (mbedtls or openssl)" +) +set_property(CACHE CRYPTO_BACKEND PROPERTY STRINGS mbedtls openssl) + +string(TOLOWER "${CRYPTO_BACKEND}" CRYPTO_BACKEND_U) + +if(ENABLE_PLUGINS OR (OS_WINDOWS AND ENABLE_FRONTEND)) + if(CRYPTO_BACKEND_U STREQUAL "mbedtls") + set(CMAKE_FIND_PACKAGE_PREFER_CONFIG TRUE) + find_package(MbedTLS REQUIRED) + set(CMAKE_FIND_PACKAGE_PREFER_CONFIG FALSE) + elseif(CRYPTO_BACKEND_U STREQUAL "openssl") + if(NOT OS_LINUX) + message(FATAL_ERROR "OpenSSL is only supported as a crypto backend on Linux.") + endif() + find_package(OpenSSL 1.1.0 REQUIRED) + else() + message(FATAL_ERROR "Unsupported CRYPTO_BACKEND value: ${CRYPTO_BACKEND}. Expected 'mbedtls' or 'openssl'.") + endif() +endif() + add_subdirectory(libobs) if(OS_WINDOWS) add_subdirectory(libobs-d3d11) diff --git a/frontend/cmake/feature-whatsnew.cmake b/frontend/cmake/feature-whatsnew.cmake index fe5fb4437ec6f8..9f67fc39d5db60 100644 --- a/frontend/cmake/feature-whatsnew.cmake +++ b/frontend/cmake/feature-whatsnew.cmake @@ -6,16 +6,21 @@ if(ENABLE_WHATSNEW AND TARGET OBS::browser-panels) if(OS_MACOS) include(cmake/feature-macos-update.cmake) elseif(OS_LINUX) - set(CMAKE_FIND_PACKAGE_PREFER_CONFIG TRUE) - find_package(MbedTLS REQUIRED) - set(CMAKE_FIND_PACKAGE_PREFER_CONFIG FALSE) find_package(nlohmann_json 3.11 REQUIRED) + if(CRYPTO_BACKEND_U STREQUAL "mbedtls") + set(crypto_helpers_source utility/crypto-helpers-mbedtls.cpp) + set(crypto_helpers_library MbedTLS::mbedtls) + elseif(CRYPTO_BACKEND_U STREQUAL "openssl") + set(crypto_helpers_source utility/crypto-helpers-openssl.cpp) + set(crypto_helpers_library OpenSSL::Crypto) + endif() + if(NOT TARGET OBS::blake2) add_subdirectory("${CMAKE_SOURCE_DIR}/deps/blake2" "${CMAKE_BINARY_DIR}/deps/blake2") endif() - target_link_libraries(obs-studio PRIVATE MbedTLS::mbedtls nlohmann_json::nlohmann_json OBS::blake2) + target_link_libraries(obs-studio PRIVATE ${crypto_helpers_library} nlohmann_json::nlohmann_json OBS::blake2) target_sources( obs-studio @@ -24,7 +29,7 @@ if(ENABLE_WHATSNEW AND TARGET OBS::browser-panels) utility/WhatsNewBrowserInitThread.hpp utility/WhatsNewInfoThread.cpp utility/WhatsNewInfoThread.hpp - utility/crypto-helpers-mbedtls.cpp + ${crypto_helpers_source} utility/crypto-helpers.hpp utility/models/whatsnew.hpp utility/update-helpers.cpp diff --git a/frontend/cmake/os-windows.cmake b/frontend/cmake/os-windows.cmake index e8c6e0ca868287..c5dacf2971ad18 100644 --- a/frontend/cmake/os-windows.cmake +++ b/frontend/cmake/os-windows.cmake @@ -6,9 +6,6 @@ if(NOT TARGET OBS::w32-pthreads) add_subdirectory("${CMAKE_SOURCE_DIR}/deps/w32-pthreads" "${CMAKE_BINARY_DIR}/deps/w32-pthreads") endif() -set(CMAKE_FIND_PACKAGE_PREFER_CONFIG TRUE) -find_package(MbedTLS REQUIRED) -set(CMAKE_FIND_PACKAGE_PREFER_CONFIG FALSE) find_package(Detours REQUIRED) find_package(nlohmann_json 3.11 REQUIRED) diff --git a/frontend/utility/crypto-helpers-openssl.cpp b/frontend/utility/crypto-helpers-openssl.cpp new file mode 100644 index 00000000000000..b5310f1a4a9307 --- /dev/null +++ b/frontend/utility/crypto-helpers-openssl.cpp @@ -0,0 +1,52 @@ +#include "crypto-helpers.hpp" + +#include + +#include +#include +#include + +bool VerifySignature(const uint8_t *pubKey, const size_t pubKeyLen, const uint8_t *buf, const size_t len, + const uint8_t *sig, const size_t sigLen) +{ + bool result = false; + BIO *bio = nullptr; + EVP_PKEY *key = nullptr; + EVP_MD_CTX *context = nullptr; + + if (pubKeyLen > static_cast(std::numeric_limits::max())) { + goto exit; + } + + bio = BIO_new_mem_buf(pubKey, static_cast(pubKeyLen)); + if (!bio) { + goto exit; + } + + key = PEM_read_bio_PUBKEY(bio, nullptr, nullptr, nullptr); + if (!key) { + goto exit; + } + + context = EVP_MD_CTX_new(); + if (!context) { + goto exit; + } + + if (EVP_DigestVerifyInit(context, nullptr, EVP_sha512(), nullptr, key) != 1) { + goto exit; + } + + if (EVP_DigestVerifyUpdate(context, buf, len) != 1) { + goto exit; + } + + result = EVP_DigestVerifyFinal(context, sig, sigLen) == 1; + +exit: + EVP_MD_CTX_free(context); + EVP_PKEY_free(key); + BIO_free(bio); + + return result; +} diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index 1bea2e4ff92d02..34b688d37ba72b 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -1,7 +1,5 @@ cmake_minimum_required(VERSION 3.28...3.30) -option(ENABLE_PLUGINS "Enable building OBS plugins" ON) - if(NOT ENABLE_PLUGINS) set_property(GLOBAL APPEND PROPERTY OBS_FEATURES_DISABLED "Plugin Support") return() diff --git a/plugins/obs-outputs/CMakeLists.txt b/plugins/obs-outputs/CMakeLists.txt index 4d3778df6c9f65..4300adc9ab36aa 100644 --- a/plugins/obs-outputs/CMakeLists.txt +++ b/plugins/obs-outputs/CMakeLists.txt @@ -1,8 +1,5 @@ cmake_minimum_required(VERSION 3.28...3.30) -set(CMAKE_FIND_PACKAGE_PREFER_CONFIG TRUE) -find_package(MbedTLS REQUIRED) -set(CMAKE_FIND_PACKAGE_PREFER_CONFIG FALSE) find_package(ZLIB REQUIRED) find_package(jansson REQUIRED) @@ -63,7 +60,15 @@ target_sources( utils.h ) -target_compile_definitions(obs-outputs PRIVATE USE_MBEDTLS CRYPTO) +target_compile_definitions(obs-outputs PRIVATE CRYPTO) + +if(CRYPTO_BACKEND_U STREQUAL "mbedtls") + target_compile_definitions(obs-outputs PRIVATE USE_MBEDTLS) + set(librtmp_crypto_libraries MbedTLS::mbedtls) +elseif(CRYPTO_BACKEND_U STREQUAL "openssl") + target_compile_definitions(obs-outputs PRIVATE USE_OPENSSL) + set(librtmp_crypto_libraries OpenSSL::SSL OpenSSL::Crypto) +endif() target_compile_options( obs-outputs @@ -79,7 +84,7 @@ target_link_libraries( OBS::happy-eyeballs OBS::opts-parser OBS::bpm - MbedTLS::mbedtls + ${librtmp_crypto_libraries} ZLIB::ZLIB jansson::jansson $<$:OBS::w32-pthreads> diff --git a/plugins/obs-outputs/librtmp/handshake.h b/plugins/obs-outputs/librtmp/handshake.h index 0114bdec0f4042..30744dee9375ed 100644 --- a/plugins/obs-outputs/librtmp/handshake.h +++ b/plugins/obs-outputs/librtmp/handshake.h @@ -24,11 +24,12 @@ /* This file is #included in rtmp.c, it is not meant to be compiled alone */ -#if defined(USE_MBEDTLS) -#include #ifndef SHA256_DIGEST_LENGTH #define SHA256_DIGEST_LENGTH 32 #endif + +#if defined(USE_MBEDTLS) +#include typedef mbedtls_md_context_t *HMAC_CTX; #define HMAC_setup(ctx, key, len) ctx = malloc(sizeof(mbedtls_md_context_t)); mbedtls_md_init(ctx); \ mbedtls_md_setup(ctx, mbedtls_md_info_from_type(MBEDTLS_MD_SHA256), 1); \ @@ -39,9 +40,6 @@ typedef mbedtls_md_context_t *HMAC_CTX; #elif defined(USE_POLARSSL) #include -#ifndef SHA256_DIGEST_LENGTH -#define SHA256_DIGEST_LENGTH 32 -#endif #define HMAC_CTX sha2_context #define HMAC_setup(ctx, key, len) sha2_hmac_starts(&ctx, (unsigned char *)key, len, 0) #define HMAC_crunch(ctx, buf, len) sha2_hmac_update(&ctx, buf, len) @@ -50,9 +48,6 @@ typedef mbedtls_md_context_t *HMAC_CTX; #elif defined(USE_GNUTLS) #include #include -#ifndef SHA256_DIGEST_LENGTH -#define SHA256_DIGEST_LENGTH 32 -#endif #undef HMAC_CTX #define HMAC_CTX struct hmac_sha256_ctx #define HMAC_setup(ctx, key, len) hmac_sha256_set_key(&ctx, len, key) @@ -60,15 +55,67 @@ typedef mbedtls_md_context_t *HMAC_CTX; #define HMAC_finish(ctx, dig) hmac_sha256_digest(&ctx, SHA256_DIGEST_LENGTH, dig) #define HMAC_close(ctx) -#else /* USE_OPENSSL */ +#elif defined(USE_OPENSSL) +#include #include -#include -#if OPENSSL_VERSION_NUMBER < 0x0090800 || !defined(SHA256_DIGEST_LENGTH) -#error Your OpenSSL is too old, need 0.9.8 or newer with SHA256 -#endif -#define HMAC_setup(ctx, key, len) HMAC_CTX_init(&ctx); HMAC_Init_ex(&ctx, key, len, EVP_sha256(), 0) -#define HMAC_crunch(ctx, buf, len) HMAC_Update(&ctx, buf, len) -#define HMAC_finish(ctx, dig, len) HMAC_Final(&ctx, dig, &len); HMAC_CTX_cleanup(&ctx) +typedef struct librtmp_hmac_ctx { + EVP_MD_CTX *md_ctx; + EVP_PKEY *key; +} RTMP_HMAC_CTX; +#define HMAC_CTX RTMP_HMAC_CTX * +static inline int +HMAC_setup_openssl(HMAC_CTX *ctx, const uint8_t *key_data, size_t len) +{ + *ctx = calloc(1, sizeof(**ctx)); + if (!*ctx) { + RTMP_Log(RTMP_LOGERROR, "%s: failed to allocate HMAC context", __FUNCTION__); + return FALSE; + } + + (*ctx)->md_ctx = EVP_MD_CTX_new(); + if (!(*ctx)->md_ctx) { + RTMP_Log(RTMP_LOGERROR, "%s: failed to allocate EVP message digest context", + __FUNCTION__); + goto fail; + } + + (*ctx)->key = EVP_PKEY_new_mac_key(EVP_PKEY_HMAC, NULL, key_data, (int)len); + if (!(*ctx)->key) { + RTMP_Log(RTMP_LOGERROR, "%s: failed to allocate EVP HMAC key", __FUNCTION__); + goto fail; + } + + if (EVP_DigestSignInit((*ctx)->md_ctx, NULL, EVP_sha256(), NULL, (*ctx)->key) != 1) { + RTMP_Log(RTMP_LOGERROR, "%s: failed to initialize EVP HMAC context", __FUNCTION__); + goto fail; + } + + return TRUE; + +fail: + EVP_MD_CTX_free((*ctx)->md_ctx); + EVP_PKEY_free((*ctx)->key); + free(*ctx); + *ctx = NULL; + return FALSE; +} +#define HMAC_setup(ctx, key_data, len) HMAC_setup_openssl(&ctx, key_data, len) +#define HMAC_crunch(ctx, buf, len) EVP_DigestSignUpdate((ctx)->md_ctx, buf, len) +#define HMAC_close(ctx) do { \ + EVP_MD_CTX_free((ctx)->md_ctx); \ + EVP_PKEY_free((ctx)->key); \ + free(ctx); \ + ctx = NULL; \ +} while (0) +static inline void +HMAC_finish_openssl(HMAC_CTX *ctx, uint8_t *dig, unsigned int *len) +{ + size_t hmac_len = SHA256_DIGEST_LENGTH; + EVP_DigestSignFinal((*ctx)->md_ctx, dig, &hmac_len); + *len = (unsigned int)hmac_len; + HMAC_close(*ctx); +} +#define HMAC_finish(ctx, dig, len) HMAC_finish_openssl(&ctx, dig, &len) #endif #define FP10 @@ -180,7 +227,7 @@ HMACsha256(const uint8_t *message, size_t messageLen, const uint8_t *key, #if defined(USE_MBEDTLS) || defined(USE_POLARSSL) || defined(USE_GNUTLS) digestLen = SHA256_DIGEST_LENGTH; HMAC_finish(ctx, digest); -#else +#elif defined(USE_OPENSSL) HMAC_finish(ctx, digest, digestLen); #endif diff --git a/plugins/obs-outputs/librtmp/hashswf.c b/plugins/obs-outputs/librtmp/hashswf.c index b9cff31f85c2a2..5c8c9951766081 100644 --- a/plugins/obs-outputs/librtmp/hashswf.c +++ b/plugins/obs-outputs/librtmp/hashswf.c @@ -25,6 +25,10 @@ #include "log.h" #include "http.h" +#ifndef SHA256_DIGEST_LENGTH +#define SHA256_DIGEST_LENGTH 32 +#endif + #ifdef CRYPTO #ifdef __APPLE__ @@ -33,9 +37,6 @@ #if defined(USE_MBEDTLS) #include -#ifndef SHA256_DIGEST_LENGTH -#define SHA256_DIGEST_LENGTH 32 -#endif typedef mbedtls_md_context_t *HMAC_CTX; #define HMAC_setup(ctx, key, len) ctx = malloc(sizeof(mbedtls_md_context_t)); mbedtls_md_init(ctx); \ mbedtls_md_setup(ctx, mbedtls_md_info_from_type(MBEDTLS_MD_SHA256), 1); \ @@ -46,9 +47,6 @@ typedef mbedtls_md_context_t *HMAC_CTX; #elif defined(USE_POLARSSL) #include -#ifndef SHA256_DIGEST_LENGTH -#define SHA256_DIGEST_LENGTH 32 -#endif #define HMAC_CTX sha2_context #define HMAC_setup(ctx, key, len) sha2_hmac_starts(&ctx, (unsigned char *)key, len, 0) #define HMAC_crunch(ctx, buf, len) sha2_hmac_update(&ctx, buf, len) @@ -57,9 +55,6 @@ typedef mbedtls_md_context_t *HMAC_CTX; #elif defined(USE_GNUTLS) #include -#ifndef SHA256_DIGEST_LENGTH -#define SHA256_DIGEST_LENGTH 32 -#endif #undef HMAC_CTX #define HMAC_CTX struct hmac_sha256_ctx #define HMAC_setup(ctx, key, len) hmac_sha256_set_key(&ctx, len, key) @@ -67,15 +62,68 @@ typedef mbedtls_md_context_t *HMAC_CTX; #define HMAC_finish(ctx, dig) hmac_sha256_digest(&ctx, SHA256_DIGEST_LENGTH, dig) #define HMAC_close(ctx) -#else /* USE_OPENSSL */ +#elif defined(USE_OPENSSL) #include +#include #include -#include #include -#define HMAC_setup(ctx, key, len) HMAC_CTX_init(&ctx); HMAC_Init_ex(&ctx, (unsigned char *)key, len, EVP_sha256(), 0) -#define HMAC_crunch(ctx, buf, len) HMAC_Update(&ctx, (unsigned char *)buf, len) -#define HMAC_finish(ctx, dig, len) HMAC_Final(&ctx, (unsigned char *)dig, &len); -#define HMAC_close(ctx) HMAC_CTX_cleanup(&ctx) +typedef struct librtmp_hmac_ctx { + EVP_MD_CTX *md_ctx; + EVP_PKEY *key; +} RTMP_HMAC_CTX; +#define HMAC_CTX RTMP_HMAC_CTX * +static inline int +HMAC_setup_openssl(HMAC_CTX *ctx, const uint8_t *key_data, size_t len) +{ + *ctx = calloc(1, sizeof(**ctx)); + if (!*ctx) { + RTMP_Log(RTMP_LOGERROR, "%s: failed to allocate HMAC context", __FUNCTION__); + return FALSE; + } + + (*ctx)->md_ctx = EVP_MD_CTX_new(); + if (!(*ctx)->md_ctx) { + RTMP_Log(RTMP_LOGERROR, "%s: failed to allocate EVP message digest context", + __FUNCTION__); + goto fail; + } + + (*ctx)->key = EVP_PKEY_new_mac_key(EVP_PKEY_HMAC, NULL, key_data, (int)len); + if (!(*ctx)->key) { + RTMP_Log(RTMP_LOGERROR, "%s: failed to allocate EVP HMAC key", __FUNCTION__); + goto fail; + } + + if (EVP_DigestSignInit((*ctx)->md_ctx, NULL, EVP_sha256(), NULL, (*ctx)->key) != 1) { + RTMP_Log(RTMP_LOGERROR, "%s: failed to initialize EVP HMAC context", __FUNCTION__); + goto fail; + } + + return TRUE; + +fail: + EVP_MD_CTX_free((*ctx)->md_ctx); + EVP_PKEY_free((*ctx)->key); + free(*ctx); + *ctx = NULL; + return FALSE; +} +#define HMAC_setup(ctx, key_data, len) HMAC_setup_openssl(&ctx, (const unsigned char *)key_data, len) +#define HMAC_crunch(ctx, buf, len) EVP_DigestSignUpdate((ctx)->md_ctx, buf, len) +#define HMAC_close(ctx) do { \ + EVP_MD_CTX_free((ctx)->md_ctx); \ + EVP_PKEY_free((ctx)->key); \ + free(ctx); \ + ctx = NULL; \ +} while (0) +static inline void +HMAC_finish_openssl(HMAC_CTX ctx, unsigned char *dig, unsigned int *len) +{ + size_t hmac_len = SHA256_DIGEST_LENGTH; + EVP_DigestSignFinal(ctx->md_ctx, dig, &hmac_len); + *len = (unsigned int)hmac_len; +} +#define HMAC_finish(ctx, dig, len) HMAC_finish_openssl(ctx, dig, &len) #endif #include @@ -184,7 +232,7 @@ HTTP_get(struct HTTP_ctx *http, const char *url, HTTP_read_callback *cb) #endif int connect_return = TLS_connect(sb.sb_ssl); - if (connect_return < 0) + if (!TLS_success(connect_return)) { RTMP_Log(RTMP_LOGERROR, "%s, TLS_Connect failed", __FUNCTION__); ret = HTTPRES_LOST_CONNECTION; @@ -659,7 +707,7 @@ RTMP_HashSWF(const char *url, unsigned int *size, unsigned char *hash, { #if defined(USE_MBEDTLS) || defined(USE_POLARSSL) || defined(USE_GNUTLS) HMAC_finish(in.ctx, hash); -#else +#elif defined(USE_OPENSSL) HMAC_finish(in.ctx, hash, hlen); #endif *size = in.size; diff --git a/plugins/obs-outputs/librtmp/rtmp.c b/plugins/obs-outputs/librtmp/rtmp.c index 1898558f51927a..099b8a82fe36dc 100644 --- a/plugins/obs-outputs/librtmp/rtmp.c +++ b/plugins/obs-outputs/librtmp/rtmp.c @@ -80,11 +80,13 @@ static const char *my_dhm_G = "4"; #define MD5_DIGEST_LENGTH 16 #include #include -#else /* USE_OPENSSL */ +#elif defined(USE_OPENSSL) #include -#include +#include #include #include +#include +#define MD5_DIGEST_LENGTH 16 #endif #endif @@ -100,6 +102,7 @@ static const char *my_dhm_G = "4"; #define RTMP_SIG_SIZE 1536 #define RTMP_LARGE_HEADER_SIZE 12 +#define RTMP_TLS_CERT_VERIFY_FAILED -0x2700 static const int packetSize[] = { 12, 8, 4, 1 }; @@ -363,9 +366,9 @@ RTMP_TLS_LoadCerts(RTMP *r) { mbedtls_x509_crt_free(chain); free(chain); r->RTMP_TLS_ctx->cacert = NULL; -#else /* USE_MBEDTLS */ +#else UNUSED_PARAMETER(r); -#endif /* USE_MBEDTLS */ +#endif } void @@ -402,14 +405,17 @@ RTMP_TLS_Init(RTMP *r) gnutls_priority_init(&RTMP_TLS_ctx->prios, "NORMAL", NULL); gnutls_certificate_set_x509_trust_file(RTMP_TLS_ctx->cred, "ca.pem", GNUTLS_X509_FMT_PEM); -#elif !defined(NO_SSL) /* USE_OPENSSL */ - /* libcrypto doesn't need anything special */ - SSL_load_error_strings(); - SSL_library_init(); - OpenSSL_add_all_digests(); - RTMP_TLS_ctx = SSL_CTX_new(SSLv23_method()); - SSL_CTX_set_options(RTMP_TLS_ctx, SSL_OP_ALL); - SSL_CTX_set_default_verify_paths(RTMP_TLS_ctx); +#elif defined(USE_OPENSSL) && !defined(NO_SSL) + r->RTMP_TLS_ctx = SSL_CTX_new(TLS_client_method()); + if (!r->RTMP_TLS_ctx) { + RTMP_Log(RTMP_LOGERROR, "RTMP_TLS_Init: Failed to create OpenSSL context"); + return; + } + + SSL_CTX_set_options(r->RTMP_TLS_ctx, SSL_OP_ALL); + SSL_CTX_set_verify(r->RTMP_TLS_ctx, SSL_VERIFY_PEER, NULL); + if (SSL_CTX_set_default_verify_paths(r->RTMP_TLS_ctx) != 1) + RTMP_Log(RTMP_LOGWARNING, "RTMP_TLS_Init: Failed to load default OpenSSL verify paths"); #endif #else UNUSED_PARAMETER(r); @@ -419,7 +425,6 @@ RTMP_TLS_Init(RTMP *r) void RTMP_TLS_Free(RTMP *r) { #if defined(CRYPTO) && defined(USE_MBEDTLS) - if (!r->RTMP_TLS_ctx) return; mbedtls_ssl_config_free(&r->RTMP_TLS_ctx->conf); @@ -435,6 +440,11 @@ RTMP_TLS_Free(RTMP *r) { // NO mbedtls_net_free() BECAUSE WE SET IT UP BY HAND! free(r->RTMP_TLS_ctx); r->RTMP_TLS_ctx = NULL; +#elif defined(USE_OPENSSL) + if (!r->RTMP_TLS_ctx) + return; + SSL_CTX_free(r->RTMP_TLS_ctx); + r->RTMP_TLS_ctx = NULL; #else UNUSED_PARAMETER(r); #endif @@ -956,12 +966,34 @@ RTMP_Connect1(RTMP *r, RTMPPacket *cp) if (mbedtls_ssl_set_hostname(r->m_sb.sb_ssl, hostname)) return FALSE; -#else - TLS_setfd(r->m_sb.sb_ssl, r->m_sb.sb_socket); +#elif defined(USE_OPENSSL) + if (!r->m_sb.sb_ssl) + return FALSE; + + char *hostname = malloc(r->Link.hostname.av_len + 1); + if (!hostname) { + TLS_close(r->m_sb.sb_ssl); + r->m_sb.sb_ssl = NULL; + return FALSE; + } + + memcpy(hostname, r->Link.hostname.av_val, r->Link.hostname.av_len); + hostname[r->Link.hostname.av_len] = 0; + + if (SSL_set_tlsext_host_name(r->m_sb.sb_ssl, hostname) != 1 || + SSL_set1_host(r->m_sb.sb_ssl, hostname) != 1 || + TLS_setfd(r->m_sb.sb_ssl, r->m_sb.sb_socket) != 1) { + free(hostname); + TLS_close(r->m_sb.sb_ssl); + r->m_sb.sb_ssl = NULL; + return FALSE; + } + + free(hostname); #endif int connect_return = TLS_connect(r->m_sb.sb_ssl); - if (connect_return < 0) + if (!TLS_success(connect_return)) { #if defined(USE_MBEDTLS) r->last_error_code = connect_return; @@ -987,9 +1019,24 @@ RTMP_Connect1(RTMP *r, RTMPPacket *cp) return FALSE; } } +#elif defined(USE_OPENSSL) + long verify_result = SSL_get_verify_result(r->m_sb.sb_ssl); + if (verify_result != X509_V_OK) { + r->last_error_code = RTMP_TLS_CERT_VERIFY_FAILED; + RTMP_Log(RTMP_LOGERROR, "%s, Cert verify failed: %ld (%s)", __FUNCTION__, verify_result, + X509_verify_cert_error_string(verify_result)); + RTMP_Close(r); + return FALSE; + } + + r->last_error_code = SSL_get_error(r->m_sb.sb_ssl, connect_return); #endif // output the error in a format that matches mbedTLS +#ifdef USE_OPENSSL + connect_return = r->last_error_code; +#else connect_return = abs(connect_return); +#endif RTMP_Log(RTMP_LOGERROR, "%s, TLS_Connect failed: -0x%x", __FUNCTION__, connect_return); RTMP_Close(r); return FALSE; @@ -2593,7 +2640,7 @@ b64enc(const unsigned char *input, int length, char *output, int maxsize) return 0; } -#else /* USE_OPENSSL */ +#elif defined(USE_OPENSSL) BIO *bmem, *b64; BUF_MEM *bptr; @@ -2640,7 +2687,22 @@ typedef struct md5_ctx MD5_CTX; #define MD5_Init(ctx) md5_init(ctx) #define MD5_Update(ctx,data,len) md5_update(ctx,len,data) #define MD5_Final(dig,ctx) md5_digest(ctx,MD5_DIGEST_LENGTH,dig) -#else +#elif defined(USE_OPENSSL) +typedef EVP_MD_CTX *MD5_CTX; +#define MD5_Init(ctx) do { \ + *(ctx) = EVP_MD_CTX_new(); \ + EVP_DigestInit_ex(*(ctx), EVP_md5(), NULL); \ +} while (0) +#define MD5_Update(ctx,data,len) EVP_DigestUpdate(*(ctx), data, len) +static inline void +MD5_Final_openssl(unsigned char *dig, MD5_CTX *ctx) +{ + unsigned int md5_len; + EVP_DigestFinal_ex(*ctx, dig, &md5_len); + EVP_MD_CTX_free(*ctx); + *ctx = NULL; +} +#define MD5_Final(dig,ctx) MD5_Final_openssl(dig,ctx) #endif static const AVal av_authmod_adobe = AVC("authmod=adobe"); diff --git a/plugins/obs-outputs/librtmp/rtmp.h b/plugins/obs-outputs/librtmp/rtmp.h index c1b8d906175111..3b8791add8a51f 100644 --- a/plugins/obs-outputs/librtmp/rtmp.h +++ b/plugins/obs-outputs/librtmp/rtmp.h @@ -99,6 +99,7 @@ typedef tls_ctx *TLS_CTX; #define TLS_setfd(s,fd) mbedtls_ssl_set_bio(s, fd, mbedtls_net_send, mbedtls_net_recv, NULL) #define TLS_connect(s) mbedtls_ssl_handshake(s) +#define TLS_success(r) ((r) == 0) #define TLS_accept(s) mbedtls_ssl_handshake(s) #define TLS_read(s,b,l) mbedtls_ssl_read(s,(unsigned char *)b,l) #define TLS_write(s,b,l) mbedtls_ssl_write(s,(unsigned char *)b,l) @@ -132,6 +133,7 @@ typedef struct tls_ctx SSL_SET_SESSION(s, 1, 600, &ctx->ssn) #define TLS_setfd(s,fd) ssl_set_bio(s, net_recv, &fd, net_send, &fd) #define TLS_connect(s) ssl_handshake(s) +#define TLS_success(r) ((r) == 0) #define TLS_accept(s) ssl_handshake(s) #define TLS_read(s,b,l) ssl_read(s,(unsigned char *)b,l) #define TLS_write(s,b,l) ssl_write(s,(unsigned char *)b,l) @@ -150,17 +152,20 @@ typedef struct tls_ctx #define TLS_client(ctx,s) gnutls_init((gnutls_session_t *)(&s), GNUTLS_CLIENT); gnutls_priority_set(s, ctx->prios); gnutls_credentials_set(s, GNUTLS_CRD_CERTIFICATE, ctx->cred) #define TLS_setfd(s,fd) gnutls_transport_set_ptr(s, (gnutls_transport_ptr_t)(long)fd) #define TLS_connect(s) gnutls_handshake(s) +#define TLS_success(r) ((r) == 0) #define TLS_accept(s) gnutls_handshake(s) #define TLS_read(s,b,l) gnutls_record_recv(s,b,l) #define TLS_write(s,b,l) gnutls_record_send(s,b,l) #define TLS_shutdown(s) gnutls_bye(s, GNUTLS_SHUT_RDWR) #define TLS_close(s) gnutls_deinit(s) -#else /* USE_OPENSSL */ +#elif defined(USE_OPENSSL) +#include #define TLS_CTX SSL_CTX * #define TLS_client(ctx,s) s = SSL_new(ctx) #define TLS_setfd(s,fd) SSL_set_fd(s,fd) #define TLS_connect(s) SSL_connect(s) +#define TLS_success(r) ((r) == 1) #define TLS_accept(s) SSL_accept(s) #define TLS_read(s,b,l) SSL_read(s,b,l) #define TLS_write(s,b,l) SSL_write(s,b,l)