Add Missing Encoder Check on Startup/Profile load

frontend/data/locale/en-US.ini

@@ -1680,3 +1680,30 @@ PluginManager.Section.Manage.Title="Manage Enabled Plugins"
 
 # Custom Widget Localization
 Accessible.Widget.Name.AlignmentSelector="Alignment Selector"
+
+# Missing Encoder Warning
+EncoderMissing.Title="Missing Encoders"
+EncoderMissing.Text="The following configured encoders are missing:\n<ul>\n%1</ul><br>\nSee the <a href=\"%2\">Knowledge Base Article</a> for further information."
+EncoderMissing.ItemText="<li>\n<h3>%1</h3>\n%2 (code: <code>%3</code>)\n</li>"
+EncoderMissing.Unknown="Unknown"
+EncoderMissing.MissingModule="Module \"%1\" not loaded"
+
+## Nvenc specific errors
+EncoderMissing.NVENC.TestProgramFailedStartup="NVENC check program start failure"
+EncoderMissing.NVENC.TestProgramExitWithError="NVENC check process failure, exit code: %1"
+EncoderMissing.NVENC.TestProgramReadFailure="NVENC check process output read failure, exit code: %1"
+EncoderMissing.NVENC.TestProgramError="NVENC check failed with reason: %1"
+EncoderMissing.NVENC.NoDevices="No NVIDIA GPUs found"
+EncoderMissing.NVENC.Unsupported.Kepler="The architecture (Kepler) of your GPU (%1) is no longer supported. Please use OBS 30.2 or earlier instead."
+EncoderMissing.NVENC.Reason.NvmlLoad="Failed loading NVML library"
+EncoderMissing.NVENC.Reason.NvmlInit="Failed initializing NVML"
+EncoderMissing.NVENC.Reason.NvencLoad="Failed loading NVENC library"
+EncoderMissing.NVENC.Reason.NvencInit="Failed initializing NVENC"
+EncoderMissing.NVENC.Reason.NvencVersion="NVENC version empty"
+EncoderMissing.NVENC.Reason.CudaLoad="Failed loading CUDA library"
+EncoderMissing.NVENC.Reason.CudaInit="Failed initializing CUDA"
+EncoderMissing.NVENC.Reason.CudaVersion="Invalid/Missing CUDA version"
+EncoderMissing.NVENC.Reason.DriverOutdated="Outdated driver"
+EncoderMissing.NVENC.Reason.NoSupportedDevices="No NVIDIA devices with NVENC support found"
+EncoderMissing.NVENC.Reason.SessionLimitExceeded="Encoder session limit exceeded"
+EncoderMissing.NVENC.Reason.AV1Unsupported="AV1 encoding is not supported by any of the system's NVIDIA GPUs"

frontend/widgets/OBSBasic.hpp

@@ -917,6 +917,7 @@ private slots:
 	std::vector<std::string> GetRestartRequirements(const ConfigFile &config) const;
 	void ResetProfileData();
 	void CheckForSimpleModeX264Fallback();
+	void CheckForMissingEncoders();
 
 public:
 	inline const OBSProfileCache &GetProfileCache() const noexcept { return profiles; };

frontend/widgets/OBSBasic_Profiles.cpp

@@ -22,7 +22,9 @@
 #endif
 #include <wizards/AutoConfig.hpp>
 
+#include <unordered_set>
 #include <qt-wrappers.hpp>
+#include <util/pipe.h>
 
 #include <QDir>
 #include <QFile>
@@ -40,6 +42,7 @@ extern void DestroyPanelCookieManager();
 extern void DuplicateCurrentCookieProfile(ConfigFile &config);
 extern void CheckExistingCookieId();
 extern void DeleteCookies();
+extern const char *get_simple_output_encoder(const char *name);
 
 // MARK: - Anonymous Namespace
 namespace {
@@ -736,6 +739,7 @@ void OBSBasic::ActivateProfile(const OBSProfile &profile, bool reset)
 void OBSBasic::UpdateProfileEncoders()
 {
 	InitBasicConfigDefaults2();
+	CheckForMissingEncoders();
 	CheckForSimpleModeX264Fallback();
 }
 
@@ -912,3 +916,195 @@ void OBSBasic::CheckForSimpleModeX264Fallback()
 		activeConfiguration.SaveSafe("tmp");
 	}
 }
+
+namespace {
+using OsProcessArgs = std::unique_ptr<os_process_args_t, decltype(&os_process_args_destroy)>;
+
+struct ErrorDetails {
+	// HTML breadcrumb to section in knowledge base article, may be empty
+	QString breadcrumb;
+	// Human-readable/localised error message
+	QString message;
+};
+using CheckResult = std::optional<ErrorDetails>;
+
+#ifndef __APPLE__
+CheckResult CheckNVENCInternal(const std::unordered_set<std::string_view> &missing_encoders)
+{
+	const std::vector<std::pair<std::string_view, std::string_view>> nvencFailureReasons = {
+		{"nvml_lib", "EncoderMissing.NVENC.Reason.NvmlLoad"},
+		// Some error messages are a prefix + numeric error code
+		{"nvml_init", "EncoderMissing.NVENC.Reason.NvmlInit"},
+		{"nvenc_lib", "EncoderMissing.NVENC.Reason.NvencLoad"},
+		{"nvenc_init", "EncoderMissing.NVENC.Reason.NvencInit"},
+		{"cuda_lib", "EncoderMissing.NVENC.Reason.CudaLoad"},
+		{"cuda_init", "EncoderMissing.NVENC.Reason.CudaInit"},
+		{"no_cuda_version", "EncoderMissing.NVENC.Reason.CudaVersion"},
+		{"no_devices", "EncoderMissing.NVENC.NoDevices"},
+		{"no_nvenc_version", "EncoderMissing.NVENC.Reason.NvencVersion"},
+		{"outdated_driver", "EncoderMissing.NVENC.Reason.DriverOutdated"},
+		{"no_supported_devices", "EncoderMissing.NVENC.Reason.NoSupportedDevices"},
+		{"session_limit", "EncoderMissing.NVENC.Reason.SessionLimitExceeded"},
+	};
+
+	// Most basic check: Is the module even loaded?
+	if (!obs_get_module("obs-nvenc")) {
+		return ErrorDetails{"plugin_not_loaded", QTStr("EncoderMissing.MissingModule").arg("obs-nvenc")};
+	}
+
+#ifdef _WIN32
+	BPtr testBinary = os_get_executable_path_ptr("obs-nvenc-test.exe");
+#else
+	BPtr testBinary = os_get_executable_path_ptr("obs-nvenc-test");
+#endif
+
+	OsProcessArgs args{os_process_args_create(testBinary), os_process_args_destroy};
+
+	os_process_pipe_t *pp = os_process_pipe_create2(args.get(), "r");
+	if (!pp) {
+		return ErrorDetails{"nvenc_test_failure", QTStr("EncoderMissing.NVENC.TestProgramFailedStartup")};
+	}
+
+	std::string caps_result;
+	for (;;) {
+		char data[4096];
+		const size_t len = os_process_pipe_read(pp, reinterpret_cast<uint8_t *>(data), sizeof(data));
+		if (!len) {
+			break;
+		}
+
+		caps_result.append(data, len);
+	}
+
+	int exitCode = os_process_pipe_destroy(pp);
+
+	if (caps_result.empty()) {
+		return ErrorDetails{"nvenc_test_failure",
+				    QTStr("EncoderMissing.NVENC.TestProgramExitWithError").arg(exitCode)};
+	}
+
+	auto config = ConfigFile();
+	if (config.OpenString(caps_result.c_str()) != CONFIG_SUCCESS) {
+		return ErrorDetails{"nvenc_test_failure",
+				    QTStr("EncoderMissing.NVENC.TestProgramReadFailure").arg(exitCode)};
+	}
+
+	// Read devices and attempt to find reason for failure
+	auto numDevices = config_get_uint(config, "general", "cuda_devices");
+	if (!numDevices) {
+		return ErrorDetails{"no_device", QTStr("EncoderMissing.NVENC.NoDevices")};
+	}
+
+	// Check device generation (architecture)
+	for (uint64_t i = 0; i < numDevices; i++) {
+		std::string section = "device." + std::to_string(i);
+
+		if (!config_has_user_value(config, section.c_str(), "name")) {
+			continue;
+		}
+
+		const QString name(config_get_string(config, section.c_str(), "name"));
+		const std::string_view arch = config_get_string(config, section.c_str(), "architecture_name");
+
+		if (arch == "Kepler") {
+			return ErrorDetails{"kepler", QTStr("EncoderMissing.NVENC.Unsupported.Kepler").arg(name)};
+		}
+	}
+
+	// All other specific failures of the test binary
+	if (config_has_user_value(config, "general", "reason")) {
+		const std::string_view reason = config_get_string(config, "general", "reason");
+
+		for (auto &[code, desc] : nvencFailureReasons) {
+			if (reason.find(code) == std::string_view::npos) {
+				continue;
+			}
+
+			return ErrorDetails{code.data(),
+					    QTStr("EncoderMissing.NVENC.TestProgramError").arg(QTStr(desc.data()))};
+		}
+	}
+
+	// Finally, if everything else looks good but the encoder missing is AV1, and AV1 is not supported on any GPUs,
+	// that's probably the issue the user is having (for example, when copying a profile from another machine).
+	if (!config_get_bool(config, "av1", "codec_supported") &&
+	    (missing_encoders.count("obs_nvenc_av1_cuda") || missing_encoders.count("obs_nvenc_av1_soft") ||
+	     missing_encoders.count("obs_nvenc_av1_tex"))) {
+		return ErrorDetails{"no_av1", QTStr("EncoderMissing.NVENC.Reason.AV1Unsupported")};
+	}
+
+	return std::nullopt;
+}
+
+CheckResult CheckNVENC(const std::unordered_set<std::string_view> &missing_encoders)
+{
+	static auto result = CheckNVENCInternal(missing_encoders);
+	return result;
+}
+#endif
+
+CheckResult CheckX264()
+{
+	static const bool module_loaded = obs_get_module("obs-x264") != nullptr;
+	// This should be the only failure mode possible here.
+	if (!module_loaded) {
+		return ErrorDetails{"plugin_not_loaded", QTStr("EncoderMissing.MissingModule").arg("obs-x264")};
+	}
+
+	return std::nullopt;
+}
+} // namespace
+
+void OBSBasic::CheckForMissingEncoders()
+{
+	constexpr QStringView kbURL(u"https://obsproject.com/kb/encoder-missing#%1");
+	const QString encoderListItem = QTStr("EncoderMissing.ItemText");
+	const QString unknownErrorText = QTStr("EncoderMissing.Unknown");
+
+	std::unordered_set<std::string_view> missing_encoders = {
+		config_get_string(activeConfiguration, "AdvOut", "Encoder"),
+		config_get_string(activeConfiguration, "AdvOut", "RecEncoder"),
+		get_simple_output_encoder(config_get_string(activeConfiguration, "SimpleOutput", "StreamEncoder")),
+		get_simple_output_encoder(config_get_string(activeConfiguration, "SimpleOutput", "RecEncoder")),
+	};
+
+	size_t idx = 0;
+	const char *id;
+	while (obs_enum_encoder_types(idx++, &id)) {
+		missing_encoders.erase(id);
+	}
+
+	if (missing_encoders.empty() || (missing_encoders.size() == 1 && missing_encoders.count("none"))) {
+		return;
+	}
+
+	QString crumb;
+	QStringList encoderList;
+
+	for (auto &encoder : missing_encoders) {
+		CheckResult res;
+
+		if (encoder.find("x264") != std::string_view::npos) {
+			res = CheckX264();
+		}
+#ifndef __APPLE__
+		else if (encoder.find("nvenc") != std::string_view::npos) {
+			res = CheckNVENC(missing_encoders);
+		}
+#endif
+
+		if (res) {
+			encoderList += encoderListItem.arg(encoder.data()).arg(res->message).arg(res->breadcrumb);
+			if (crumb.isEmpty() && !res->breadcrumb.isEmpty()) {
+				crumb = res->breadcrumb;
+			}
+		} else {
+			encoderList += encoderListItem.arg(encoder.data()).arg(unknownErrorText).arg("unknown");
+		}
+	}
+
+	// Format and show error message
+	const QString encoderMissingMessage =
+		QTStr("EncoderMissing.Text").arg(encoderList.join("\n")).arg(kbURL.arg(crumb));
+	OBSMessageBox::warning(this, QTStr("EncoderMissing.Title"), encoderMissingMessage, true);
+}