From 941e7c63dcb043bf1b825f5f67c3184812715809 Mon Sep 17 00:00:00 2001 From: Zortos Date: Mon, 16 Feb 2026 22:24:56 +0100 Subject: [PATCH 1/4] chore: remove legacy opennow-streamer (Rust) directory [skip ci] --- opennow-streamer/Cargo.toml | 200 - opennow-streamer/GSTREAMER_SETUP.md | 96 - opennow-streamer/macos/Info.plist | 44 - opennow-streamer/macos/bundle.sh | 249 - opennow-streamer/reverse/audio.md | 242 - opennow-streamer/reverse/controller.md | 268 - opennow-streamer/reverse/cursor.md | 274 - opennow-streamer/reverse/datachannel.md | 306 - opennow-streamer/reverse/index.md | 147 - opennow-streamer/reverse/keyboard.md | 273 - opennow-streamer/reverse/protocol.md | 315 - opennow-streamer/reverse/rendering.md | 314 - opennow-streamer/reverse/session.md | 396 - opennow-streamer/reverse/statistics.md | 347 - opennow-streamer/reverse/video.md | 276 - opennow-streamer/reverse/win.md | 745 -- opennow-streamer/scripts/bundle-gstreamer.ps1 | 177 - opennow-streamer/scripts/dev.ps1 | 1 - opennow-streamer/scripts/run.ps1 | 1 - opennow-streamer/scripts/test-decoders.ps1 | 148 - opennow-streamer/src/api/cloudmatch.rs | 1056 --- opennow-streamer/src/api/error_codes.rs | 793 -- opennow-streamer/src/api/games.rs | 746 -- opennow-streamer/src/api/mod.rs | 479 -- opennow-streamer/src/api/queue.rs | 327 - opennow-streamer/src/app/cache.rs | 774 -- opennow-streamer/src/app/config.rs | 567 -- opennow-streamer/src/app/mod.rs | 2227 ------ opennow-streamer/src/app/session.rs | 600 -- opennow-streamer/src/app/types.rs | 311 - opennow-streamer/src/auth/mod.rs | 722 -- opennow-streamer/src/gui/image_cache.rs | 188 - opennow-streamer/src/gui/mod.rs | 13 - opennow-streamer/src/gui/renderer.rs | 6461 ----------------- opennow-streamer/src/gui/screens/login.rs | 212 - opennow-streamer/src/gui/screens/mod.rs | 866 --- opennow-streamer/src/gui/screens/session.rs | 61 - opennow-streamer/src/gui/shaders.rs | 328 - opennow-streamer/src/gui/stats_panel.rs | 248 - opennow-streamer/src/input/controller.rs | 733 -- opennow-streamer/src/input/linux.rs | 690 -- opennow-streamer/src/input/macos.rs | 509 -- opennow-streamer/src/input/mod.rs | 747 -- opennow-streamer/src/input/protocol.rs | 103 - opennow-streamer/src/input/wheel.rs | 774 -- opennow-streamer/src/input/windows.rs | 665 -- opennow-streamer/src/lib.rs | 16 - opennow-streamer/src/main.rs | 665 -- opennow-streamer/src/media/audio.rs | 1246 ---- opennow-streamer/src/media/d3d11.rs | 406 -- opennow-streamer/src/media/dxva_decoder.rs | 1597 ---- .../src/media/gstreamer_decoder.rs | 942 --- opennow-streamer/src/media/hevc_parser.rs | 1014 --- opennow-streamer/src/media/mod.rs | 359 - opennow-streamer/src/media/native_video.rs | 350 - opennow-streamer/src/media/rtp.rs | 647 -- opennow-streamer/src/media/v4l2.rs | 363 - opennow-streamer/src/media/vaapi.rs | 601 -- opennow-streamer/src/media/video.rs | 2587 ------- opennow-streamer/src/media/videotoolbox.rs | 1005 --- opennow-streamer/src/profiling.rs | 70 - opennow-streamer/src/utils/logging.rs | 157 - opennow-streamer/src/utils/mod.rs | 44 - opennow-streamer/src/utils/time.rs | 128 - opennow-streamer/src/webrtc/datachannel.rs | 661 -- opennow-streamer/src/webrtc/mod.rs | 1099 --- opennow-streamer/src/webrtc/peer.rs | 733 -- opennow-streamer/src/webrtc/sdp.rs | 480 -- 68 files changed, 41189 deletions(-) delete mode 100644 opennow-streamer/Cargo.toml delete mode 100644 opennow-streamer/GSTREAMER_SETUP.md delete mode 100644 opennow-streamer/macos/Info.plist delete mode 100755 opennow-streamer/macos/bundle.sh delete mode 100644 opennow-streamer/reverse/audio.md delete mode 100644 opennow-streamer/reverse/controller.md delete mode 100644 opennow-streamer/reverse/cursor.md delete mode 100644 opennow-streamer/reverse/datachannel.md delete mode 100644 opennow-streamer/reverse/index.md delete mode 100644 opennow-streamer/reverse/keyboard.md delete mode 100644 opennow-streamer/reverse/protocol.md delete mode 100644 opennow-streamer/reverse/rendering.md delete mode 100644 opennow-streamer/reverse/session.md delete mode 100644 opennow-streamer/reverse/statistics.md delete mode 100644 opennow-streamer/reverse/video.md delete mode 100644 opennow-streamer/reverse/win.md delete mode 100644 opennow-streamer/scripts/bundle-gstreamer.ps1 delete mode 100644 opennow-streamer/scripts/dev.ps1 delete mode 100644 opennow-streamer/scripts/run.ps1 delete mode 100644 opennow-streamer/scripts/test-decoders.ps1 delete mode 100644 opennow-streamer/src/api/cloudmatch.rs delete mode 100644 opennow-streamer/src/api/error_codes.rs delete mode 100644 opennow-streamer/src/api/games.rs delete mode 100644 opennow-streamer/src/api/mod.rs delete mode 100644 opennow-streamer/src/api/queue.rs delete mode 100644 opennow-streamer/src/app/cache.rs delete mode 100644 opennow-streamer/src/app/config.rs delete mode 100644 opennow-streamer/src/app/mod.rs delete mode 100644 opennow-streamer/src/app/session.rs delete mode 100644 opennow-streamer/src/app/types.rs delete mode 100644 opennow-streamer/src/auth/mod.rs delete mode 100644 opennow-streamer/src/gui/image_cache.rs delete mode 100644 opennow-streamer/src/gui/mod.rs delete mode 100644 opennow-streamer/src/gui/renderer.rs delete mode 100644 opennow-streamer/src/gui/screens/login.rs delete mode 100644 opennow-streamer/src/gui/screens/mod.rs delete mode 100644 opennow-streamer/src/gui/screens/session.rs delete mode 100644 opennow-streamer/src/gui/shaders.rs delete mode 100644 opennow-streamer/src/gui/stats_panel.rs delete mode 100644 opennow-streamer/src/input/controller.rs delete mode 100644 opennow-streamer/src/input/linux.rs delete mode 100644 opennow-streamer/src/input/macos.rs delete mode 100644 opennow-streamer/src/input/mod.rs delete mode 100644 opennow-streamer/src/input/protocol.rs delete mode 100644 opennow-streamer/src/input/wheel.rs delete mode 100644 opennow-streamer/src/input/windows.rs delete mode 100644 opennow-streamer/src/lib.rs delete mode 100644 opennow-streamer/src/main.rs delete mode 100644 opennow-streamer/src/media/audio.rs delete mode 100644 opennow-streamer/src/media/d3d11.rs delete mode 100644 opennow-streamer/src/media/dxva_decoder.rs delete mode 100644 opennow-streamer/src/media/gstreamer_decoder.rs delete mode 100644 opennow-streamer/src/media/hevc_parser.rs delete mode 100644 opennow-streamer/src/media/mod.rs delete mode 100644 opennow-streamer/src/media/native_video.rs delete mode 100644 opennow-streamer/src/media/rtp.rs delete mode 100644 opennow-streamer/src/media/v4l2.rs delete mode 100644 opennow-streamer/src/media/vaapi.rs delete mode 100644 opennow-streamer/src/media/video.rs delete mode 100644 opennow-streamer/src/media/videotoolbox.rs delete mode 100644 opennow-streamer/src/profiling.rs delete mode 100644 opennow-streamer/src/utils/logging.rs delete mode 100644 opennow-streamer/src/utils/mod.rs delete mode 100644 opennow-streamer/src/utils/time.rs delete mode 100644 opennow-streamer/src/webrtc/datachannel.rs delete mode 100644 opennow-streamer/src/webrtc/mod.rs delete mode 100644 opennow-streamer/src/webrtc/peer.rs delete mode 100644 opennow-streamer/src/webrtc/sdp.rs diff --git a/opennow-streamer/Cargo.toml b/opennow-streamer/Cargo.toml deleted file mode 100644 index 2266578..0000000 --- a/opennow-streamer/Cargo.toml +++ /dev/null @@ -1,200 +0,0 @@ -[package] -name = "opennow-streamer" -version = "0.1.0" -edition = "2021" -description = "High-performance native streaming client for GeForce NOW" -authors = ["OpenNow"] -license = "MIT" - -[dependencies] -# Async runtime -tokio = { version = "1", features = ["full", "sync", "time", "rt-multi-thread", "macros"] } - -# WebRTC - using forked version with provisional SSRC support for GFN -webrtc = { git = "https://github.com/zortos293/webrtc-rs-gfn", branch = "gfn-ssrc-fix" } -webrtc-util = { git = "https://github.com/zortos293/webrtc-rs-gfn", branch = "gfn-ssrc-fix" } - -# HTTP client -reqwest = { version = "0.12", features = ["json", "rustls-tls", "gzip"] } - -# WebSocket (signaling) -tokio-tungstenite = { version = "0.24", features = ["rustls-tls-native-roots"] } -futures-util = "0.3" - -# TLS -native-tls = "0.2" -tokio-native-tls = "0.3" - -# Keep OpenH264 as fallback (all platforms) -openh264 = "0.6" - -# Audio decoding (stub for now - will add opus decoding later) -# audiopus requires CMake to build, so we'll stub audio for now - -# Audio playback (cross-platform) -cpal = "0.15" - -# Gamepad/Controller support (cross-platform) -gilrs = "0.11" - -# Window & Graphics -winit = "0.30" -wgpu = { version = "28", features = ["wgpu-core", "metal"] } -pollster = "0.4" -bytemuck = { version = "1", features = ["derive"] } - -# GUI overlay -egui = "0.33" -egui-wgpu = "0.33" -egui-winit = "0.33" - -# Serialization -serde = { version = "1", features = ["derive"] } -serde_json = "1" - -# Image loading (for game art) -image = { version = "0.25", default-features = false, features = ["jpeg", "png", "webp"] } - -# Utilities -libc = "0.2" -anyhow = "1" -thiserror = "2" -log = "0.4" -env_logger = "0.11" - -# Profiling (optional) -tracing = "0.1" -tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } -tracing-tracy = { version = "0.11", optional = true, default-features = false, features = ["enable"] } -tracing-log = { version = "0.2", optional = true } -parking_lot = "0.12" - -# Logitech G29 steering wheel support (force feedback via HID) -g29 = "0.2" -bytes = "1" -base64 = "0.22" -sha2 = "0.10" -uuid = { version = "1", features = ["v4"] } -chrono = "0.4" -rand = "0.8" -urlencoding = "2" -http = "1" -dirs = "5" -regex = "1" -lazy_static = "1.4" -hex = "0.4" -open = "5" -once_cell = "1.19" - -# Clipboard support (cross-platform) -arboard = "3" - -# Platform-specific dependencies -[target.'cfg(windows)'.dependencies] -windows-numerics = "0.3" -windows = { version = "0.62", features = [ - "Win32_UI_WindowsAndMessaging", - "Win32_UI_Input", - "Win32_UI_Input_KeyboardAndMouse", - "Win32_Graphics_Gdi", - "Win32_Foundation", - "Win32_System_LibraryLoader", - # D3D11/DXGI for zero-copy video - "Win32_Graphics_Direct3D", - "Win32_Graphics_Direct3D11", - "Win32_Graphics_Dxgi", - "Win32_Graphics_Dxgi_Common", - "Win32_Security", - # Windows.Gaming.Input for racing wheel and force feedback support - "Gaming_Input", - "Gaming_Input_ForceFeedback", - "Foundation", - "Foundation_Collections", - "Foundation_Numerics", -] } - -[target.'cfg(target_os = "macos")'.dependencies] -core-foundation = "0.10" -core-graphics = "0.24" -cocoa = "0.26" -objc = "0.2" -# Zero-copy video: VideoToolbox -> IOSurface -> Metal texture -> wgpu -metal = "0.33" -wgpu-hal = { version = "28", features = ["metal"] } -foreign-types = "0.5" -block = "0.1" -# FFmpeg for video/audio decoding on macOS (VideoToolbox hardware acceleration) -ffmpeg-next = "8" - -[target.'cfg(windows)'.dependencies.wgpu-hal] -version = "28" -features = ["dx12"] - -# GStreamer for H.264 hardware decoding via D3D11 (Windows x64 only) -# Bundle GStreamer runtime with the app - see scripts/bundle-gstreamer.ps1 -# Note: GStreamer ARM64 Windows binaries are not available, so ARM64 uses native DXVA only -[target.'cfg(all(windows, target_arch = "x86_64"))'.dependencies.gstreamer] -version = "0.23" - -[target.'cfg(all(windows, target_arch = "x86_64"))'.dependencies.gstreamer-video] -version = "0.23" - -[target.'cfg(all(windows, target_arch = "x86_64"))'.dependencies.gstreamer-app] -version = "0.23" - -[target.'cfg(target_os = "linux")'.dependencies] -evdev = "0.12" -libloading = "0.8" -# Note: libc is already a top-level dependency -# Vulkan for Vulkan Video decoding (GFN-style cross-GPU hardware decode) -ash = "0.38" -ash-window = "0.13" -# GStreamer for V4L2 hardware decoding (Raspberry Pi and embedded) -gstreamer = "0.23" -gstreamer-video = "0.23" -gstreamer-app = "0.23" - -# X11 is optional - only needed for XInput2 fallback when evdev isn't available -# To enable: cargo build --features x11-input -# Required packages: libxi-dev libx11-dev -[target.'cfg(target_os = "linux")'.dependencies.x11] -version = "2.21" -features = ["xlib", "xinput"] -optional = true - -[features] -default = [] -x11-input = ["x11"] -# Enable Tracy profiler integration for performance analysis -# Build with: cargo build --release --features tracy -# Then run Tracy Profiler (https://github.com/wolfpld/tracy) and connect -tracy = ["tracing-tracy", "tracing-log"] -# Legacy macOS support for Intel Macs (2015 and older) -# Disables zero-copy Metal rendering and uses CPU-copy fallback -# Build with: cargo build --release --features legacy-macos -legacy-macos = [] - -[target.'cfg(target_os = "linux")'.dependencies.wgpu-hal] -version = "28" -features = ["vulkan"] - -[profile.release] -opt-level = 3 -lto = true -codegen-units = 1 -strip = true - -[profile.dev] -opt-level = 1 - -# Force all dependencies to use wgpu 28 (needed for External Texture support) -# Using forked egui with wgpu 28 support -[patch.crates-io] -wgpu = { git = "https://github.com/gfx-rs/wgpu", tag = "v28.0.0" } -wgpu-core = { git = "https://github.com/gfx-rs/wgpu", tag = "v28.0.0" } -wgpu-types = { git = "https://github.com/gfx-rs/wgpu", tag = "v28.0.0" } -wgpu-hal = { git = "https://github.com/gfx-rs/wgpu", tag = "v28.0.0" } -naga = { git = "https://github.com/gfx-rs/wgpu", tag = "v28.0.0" } -egui-wgpu = { git = "https://github.com/zortos293/egui", branch = "wgpu-28" } -egui-winit = { git = "https://github.com/zortos293/egui", branch = "wgpu-28" } -egui = { git = "https://github.com/zortos293/egui", branch = "wgpu-28" } diff --git a/opennow-streamer/GSTREAMER_SETUP.md b/opennow-streamer/GSTREAMER_SETUP.md deleted file mode 100644 index 9cbe3c7..0000000 --- a/opennow-streamer/GSTREAMER_SETUP.md +++ /dev/null @@ -1,96 +0,0 @@ -# GStreamer Setup for OpenNow Streamer - -OpenNow uses GStreamer for H.264 video decoding on Windows with D3D11 hardware acceleration. - -## Installation - -### Step 1: Download GStreamer Runtime - -1. Go to https://gstreamer.freedesktop.org/download/ -2. Under **Windows**, download the **MSVC 64-bit (VS 2019)** runtime installer - - File: `gstreamer-1.0-msvc-x86_64-X.XX.X.msi` - - Choose the **runtime** package (not development) - -### Step 2: Install GStreamer - -1. Run the downloaded MSI installer -2. Choose **Complete** installation (or Custom and select all components) -3. Install to default location: `C:\gstreamer\1.0\msvc_x86_64` - -### Step 3: Set Environment Variable (for building) - -The installer should set this automatically, but if not: - -```powershell -# PowerShell (Admin) -[Environment]::SetEnvironmentVariable("GSTREAMER_1_0_ROOT_MSVC_X86_64", "C:\gstreamer\1.0\msvc_x86_64", "Machine") -[Environment]::SetEnvironmentVariable("PATH", "$env:PATH;C:\gstreamer\1.0\msvc_x86_64\bin", "Machine") -``` - -### Step 4: Restart your terminal/IDE - -Close and reopen your terminal or IDE to pick up the new environment variables. - -## Building - -After installing GStreamer, build normally: - -```bash -cargo build --release -``` - -## Bundling for Distribution - -To bundle GStreamer with your app for distribution: - -```powershell -# From the opennow-streamer directory -.\scripts\bundle-gstreamer.ps1 -OutputDir "target\release" - -# For minimal bundle (only H.264 decoding): -.\scripts\bundle-gstreamer.ps1 -OutputDir "target\release" -Minimal -``` - -This creates a `gstreamer/` folder next to your executable with all required DLLs. - -## How It Works - -- **H.264 streams**: Decoded using GStreamer with D3D11 hardware acceleration (`d3d11h264dec`) -- **HEVC streams**: Decoded using native DXVA decoder (better performance for HEVC) - -The app automatically detects bundled GStreamer in the `gstreamer/` subfolder, or falls back to system-installed GStreamer. - -## Troubleshooting - -### Build Error: "pkg-config not found" - -Install pkg-config: -```powershell -# Using Chocolatey -choco install pkgconfiglite - -# Or using Scoop -scoop install pkg-config -``` - -### Build Error: "gstreamer-1.0 not found" - -Make sure the environment variable is set correctly: -```powershell -echo $env:GSTREAMER_1_0_ROOT_MSVC_X86_64 -# Should output: C:\gstreamer\1.0\msvc_x86_64 -``` - -### Runtime Error: "DLL not found" - -1. If using system GStreamer: Add `C:\gstreamer\1.0\msvc_x86_64\bin` to PATH -2. If using bundled GStreamer: Run the bundle script and ensure `gstreamer/` folder is next to the executable - -## Required GStreamer Plugins - -For H.264 decoding, these plugins are required: -- `gstd3d11.dll` - D3D11 hardware decoder (d3d11h264dec) -- `gstvideoparsersbad.dll` - H.264 parser (h264parse) -- `gstvideoconvertscale.dll` - Video format conversion -- `gstapp.dll` - appsrc/appsink elements -- `gstlibav.dll` - Software decoder fallback (avdec_h264) diff --git a/opennow-streamer/macos/Info.plist b/opennow-streamer/macos/Info.plist deleted file mode 100644 index 950ab26..0000000 --- a/opennow-streamer/macos/Info.plist +++ /dev/null @@ -1,44 +0,0 @@ - - - - - CFBundleName - OpenNOW - CFBundleDisplayName - OpenNOW Streamer - CFBundleIdentifier - com.opennow.streamer - CFBundleVersion - 0.2.0 - CFBundleShortVersionString - 0.2.0 - CFBundleExecutable - opennow-streamer - CFBundlePackageType - APPL - LSMinimumSystemVersion - 12.0 - NSHighResolutionCapable - - - - GCSupportsGameMode - - - - NSSupportsAutomaticTermination - - NSSupportsSuddenTermination - - NSAppSleepDisabled - - - - GPUSelectionPolicy - highPerformance - - - LSApplicationCategoryType - public.app-category.games - - diff --git a/opennow-streamer/macos/bundle.sh b/opennow-streamer/macos/bundle.sh deleted file mode 100755 index 7dc405c..0000000 --- a/opennow-streamer/macos/bundle.sh +++ /dev/null @@ -1,249 +0,0 @@ -#!/bin/bash -# Bundle opennow-streamer as a macOS .app for Game Mode support - -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_DIR="$(dirname "$SCRIPT_DIR")" -APP_NAME="OpenNOW.app" -APP_DIR="$PROJECT_DIR/target/release/$APP_NAME" -CONTENTS_DIR="$APP_DIR/Contents" -MACOS_DIR="$CONTENTS_DIR/MacOS" -FRAMEWORKS_DIR="$CONTENTS_DIR/Frameworks" -BINARY="$PROJECT_DIR/target/release/opennow-streamer" - -# Build release -echo "Building release..." -cd "$PROJECT_DIR" - -# Extract version from Cargo.toml if not provided -if [ -z "$VERSION" ]; then - VERSION=$(grep "^version" Cargo.toml | head -1 | awk -F '"' '{print $2}') - echo "Detected version from Cargo.toml: $VERSION" -fi -if [ -z "$VERSION" ]; then - VERSION="0.1.0" - echo "Could not detect version, defaulting to $VERSION" -fi - -cargo build --release - -# Create app bundle structure -echo "Creating app bundle..." -rm -rf "$APP_DIR" -mkdir -p "$APP_DIR/Contents/MacOS" -mkdir -p "$APP_DIR/Contents/Resources" - -# Copy and rename binary -echo "Copying binary..." -cp "$BINARY" "$MACOS_DIR/$APP_NAME" -chmod +x "$MACOS_DIR/$APP_NAME" - -# Create Info.plist -echo "Creating Info.plist..." -cat > "$CONTENTS_DIR/Info.plist" < - - - - CFBundleExecutable - $APP_NAME - CFBundleIdentifier - com.opennow.streamer - CFBundleName - $APP_NAME - CFBundleDisplayName - $APP_NAME - CFBundleVersion - ${VERSION} - CFBundleShortVersionString - ${VERSION} - CFBundlePackageType - APPL - LSMinimumSystemVersion - 12.0 - NSHighResolutionCapable - - - -EOF - -# Function to copy a library and fix its install name -copy_lib() { - local lib="$1" - local libname=$(basename "$lib") - - if [ ! -f "$FRAMEWORKS_DIR/$libname" ] && [ -f "$lib" ]; then - echo "Copying: $libname" - cp "$lib" "$FRAMEWORKS_DIR/" - chmod 755 "$FRAMEWORKS_DIR/$libname" - - # Fix the library's own install name to be relative to the framework folder - install_name_tool -id "@executable_path/../Frameworks/$libname" "$FRAMEWORKS_DIR/$libname" 2>/dev/null || true - return 0 - fi - return 0 -} - -# Function to fix references in a binary/library -fix_refs() { - local target="$1" - # Only fix references to Homebrew/system libs that we are bundling - for dep in $(otool -L "$target" 2>/dev/null | grep -E '/opt/homebrew|/usr/local' | awk '{print $1}'); do - local depname=$(basename "$dep") - install_name_tool -change "$dep" "@executable_path/../Frameworks/$depname" "$target" 2>/dev/null || true - done -} - -echo "" -echo "=== Phase 1: Explicitly copy FFmpeg libraries ===" -# FFmpeg libraries might not show up in otool if dlopened or linked with @rpath -# So we explicitly find and copy them - -FFMPEG_PREFIX=$(brew --prefix ffmpeg) -echo "FFmpeg prefix: $FFMPEG_PREFIX" - -# List of core FFmpeg libraries to bundle -FFMPEG_LIBS=( - "libavcodec" - "libavdevice" - "libavfilter" - "libavformat" - "libavutil" - "libswresample" - "libswscale" -) - -for lib_base in "${FFMPEG_LIBS[@]}"; do - echo "Looking for $lib_base in $FFMPEG_PREFIX/lib..." - FOUND_LIBS=$(find "$FFMPEG_PREFIX/lib" -name "${lib_base}.*.dylib" -type f ) - - for lib in $FOUND_LIBS; do - echo "Found FFmpeg lib: $lib" - copy_lib "$lib" - done -done - - -echo "" -echo "=== Phase 2: Copy direct dependencies detected by otool ===" -DIRECT_DEPS=$(otool -L "$MACOS_DIR/$APP_NAME" 2>/dev/null | grep -E '/opt/homebrew|/usr/local' | awk '{print $1}' || true) -if [ -z "$DIRECT_DEPS" ]; then - echo "No Homebrew dependencies found in binary (may be statically linked or using system libs)" -else - for lib in $DIRECT_DEPS; do - copy_lib "$lib" - done -fi - -echo "" -echo "=== Phase 3: Copy transitive dependencies (3 passes) ===" -BREW_PREFIX=$(brew --prefix) -echo "Resolving @rpath against: $BREW_PREFIX/lib" - -for pass in 1 2 3; do - echo "Pass $pass..." - if ls "$FRAMEWORKS_DIR/"*.dylib 1>/dev/null 2>&1; then - for bundled_lib in "$FRAMEWORKS_DIR/"*.dylib; do - [ -f "$bundled_lib" ] || continue - # Grep for /opt/homebrew, /usr/local, AND @rpath - for dep in $(otool -L "$bundled_lib" 2>/dev/null | grep -E '/opt/homebrew|/usr/local|@rpath' | awk '{print $1}'); do - - # Handle @rpath references - if [[ "$dep" == "@rpath/"* ]]; then - # Extract filename - filename="${dep#@rpath/}" - # Construct absolute path assuming it's in Homebrew lib - resolved_path="$BREW_PREFIX/lib/$filename" - - if [ -f "$resolved_path" ]; then - # echo "Resolved $dep to $resolved_path" - copy_lib "$resolved_path" - else - # Try strict FFmpeg prefix if different? usually brew prefix is enough - : - fi - else - # Absolute path - copy_lib "$dep" - fi - done - done - else - echo " No dylibs to process" - break - fi -done - -echo "" -echo "=== Phase 4: Fix all library references ===" -# Fix the main binary -fix_refs "$MACOS_DIR/$APP_NAME" - -# Fix all bundled libraries -if ls "$FRAMEWORKS_DIR/"*.dylib 1>/dev/null 2>&1; then - for bundled_lib in "$FRAMEWORKS_DIR/"*.dylib; do - [ -f "$bundled_lib" ] || continue - fix_refs "$bundled_lib" - done -fi - -# Special pass: Fix FFmpeg internal references -echo "Fixing internal references in bundled libs..." -if ls "$FRAMEWORKS_DIR/"*.dylib 1>/dev/null 2>&1; then - for bundled_lib in "$FRAMEWORKS_DIR/"*.dylib; do - for dep in $(otool -L "$bundled_lib" 2>/dev/null | grep -E '/opt/homebrew|/usr/local' | awk '{print $1}'); do - local depname=$(basename "$dep") - if [ -f "$FRAMEWORKS_DIR/$depname" ]; then - echo " Fixing ref in $(basename "$bundled_lib") to $depname" - install_name_tool -change "$dep" "@loader_path/$depname" "$bundled_lib" 2>/dev/null || true - fi - done - done -fi - -# Special pass: Rewrite @rpath references (e.g. libwebp -> libsharpyuv) -echo "Rewriting @rpath references in bundled libs..." -if ls "$FRAMEWORKS_DIR/"*.dylib 1>/dev/null 2>&1; then - for bundled_lib in "$FRAMEWORKS_DIR/"*.dylib; do - # Grep for @rpath references - for dep in $(otool -L "$bundled_lib" 2>/dev/null | grep "@rpath/" | awk '{print $1}'); do - filename="${dep#@rpath/}" - if [ -f "$FRAMEWORKS_DIR/$filename" ]; then - echo " Rewriting @rpath ref in $(basename "$bundled_lib"): $dep -> @loader_path/$filename" - install_name_tool -change "$dep" "@loader_path/$filename" "$bundled_lib" 2>/dev/null || true - fi - done - done -fi - -echo "" -echo "=== Final verification ===" -echo "Binary dependencies:" -otool -L "$MACOS_DIR/$APP_NAME" - -echo "" -echo "Bundled Frameworks:" -ls -1 "$FRAMEWORKS_DIR" - -echo "" -echo "=== Phase 5: Code Signing ===" -echo "Signing app bundle..." -# Sign with ad-hoc signature (-) -# Use --force to replace any existing signature -# Use --deep to sign all nested frameworks and plugins -if codesign --force --deep --sign - "$APP_DIR"; then - echo "Code signing successful" -else - echo "Code signing failed" - exit 1 -fi - -echo "" -echo "App bundle created: $APP_DIR" -echo "" -echo "To run with Game Mode support:" -echo " open '$APP_DIR'" -echo "" -echo "Or run directly:" -echo " '$APP_DIR/Contents/MacOS/opennow-streamer'" diff --git a/opennow-streamer/reverse/audio.md b/opennow-streamer/reverse/audio.md deleted file mode 100644 index 64d4123..0000000 --- a/opennow-streamer/reverse/audio.md +++ /dev/null @@ -1,242 +0,0 @@ -# GeForce NOW Audio Handling - Reverse Engineering Documentation - -## 1. Audio Codec Details - -### Opus Configuration -- **Codec**: Opus (RFC 6716) -- **Sample Rate**: 48000 Hz -- **Channels**: 2 (stereo) or up to 8 (multiopus) -- **Payload Types**: - - 101: opus/48000/2 (standard stereo) - - 100: multiopus/48000/N (N = 2, 4, 6, or 8 channels) - -### SDP Configuration -``` -a=rtpmap:101 opus/48000/2 -a=fmtp:101 minptime=10;useinbandfec=1 - -a=rtpmap:100 multiopus/48000/2 -a=fmtp:100 minptime=10;useinbandfec=1;coupled_streams=2 -``` - -### Multiopus Channel Mapping -``` -4-channel: FL, FR, BL, BR -6-channel: FL, FR, C, LFE, BL, BR (5.1 surround) -8-channel: FL, FR, C, LFE, BL, BR, SL, SR (7.1 surround) -``` - ---- - -## 2. RTP Packet Structure - -### RTP Header for Audio -``` -0 1 2 3 -0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -|V=2|P|X| CC |M| PT | sequence number | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| timestamp | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| synchronization source (SSRC) identifier | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -``` - -**Fields:** -- V: 2 (RTP version) -- P: 0 (no padding) -- X: 0 (no extension typically) -- CC: 0 (no CSRC) -- M: 1 = last packet of frame -- PT: 100 (multiopus) or 101 (opus) -- Timestamp: 48 kHz audio clock - -### Opus RTP Payload (RFC 7587) -``` -[TOC Byte] [Opus Frame Data...] - -TOC Byte: F(1) | C(1) | VBR(2) | MODE(4) -``` - ---- - -## 3. Frame Sizes - -### Opus at 48 kHz -``` -10 ms = 480 samples -20 ms = 960 samples (default) -40 ms = 1920 samples -60 ms = 2880 samples -``` - -### Typical Configuration -- Default frame size: 20 ms (960 samples) -- Minimum packet interval: 10 ms -- Bitrate: 64-96 kbps for stereo - ---- - -## 4. Audio Buffer Management - -### OpenNow AudioBuffer Structure -```rust -struct AudioBuffer { - samples: Vec, - read_pos: usize, - write_pos: usize, - capacity: usize, // ~200ms at 48kHz - total_written: u64, - total_read: u64, -} -``` - -### Buffer Size Calculation -```rust -// At 48000 Hz, 2 channels: -// 48000 * 2 / 5 = 19200 samples = ~200ms buffering -let buffer_size = (sample_rate as usize) * (channels as usize) / 5; -``` - -### Jitter Handling -- Circular buffer with read/write pointers -- Underrun: Output silence (zeros) -- Overrun: Drop oldest samples -- RTP sequence numbers for packet ordering - ---- - -## 5. Sample Format & Conversion - -### PCM Sample Format -``` -Output: 16-bit signed PCM (i16) -Range: -32768 to +32767 -Channels: Interleaved stereo [L0, R0, L1, R1, ...] -``` - -### Format Conversion - -**From F32 Planar:** -```rust -let sample = (plane[i] * 32767.0).clamp(-32768.0, 32767.0) as i16; -``` - -**From I16 Planar (Interleave):** -```rust -for i in 0..nb_samples { - for ch in 0..channels { - let plane = frame.plane::(ch); - output.push(plane[i]); - } -} -``` - ---- - -## 6. Audio/Video Synchronization - -### RTP Timestamp Alignment -``` -Video: 90 kHz clock -Audio: 48 kHz clock - -Frame at 60 FPS: - Video: 1500 RTP ticks (90000/60) - Audio: 800 RTP ticks for 16.67ms (48000 * 0.01667) -``` - -### OpenNow Sync Method -- RTP timestamps provide absolute timing -- Both streams timestamped from server clock -- Audio buffer maintains timing through sample count - ---- - -## 7. Decode Process Flow - -``` -RTP Packet Received (peer.rs) - ↓ -Extract RTP Payload - ↓ -Send to AudioDecoder (mpsc channel) - ↓ -FFmpeg Opus Decode - ↓ -Convert to i16 samples - ↓ -Write to AudioBuffer - ↓ -AudioPlayer reads from buffer - ↓ -Output to audio device (cpal) -``` - -### OpenNow Implementation -```rust -// webrtc/mod.rs line 265-276 -let mut audio_decoder = AudioDecoder::new(48000, 2)?; -let (audio_tx, mut audio_rx) = mpsc::channel::>(32); - -std::thread::spawn(move || { - if let Ok(audio_player) = AudioPlayer::new(48000, 2) { - while let Some(samples) = audio_rx.blocking_recv() { - audio_player.push_samples(&samples); - } - } -}); -``` - ---- - -## 8. Device Configuration - -### Sample Rate Selection Priority -```rust -1. Use requested 48 kHz if supported -2. Fallback to 44.1 kHz if 48 kHz not supported -3. Use device maximum as last resort -``` - -### Output Format Selection -```rust -// Scoring system: -// F32 format: 100 points (preferred) -// I16 format: 50 points -// Matching channels: +50 points -// Matching sample rate: +100 points -``` - ---- - -## 9. Comparison - -| Feature | Web Client | Official Client | OpenNow | -|---------|-----------|-----------------|---------| -| Codec | Opus/Multiopus | Opus | Opus (FFmpeg) | -| Sample Rate | 48 kHz | 48 kHz | 48 kHz | -| Channels | 2-8 (dynamic) | 2-8 | 2 (hardcoded) | -| Decoding | Browser | Native C++ | FFmpeg | -| Output | WebAudio API | WASAPI/etc | cpal | -| Jitter Buffer | Built-in | Yes | RTP seq-based | -| FEC Support | useinbandfec=1 | Yes | No | -| Surround | Yes (8ch) | Yes | Not yet | -| Latency | 20-50ms | 15-30ms | 20-40ms | -| Buffer Size | ~500ms | ~200ms | ~200ms | - ---- - -## 10. OpenNow Limitations - -**Current:** -- Hardcoded stereo (2 channels) -- No explicit jitter buffer -- No FEC recovery implementation -- Basic circular buffer - -**Future Extensions:** -1. Add surround sound support (modify `AudioDecoder::new()`) -2. Implement jitter buffer with RTP reordering -3. Add FEC parsing for packet loss recovery diff --git a/opennow-streamer/reverse/controller.md b/opennow-streamer/reverse/controller.md deleted file mode 100644 index 052e21c..0000000 --- a/opennow-streamer/reverse/controller.md +++ /dev/null @@ -1,268 +0,0 @@ -# GeForce NOW Controller/Gamepad Input - Reverse Engineering Documentation - -## 1. Controller Detection - -### Web Client (Gamepad API) -```javascript -let gamepads = navigator.getGamepads(); - -// Event-driven detection -window.addEventListener("gamepadconnected", handler); -window.addEventListener("gamepaddisconnected", handler); -``` - -### Detection Priority -1. **PlayStation 4/5**: Vendor ID `054c`, 18+ buttons -2. **Xbox Controllers**: Device ID contains "Xbox" or "xinput" -3. **Nvidia Shield**: Shield ID check -4. **Standard Gamepad**: HID-compliant fallback -5. **Virtual Gamepad**: Software emulation - -### Polling Rate -- Default: 4ms (250Hz) -- Configurable via URL: `?gamepadpoll=X` - ---- - -## 2. Gamepad State Structure - -### Standard Format (XInput-style) -``` -[Offset] [Size] [Field] [Type] -0x00 4B Type u32 LE (event type) -0x04 2B Index u16 BE (gamepad index 0-3) -0x06 2B Bitmap u16 BE (button state bitmask) -0x08 2B Reserved u16 BE -0x0A 2B Buttons u16 BE (button bitmask) -0x0C 2B Trigger u16 BE (combined analog triggers) -0x0E 4×2B Axes[0-3] 4× i16 BE (left X/Y, right X/Y) -0x16 8B CaptureTs u64 BE (timestamp) -``` - ---- - -## 3. Button Mapping - -### Standard Button IDs (Xbox Layout) -``` -Index Xbox Name PlayStation Physical Position -0 A ○ (Circle) Bottom/Right -1 B ✕ (Cross) Right -2 X □ (Square) Left -3 Y △ (Triangle) Top -4 LB L1 Left Shoulder -5 RB R1 Right Shoulder -6 LT L2 Left Trigger (analog) -7 RT R2 Right Trigger (analog) -8 Back Select/Share Left Center -9 Start Options Right Center -10 Left Stick L3 Left Stick Click -11 Right Stick R3 Right Stick Click -12 Guide PS Button Center -13-15 [Reserved] [Reserved] [Reserved] -``` - -### Button Bitmap Encoding -- Bit 0: Y / △ -- Bit 1: X / □ -- Bit 2: A / ○ -- Bit 3: B / ✕ -- Bit 4-7: Shoulder buttons -- Bit 8-11: Start/Select/Sticks -- Bit 12: Guide button - ---- - -## 4. Trigger Handling - -### Packed Format (u16) -``` -Low byte (0xFF): Left Trigger (L2/LT) 0-255 -High byte (0xFF): Right Trigger (R2/RT) 0-255 - -Example: 0xFF00 = LT fully pressed (255), RT released (0) -``` - -### Quantization -```javascript -let left_trigger = Math.round(255 * (axis_lt + 1) / 2); -let right_trigger = Math.round(255 * (axis_rt + 1) / 2); -let packed = (right_trigger << 8) | left_trigger; -``` - ---- - -## 5. Analog Stick Handling - -### Axis Mapping -- Axis[0]: Left Stick X (-1.0 to 1.0) -- Axis[1]: Left Stick Y (-1.0 to 1.0, inverted) -- Axis[2]: Right Stick X (-1.0 to 1.0) -- Axis[3]: Right Stick Y (-1.0 to 1.0, inverted) - -### Dead Zone -- Typical: 0.15 (15% of full range) -- Applied per-axis before quantization - -### Quantization to i16 -```javascript -if (Math.abs(axis_value) < 0.15) { - axis_value = 0; // Dead zone -} -let quantized = Math.round(axis_value * 32767); - -// Special value for unchanged axes -if (axis_value === last_axis_value) { - quantized = 2; // "Unchanged" marker -} -``` - ---- - -## 6. Vibration/Rumble - -### Dual-Rumble API -```javascript -if (gamepad.vibrationActuator?.type === "dual-rumble") { - gamepad.vibrationActuator.playEffect("dual-rumble", { - startDelay: 0, - duration: milliseconds, - strongMagnitude: 0.0-1.0, // Left motor - weakMagnitude: 0.0-1.0, // Right motor - }); -} -``` - -### Stop Rumble -```javascript -gamepad.vibrationActuator.playEffect("dual-rumble", { - duration: 0, - strongMagnitude: 0, - weakMagnitude: 0, -}); -``` - -### Support Matrix -- Xbox Controllers: Full dual-rumble -- DualSense: Full dual-rumble -- DualShock 4: Limited (single motor emulated) -- Generic: Varies by device - ---- - -## 7. Controller Type Identification - -### Vendor IDs -``` -Sony (DualShock/DualSense): 054c -Microsoft (Xbox): 045e -Nintendo (Switch): 057e -Nvidia (Shield): Custom -Generic HID: Various -``` - -### Type Classification -| Type | Detected By | Button Mapper | -|------|-------------|---------------| -| Xbox Series | "Xbox" in ID | KA() | -| Xbox Wired | "Xbox" in ID | YA() | -| DualShock 4 | VID=054c, 18+ btns | NA() | -| DualSense | Device ID check | _A() | -| Shield | Shield ID check | GA() | -| Standard | Generic HID | NA() | - ---- - -## 8. DualShock 4/DualSense Format - -### Extended Format -``` -[Offset] [Size] [Field] -...standard header... -0x0E 3B ds4Btns[3] Sony-specific buttons -0x11 2B triggers[2] L2/R2 analog (0-255) -0x13 4B axes[4] Analog (0-255, centered at 128) -``` - ---- - -## 9. Data Channel Configuration - -### WebRTC Data Channel -- Name: `input_channel_v1` (reliable) -- Ordered: Yes -- Reliable: Yes -- Used for: Gamepad state updates - ---- - -## 10. Handshake Protocol - -### Server → Client -``` -[0]: 0x0E (handshake marker) -[1]: Major version -[2]: Minor version -[3]: Flags -``` - -### Client → Server -Echo same bytes back to confirm ready state. - ---- - -## 11. Protocol Versions - -### Version 2 (Legacy) -- Direct event encoding -- No wrapper - -### Version 3+ (Modern) -- Events wrapped with 0x22 prefix -``` -[0]: 0x22 (wrapper marker) -[1...]: Event payload -``` - ---- - -## 12. Comparison - -| Feature | Web Client | Official Client | OpenNow | -|---------|-----------|-----------------|---------| -| Input API | Gamepad API | HID (Raw Input) | HID + Gamepad | -| Controllers | 19 types | Static list | Mouse/KB only | -| Polling Rate | 4ms (250Hz) | Hardware interrupt | 4ms | -| Triggers | Packed u16 or u8 | Analog u8 | N/A | -| Vibration | Full dual-rumble | Full support | Not implemented | -| Dead Zone | 0.15 per-axis | Hardware-level | Mouse only | -| Multi-controller | Up to 4 | Up to 4 | Not supported | - ---- - -## 13. Limitations - -### Web Client -- Max 4 controllers (Gamepad API limit) -- Vibration not supported on all devices - -### OpenNow -- Controller support not implemented -- Only mouse/keyboard input -- No rumble feedback - -### Official Client -- VID-based detection may miss non-standard controllers -- 18+ button requirement excludes older gamepads - ---- - -## 14. Implementation Notes - -1. **Always initialize session timing** before first input -2. **Flush mouse events before button events** -3. **Use correct timestamp format** (microseconds) -4. **Implement 4ms coalescing** for consistency -5. **Handle protocol version negotiation** -6. **Test with multiple controller types** -7. **Reserve unused fields as zeros** diff --git a/opennow-streamer/reverse/cursor.md b/opennow-streamer/reverse/cursor.md deleted file mode 100644 index 9804be5..0000000 --- a/opennow-streamer/reverse/cursor.md +++ /dev/null @@ -1,274 +0,0 @@ -# GeForce NOW Mouse/Cursor Handling - Reverse Engineering Documentation - -## 1. Data Channels - -### Input Channels - -**Primary Input Channel (Reliable)** -- Name: `input_channel_v1` -- Ordered: Yes -- Reliable: Yes -- Used for: Keyboard, mouse buttons, wheel, handshake - -**Mouse Channel (Partially Reliable)** -- Name: `input_channel_partially_reliable` -- Ordered: No -- Reliable: No (8ms max lifetime) -- Used for: Low-latency mouse movement - -**Cursor Channel** -- Name: `cursor_channel` -- Ordered: Yes -- Reliable: Yes -- Used for: Cursor image updates, hotspot coordinates - ---- - -## 2. Mouse Movement (Type 7 - INPUT_MOUSE_REL) - -### Binary Format (22 bytes) -``` -[0-3] Type: 0x07 (4 bytes, Little Endian) -[4-5] Delta X: i16 (2 bytes, Big Endian, signed) -[6-7] Delta Y: i16 (2 bytes, Big Endian, signed) -[8-9] Reserved: u16 (0x00 0x00) -[10-13] Reserved: u32 (0x00 0x00 0x00 0x00) -[14-21] Timestamp: u64 (8 bytes, Big Endian, microseconds) -``` - -### Coalescing -- Interval: 4ms (250Hz effective rate) -- Accumulates dx/dy deltas -- Flushed on interval expiry OR before button events - ---- - -## 3. Mouse Button Down (Type 8) - -### Binary Format (18 bytes) -``` -[0-3] Type: 0x08 (4 bytes, Little Endian) -[4] Button: u8 (0=Left, 1=Right, 2=Middle, 3=Back, 4=Forward) -[5] Padding: u8 (0) -[6-9] Reserved: u32 (0) -[10-17] Timestamp: u64 (Big Endian, microseconds) -``` - ---- - -## 4. Mouse Button Up (Type 9) - -### Binary Format (18 bytes) -``` -[0-3] Type: 0x09 (4 bytes, Little Endian) -[4] Button: u8 (0=Left, 1=Right, 2=Middle, 3=Back, 4=Forward) -[5] Padding: u8 (0) -[6-9] Reserved: u32 (0) -[10-17] Timestamp: u64 (Big Endian, microseconds) -``` - ---- - -## 5. Mouse Wheel (Type 10) - -### Binary Format (22 bytes) -``` -[0-3] Type: 0x0A (4 bytes, Little Endian) -[4-5] Horizontal Delta: i16 (Big Endian, usually 0) -[6-7] Vertical Delta: i16 (Big Endian, positive=scroll up) -[8-9] Reserved: u16 (0) -[10-13] Reserved: u32 (0) -[14-21] Timestamp: u64 (Big Endian, microseconds) -``` - -### Wheel Delta Values -- Standard: WHEEL_DELTA = 120 per notch -- Positive = scroll up -- Negative = scroll down - ---- - -## 6. Cursor Capture Modes - -### Windows Implementation -```rust -// Preferred: Confined to window -CursorGrabMode::Confined - -// Fallback: Locked (hidden) -CursorGrabMode::Locked - -// Released: Normal cursor -CursorGrabMode::None -``` - -### macOS Implementation -- Uses Core Graphics Event Taps -- Captures at HID level: `CGEventTapLocation::HIDEventTap` - ---- - -## 7. Raw Input (Windows) - -### HID Registration -```rust -let device = RAWINPUTDEVICE { - usage_page: 0x01, // HID_USAGE_PAGE_GENERIC - usage: 0x02, // HID_USAGE_GENERIC_MOUSE - flags: 0, // Only when window focused - hwnd_target: hwnd, -}; -``` - -### Benefits -- No OS acceleration applied -- Hardware-level relative deltas -- Lower latency than standard events - ---- - -## 8. Local Cursor Rendering - -### Position Tracking -```rust -struct LocalCursor { - x: AtomicI32, - y: AtomicI32, - visible: AtomicBool, - stream_width: AtomicU32, - stream_height: AtomicU32, -} -``` - -### Update Logic -- Updated on every raw input event -- Bounded to stream dimensions -- Provides instant visual feedback - ---- - -## 9. Mouse Coalescing - -### Implementation -```rust -pub struct MouseCoalescer { - accumulated_dx: AtomicI32, - accumulated_dy: AtomicI32, - last_send_us: AtomicU64, - coalesce_interval_us: u64, // 4000 (4ms) -} - -pub fn accumulate(&self, dx: i32, dy: i32) -> Option<(i16, i16, u64)> { - self.accumulated_dx.fetch_add(dx, Ordering::Relaxed); - self.accumulated_dy.fetch_add(dy, Ordering::Relaxed); - - let now_us = session_elapsed_us(); - if now_us - last_send >= coalesce_interval_us { - // Flush accumulated movement - Some((dx as i16, dy as i16, timestamp_us)) - } else { - None - } -} -``` - -### Event Ordering -Movement is **flushed BEFORE button events**: -``` -MouseMove(100,200) → MouseButtonDown → MouseMove(50,50) -``` - ---- - -## 10. Cursor Image Updates - -### Cursor Channel Messages -- Image data (PNG format) -- Hotspot coordinates (X, Y) -- Visibility state - -### Cursor Type Values -``` -CursorType = { - None: 0, - Mouse: 1, - Keyboard: 2, - Gamepad: 4, - Touch: 8, - All: 15 -} -``` - ---- - -## 11. Timestamp Generation - -```rust -pub fn get_timestamp_us() -> u64 { - if let Some(ref t) = *SESSION_TIMING.read() { - let elapsed_us = t.start.elapsed().as_micros() as u64; - t.unix_us.wrapping_add(elapsed_us) - } else { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_micros() as u64) - .unwrap_or(0) - } -} -``` - ---- - -## 12. Comparison - -| Feature | Web Client | Official Client | OpenNow | -|---------|-----------|-----------------|---------| -| Input API | Pointer Lock API | Raw Input | Raw Input + winit | -| Coalescing | 4-16ms | Hardware optimized | 4ms | -| Cursor Capture | Pointer Lock | Native capture | CursorGrabMode | -| Local Cursor | Canvas rendering | Native rendering | GPU rendering | -| Latency | 40-80ms | 10-30ms | 20-40ms | - ---- - -## 13. Error Codes - -- `StreamCursorChannelError` (3237093897): Cursor channel failure -- `StreamerCursorChannelNotOpen` (3237093920): Channel not established -- `ServerDisconnectedInvalidMouseState` (3237094150): Invalid mouse state - ---- - -## 14. Protocol v3+ Wrapper - -For protocol version 3+: -``` -[0] 0x22 (wrapper marker) -[1...N] Raw mouse event bytes -``` - ---- - -## 15. Byte-Level Example - -### Mouse Movement (dx=100, dy=-50) -``` -Hex: 07 00 00 00 00 64 FF CE 00 00 00 00 00 00 12 34 56 78 9A BC DE F0 - -[00-03] 07 00 00 00 = Type 7 (LE) = MOUSE_REL -[04-05] 00 64 = dx=100 (BE i16) -[06-07] FF CE = dy=-50 (BE i16, two's complement) -[08-13] 00 00 00 00 00 00 = Reserved -[14-21] Timestamp (BE u64) -``` - -### Mouse Button Down (Left Click) -``` -Hex: 08 00 00 00 00 00 00 00 00 00 12 34 56 78 9A BC DE F0 - -[00-03] 08 00 00 00 = Type 8 (LE) = MOUSE_BUTTON_DOWN -[04] 00 = Button 0 (Left) -[05] 00 = Padding -[06-09] 00 00 00 00 = Reserved -[10-17] Timestamp (BE u64) -``` diff --git a/opennow-streamer/reverse/datachannel.md b/opennow-streamer/reverse/datachannel.md deleted file mode 100644 index d9d6ddc..0000000 --- a/opennow-streamer/reverse/datachannel.md +++ /dev/null @@ -1,306 +0,0 @@ -# GeForce NOW Data Channel Protocol - Reverse Engineering Documentation - -## 1. Data Channel Names & Configuration - -### Input Channels - -**input_channel_v1 (Reliable)** -``` -Name: input_channel_v1 -Ordered: true -Reliable: true -MaxRetransmits: 0 -binaryType: arraybuffer -Used for: Keyboard, mouse buttons, wheel, handshake -``` - -**input_channel_partially_reliable (Low Latency)** -``` -Name: input_channel_partially_reliable -Ordered: false -Reliable: false -MaxPacketLifeTime: 8ms -Used for: Mouse movement (can drop packets) -``` - -### Other Channels - -**cursor_channel** -``` -Ordered: true -Reliable: true -Used for: Cursor image updates, hotspot coordinates -``` - -**control_channel** -``` -Ordered: true -Reliable: true -Used for: Server-to-client JSON messages (network test results) -``` - -**stats_channel** -``` -Used for: Telemetry data (optional) -``` - ---- - -## 2. Input Message Types - -| Type | Value | Size | Description | -|------|-------|------|-------------| -| HEARTBEAT | 0x02 | 4B | Keep-alive | -| KEY_DOWN | 0x03 | 18B | Keyboard pressed | -| KEY_UP | 0x04 | 18B | Keyboard released | -| MOUSE_ABS | 0x05 | - | Absolute mouse position | -| MOUSE_REL | 0x07 | 22B | Relative mouse movement | -| MOUSE_BUTTON_DOWN | 0x08 | 18B | Mouse button pressed | -| MOUSE_BUTTON_UP | 0x09 | 18B | Mouse button released | -| MOUSE_WHEEL | 0x0A | 22B | Mouse wheel scroll | - ---- - -## 3. Binary Message Structures - -### Heartbeat (4 bytes) -``` -[0-3] Type: 0x02 (Little Endian u32) -``` - -### Key Down/Up (18 bytes) -``` -[0-3] Type: 0x03 or 0x04 (LE u32) -[4-5] Keycode: u16 (BE) - Windows VK code -[6-7] Modifiers: u16 (BE) - SHIFT|CTRL|ALT|META|CAPS|NUMLOCK -[8-9] Scancode: u16 (BE) - USB HID scancode -[10-17] Timestamp: u64 (BE) - Microseconds -``` - -### Mouse Relative (22 bytes) -``` -[0-3] Type: 0x07 (LE u32) -[4-5] dx: i16 (BE) - Relative X movement -[6-7] dy: i16 (BE) - Relative Y movement -[8-9] Reserved: u16 (0) -[10-13] Reserved: u32 (0) -[14-21] Timestamp: u64 (BE) -``` - -### Mouse Button Down/Up (18 bytes) -``` -[0-3] Type: 0x08 or 0x09 (LE u32) -[4] Button: u8 (0=LEFT, 1=RIGHT, 2=MIDDLE, 3=BACK, 4=FORWARD) -[5] Padding: u8 (0) -[6-9] Reserved: u32 (0) -[10-17] Timestamp: u64 (BE) -``` - -### Mouse Wheel (22 bytes) -``` -[0-3] Type: 0x0A (LE u32) -[4-5] Horizontal: i16 (BE) - Usually 0 -[6-7] Vertical: i16 (BE) - Positive=scroll up -[8-9] Reserved: u16 (0) -[10-13] Reserved: u32 (0) -[14-21] Timestamp: u64 (BE) -``` - ---- - -## 4. Handshake Protocol - -### Server Initiates -Sends handshake on input_channel_v1: -``` -New Format: [0x0E, major_version, minor_version, flags] -Old Format: Direct version bytes -``` - -### Client Response -Echo the same bytes back to signal ready state. - -### Version Detection -```rust -if data.len() >= 4 { - // New format: version at bytes 2-4 - let version = u16::from_le_bytes([data[2], data[3]]); -} else { - // Old format: version is first word - let version = u16::from_le_bytes([data[0], data[1]]); -} -``` - ---- - -## 5. Protocol Versions - -### Version 2 (Legacy) -- Direct event encoding -- No wrapper - -### Version 3+ (Modern) -Each event wrapped with marker byte: -``` -[0] 0x22 (wrapper marker = 34 decimal) -[1...N] Original message bytes -``` - ---- - -## 6. Modifier Flags - -``` -SHIFT: 0x01 -CTRL: 0x02 -ALT: 0x04 -META: 0x08 -CAPS_LOCK: 0x10 -NUM_LOCK: 0x20 -``` - ---- - -## 7. USB HID Scancodes - -``` -A-Z: 0x04-0x1D -0-9: 0x1E-0x27 -ENTER: 0x28 -ESCAPE: 0x29 -BACKSPACE: 0x2A -TAB: 0x2B -SPACE: 0x2C -F1-F12: 0x3A-0x45 -LEFT_CTRL: 0xE0 -LEFT_SHIFT: 0xE1 -LEFT_ALT: 0xE2 -LEFT_META: 0xE3 -RIGHT_CTRL: 0xE4 -RIGHT_SHIFT: 0xE5 -RIGHT_ALT: 0xE6 -RIGHT_META: 0xE7 -``` - ---- - -## 8. Mouse Coalescing - -### Configuration -- Interval: 4ms (250Hz effective rate) -- Constant: `MOUSE_COALESCE_INTERVAL_US = 4_000` - -### Behavior -- Accumulates dx/dy deltas atomically -- Flushed when interval expires OR on button events -- Button events always flush pending movement first - ---- - -## 9. Control Channel Messages - -### finAck (Network Test Result) -```json -{ - "finAck": { - "downlinkBandwidth": , - "packetLoss": , - "latency": - } -} -``` - -### fin (Graceful Shutdown) -```json -{ - "fin": { - "sessionId": "", - "packetsLost": , - "packetsReceived": - } -} -``` - ---- - -## 10. Timestamp Encoding - -### Format -- Type: u64 -- Unit: Microseconds since session start -- Encoding: Big-Endian in event messages - -### Generation -```rust -pub fn get_timestamp_us() -> u64 { - let elapsed = session_start.elapsed().as_micros() as u64; - unix_start_us.wrapping_add(elapsed) -} -``` - ---- - -## 11. Reliability & Ordering - -| Channel | Ordered | Reliable | MaxLifetime | Use Case | -|---------|---------|----------|-------------|----------| -| input_channel_v1 | YES | YES | ∞ | Keyboard, handshake | -| input_channel_partially_reliable | NO | NO | 8ms | Mouse movement | -| cursor_channel | YES | YES | ∞ | Cursor images | -| control_channel | YES | YES | ∞ | Bidirectional control | -| stats_channel | YES | YES | ∞ | Telemetry | - ---- - -## 12. Byte-Level Examples - -### Key Down (Shift+A) -``` -Hex: 03 00 00 00 00 41 00 01 00 00 12 34 56 78 9A BC DE F0 - -[00-03] 03 00 00 00 = Type 3 (LE) = KEY_DOWN -[04-05] 00 41 = Keycode 0x0041 (VK_A) (BE) -[06-07] 00 01 = Modifiers 0x0001 (SHIFT) (BE) -[08-09] 00 00 = Scancode 0x0000 (BE) -[10-17] = Timestamp (BE u64) -``` - -### Mouse Movement (dx=100, dy=-50) -``` -Hex: 07 00 00 00 00 64 FF CE 00 00 00 00 00 00 12 34 56 78 9A BC DE F0 - -[00-03] 07 00 00 00 = Type 7 (LE) = MOUSE_REL -[04-05] 00 64 = dx=100 (BE i16) -[06-07] FF CE = dy=-50 (BE i16, two's complement) -[08-13] = Reserved -[14-21] = Timestamp (BE u64) -``` - -### Protocol v3+ Wrapped Event -``` -[0] 0x22 = Wrapper marker -[1...] Raw event bytes -``` - ---- - -## 13. Implementation Notes - -1. **Channel Creation Order**: Input channels MUST be created BEFORE SDP negotiation -2. **Timestamp Synchronization**: Microseconds relative to session start -3. **Mouse Channel Fallback**: Use reliable channel if partially_reliable not ready -4. **Handshake Required**: No input processed until handshake response echoed -5. **Event Coalescing**: Mouse events coalesce every 4ms -6. **Data Types**: Mixed endianness - opcodes LE, fields BE - ---- - -## 14. Comparison - -| Feature | Web Client | Official Client | OpenNow | -|---------|-----------|-----------------|---------| -| Channel Creation | Explicit | Native C++ | webrtc-rs | -| Coalescing | 4-16ms | Hardware | 4ms | -| Protocol Version | v2/v3+ | Proprietary | v2/v3+ | -| Input Encoding | DataView | Native | bytes crate | -| Channel Types | 4+ channels | Similar | 2 input channels | diff --git a/opennow-streamer/reverse/index.md b/opennow-streamer/reverse/index.md deleted file mode 100644 index 8ecb5c6..0000000 --- a/opennow-streamer/reverse/index.md +++ /dev/null @@ -1,147 +0,0 @@ -# GeForce NOW Reverse Engineering Documentation - -**Last Updated:** 2026-01-01 -**Sources Analyzed:** -- Official Web Client: `C:\Users\Zortos\CustomGFNClient\research` -- Official GFN Client: `C:\Users\Zortos\AppData\Local\NVIDIA Corporation\GeForceNOW` -- OpenNow Implementation: `C:\Users\Zortos\CustomGFNClient\gfn-client\opennow-streamer` - ---- - -## Overview - -This documentation provides comprehensive reverse engineering analysis of NVIDIA GeForce NOW's streaming protocol, comparing three implementations: the official web client, official native client, and the OpenNow open-source implementation. - ---- - -## Documentation Index - -### Core Protocol - -| Document | Description | -|----------|-------------| -| [protocol.md](protocol.md) | WebRTC/RTP protocol details, SDP, ICE, signaling | -| [session.md](session.md) | CloudMatch API, authentication, session management | -| [datachannel.md](datachannel.md) | Data channel message formats and binary protocols | - -### Media - -| Document | Description | -|----------|-------------| -| [video.md](video.md) | Video decoding, RTP packetization, codecs (H.264/H.265/AV1) | -| [audio.md](audio.md) | Audio handling, Opus codec, RTP, synchronization | -| [rendering.md](rendering.md) | GPU rendering, shaders, YUV-RGB conversion | - -### Input - -| Document | Description | -|----------|-------------| -| [keyboard.md](keyboard.md) | Keyboard input protocol, keycodes, modifiers | -| [cursor.md](cursor.md) | Mouse/cursor handling, capture, rendering | -| [controller.md](controller.md) | Gamepad/controller input, button mapping, rumble | - -### Telemetry - -| Document | Description | -|----------|-------------| -| [statistics.md](statistics.md) | QoS metrics, bitrate adaptation, RTCP stats | - ---- - -## Quick Reference - -### Key Endpoints - -``` -Authentication: https://login.nvidia.com/authorize -Token: https://login.nvidia.com/token -CloudMatch: https://{zone}.cloudmatchbeta.nvidiagrid.net/v2/session -Games: https://games.geforce.com/graphql -Service URLs: https://pcs.geforcenow.com/v1/serviceUrls -``` - -### Key Headers - -``` -Authorization: GFNJWT {token} -nv-client-id: {uuid} -nv-client-type: NATIVE -nv-client-version: 2.0.80.173 -nv-client-streamer: NVIDIA-CLASSIC -``` - -### Data Channel Names - -| Channel | Purpose | Reliability | -|---------|---------|-------------| -| `input_channel_v1` | Keyboard, handshake | Reliable, ordered | -| `input_channel_partially_reliable` | Mouse movement | Unreliable, 8ms lifetime | -| `cursor_channel` | Cursor updates | Reliable, ordered | -| `control_channel` | Control messages | Reliable, ordered | - -### Input Message Types - -| Type | Value | Size | Description | -|------|-------|------|-------------| -| HEARTBEAT | 0x02 | 4B | Keep-alive | -| KEY_DOWN | 0x03 | 18B | Keyboard press | -| KEY_UP | 0x04 | 18B | Keyboard release | -| MOUSE_REL | 0x07 | 22B | Relative mouse movement | -| MOUSE_BUTTON_DOWN | 0x08 | 18B | Mouse button press | -| MOUSE_BUTTON_UP | 0x09 | 18B | Mouse button release | -| MOUSE_WHEEL | 0x0A | 22B | Mouse scroll | - -### Video Codecs - -| Codec | Payload Type | Clock Rate | -|-------|--------------|------------| -| H.264 | 96 | 90000 Hz | -| H.265/HEVC | 127 | 90000 Hz | -| AV1 | 98 | 90000 Hz | - -### Audio Codec - -| Codec | Payload Type | Sample Rate | Channels | -|-------|--------------|-------------|----------| -| Opus | 111 | 48000 Hz | 2 (stereo) | -| Multiopus | 100 | 48000 Hz | 2-8 | - -### Color Space (BT.709) - -``` -Y' = (Y - 16/255) * 1.1644 -U' = (U - 128/255) * 1.1384 -V' = (V - 128/255) * 1.1384 - -R = Y' + 1.5748 * V' -G = Y' - 0.1873 * U' - 0.4681 * V' -B = Y' + 1.8556 * U' -``` - ---- - -## Implementation Comparison - -| Feature | Web Client | Official Client | OpenNow | -|---------|-----------|-----------------|---------| -| **Language** | JavaScript | C++ (CEF) | Rust | -| **WebRTC** | Browser native | libwebrtc | webrtc-rs | -| **Video Decode** | Browser | NVDEC | FFmpeg + CUVID | -| **Rendering** | WebGL/WebGPU | DirectX 12 | wgpu | -| **Input** | DOM Events | Raw Input | winit + Raw Input | -| **Audio** | WebAudio | Native | cpal + Opus | - ---- - -## Protocol Version - -- **Client Version:** 2.0.80.173 -- **Input Protocol:** v2/v3+ -- **WebRTC SDP:** Custom nvstSdp extensions -- **OAuth:** PKCE with code_challenge_method=S256 - ---- - -## License - -This documentation is for educational and reverse engineering purposes only. diff --git a/opennow-streamer/reverse/keyboard.md b/opennow-streamer/reverse/keyboard.md deleted file mode 100644 index 57c8ea6..0000000 --- a/opennow-streamer/reverse/keyboard.md +++ /dev/null @@ -1,273 +0,0 @@ -# GeForce NOW Keyboard Input Protocol - Reverse Engineering Documentation - -## 1. Key Event Structure - -### Binary Message Format - -#### KeyDown Event (Type 3) -``` -Byte Layout (18 bytes total): -[0-3] Type: 0x03 (4 bytes, Little Endian) = INPUT_KEY_DOWN -[4-5] Keycode: u16 (2 bytes, Big Endian) = Windows Virtual Key code -[6-7] Modifiers: u16 (2 bytes, Big Endian) = Modifier flags bitmask -[8-9] Scancode: u16 (2 bytes, Big Endian) = USB HID scancode (usually 0) -[10-17] Timestamp: u64 (8 bytes, Big Endian) = Microseconds since session start -``` - -#### KeyUp Event (Type 4) -``` -Byte Layout (18 bytes total): -[0-3] Type: 0x04 (4 bytes, Little Endian) = INPUT_KEY_UP -[4-5] Keycode: u16 (2 bytes, Big Endian) = Windows Virtual Key code -[6-7] Modifiers: u16 (2 bytes, Big Endian) = Modifier flags bitmask -[8-9] Scancode: u16 (2 bytes, Big Endian) = USB HID scancode (usually 0) -[10-17] Timestamp: u64 (8 bytes, Big Endian) = Microseconds since session start -``` - -### Protocol v3+ Wrapper -For protocol version 3+, single events are wrapped: -``` -[0] Wrapper Marker: 0x22 (34 decimal) -[1-18] Keyboard event payload -Total: 19 bytes for v3+ -``` - ---- - -## 2. Virtual Key Codes - -GFN uses **Windows Virtual Key codes** (VK codes), NOT scancodes. - -### Alphabetic Keys -``` -VK_A (0x41) through VK_Z (0x5A) -``` - -### Numeric Keys -``` -VK_0 (0x30) through VK_9 (0x39) -``` - -### Function Keys -``` -VK_F1 (0x70) through VK_F12 (0x7B) -VK_F13 (0x7C) through VK_F24 (0x87) -``` - -### Special Keys -``` -VK_ESCAPE (0x1B) -VK_TAB (0x09) -VK_CAPITAL (0x14) - CapsLock -VK_SPACE (0x20) -VK_ENTER (0x0D) -VK_BACKSPACE (0x08) -VK_DELETE (0x2E) -VK_INSERT (0x2D) -VK_HOME (0x24) -VK_END (0x23) -VK_PRIOR (0x21) - Page Up -VK_NEXT (0x22) - Page Down -``` - -### Arrow Keys -``` -VK_UP (0x26) -VK_DOWN (0x28) -VK_LEFT (0x25) -VK_RIGHT (0x27) -``` - -### Numpad Keys -``` -VK_NUMPAD0 (0x60) through VK_NUMPAD9 (0x69) -VK_MULTIPLY (0x6A) -VK_ADD (0x6B) -VK_SUBTRACT (0x6D) -VK_DECIMAL (0x6E) -VK_DIVIDE (0x6F) -VK_NUMLOCK (0x90) -``` - -### Modifier Keys -``` -VK_LSHIFT (0xA0) - Left Shift -VK_RSHIFT (0xA1) - Right Shift -VK_LCONTROL (0xA2) - Left Control -VK_RCONTROL (0xA3) - Right Control -VK_LMENU (0xA4) - Left Alt -VK_RMENU (0xA5) - Right Alt -VK_LWIN (0x5B) - Left Windows/Meta -VK_RWIN (0x5C) - Right Windows/Meta -``` - -### Punctuation Keys -``` -VK_OEM_MINUS (0xBD) - Minus/Underscore -VK_OEM_PLUS (0xBB) - Plus/Equals -VK_OEM_LBRACKET (0xDB) - Left Bracket -VK_OEM_RBRACKET (0xDD) - Right Bracket -VK_OEM_BACKSLASH (0xDC) - Backslash -VK_OEM_SEMICOLON (0xBA) - Semicolon -VK_OEM_QUOTE (0xDE) - Quote -VK_OEM_TILDE (0xC0) - Backtick/Tilde -VK_OEM_COMMA (0xBC) - Comma -VK_OEM_PERIOD (0xBE) - Period -VK_OEM_SLASH (0xBF) - Forward Slash -``` - ---- - -## 3. Modifier Flags - -Modifiers are encoded as a 16-bit bitmask: - -``` -SHIFT: 0x01 -CTRL: 0x02 -ALT: 0x04 -META: 0x08 -CAPS_LOCK: 0x10 -NUM_LOCK: 0x20 -``` - -### Important Modifier Behavior - -When a modifier key itself is pressed, the modifiers field should be **0x0000**: -``` -Shift key down: keycode=0xA0, modifiers=0x00 (not 0x01) -A key down (with Shift held): keycode=0x41, modifiers=0x01 -``` - ---- - -## 4. USB HID Scancodes - -The scancode field is typically set to **0x0000** (unused) in GFN. - -Reference scancodes (if needed): -``` -0x04 = A through 0x1D = Z -0x1E = 1 through 0x27 = 0 -0x28 = Enter -0x29 = Escape -0x2A = Backspace -0x2B = Tab -0x2C = Space -0x3A - 0x45 = F1 through F12 -0xE0-0xE7 = Modifier keys -``` - ---- - -## 5. Key Repeat Handling - -Key repeat events are **filtered out**: - -```rust -if event.repeat { - return; // Skip key repeat events -} -``` - -### Key State Tracking - -Both clients track currently pressed keys: -```rust -pub struct InputHandler { - pressed_keys: Mutex>, -} -``` - -### Focus Loss Handling - -When window loses focus, all keys are released: -```rust -pub fn release_all_keys(&self) { - let keys_to_release: Vec = pressed_keys.drain().collect(); - for keycode in keys_to_release { - send_key_up(keycode, 0, 0, timestamp_us); - } -} -``` - ---- - -## 6. IME (Input Method Editor) Support - -### Two Independent Channels - -1. **Raw Keyboard Events**: KeyDown/KeyUp via WebRTC data channel -2. **Text Composition Events**: UTF-8 text via event emitter - -### Text Composition Event -```javascript -this.emit("TextComposition", { - compositionText: "input_text", - imeRecommendation: true -}); -``` - ---- - -## 7. Data Channel Usage - -- **Channel Name**: `input_channel_v1` -- **Ordered**: Yes -- **Reliable**: Yes - -Keyboard events are always sent via reliable channel (no tolerance for dropped events). - ---- - -## 8. Timestamp Format - -Each keyboard event carries a microsecond-precision timestamp: - -```rust -fn get_timestamp_us() -> u64 { - let elapsed_us = session_start.elapsed().as_micros() as u64; - unix_start_us.wrapping_add(elapsed_us) -} -``` - ---- - -## 9. Byte-Level Example - -### KeyDown for Shift+A -``` -Hex: 03 00 00 00 41 00 01 00 00 00 12 34 56 78 9A BC DE F0 - -[00-03] 03 00 00 00 = Type 3 (LE) = KeyDown -[04-05] 00 41 = Keycode 0x0041 (VK_A) (BE) -[06-07] 00 01 = Modifiers 0x0001 (SHIFT) (BE) -[08-09] 00 00 = Scancode 0x0000 (BE) -[10-17] 12 34 56 78 9A BC DE F0 = Timestamp (BE) -``` - ---- - -## 10. Comparison - -| Feature | Web Client | Official Client | OpenNow | -|---------|-----------|-----------------|---------| -| Key Mapping | so.get()/Fo.get() maps | Platform-native | Rust match | -| Timestamp | Browser timeStamp | System clock | Session-relative | -| IME Support | Full composition | Basic | Not implemented | -| Key Repeat | DOM event.repeat | Manual filtering | Manual filtering | -| Scancode | Always 0x0000 | Always 0x0000 | Always 0x0000 | - ---- - -## 11. Implementation Checklist - -- [ ] Map event.code to Windows VK codes -- [ ] Extract modifier state (ctrl/alt/shift/meta) -- [ ] Skip events with event.repeat === true -- [ ] Create 18-byte binary message -- [ ] Include microsecond timestamp -- [ ] Set scancode to 0x0000 -- [ ] Send via `input_channel_v1` -- [ ] Track pressed keys to avoid duplicates -- [ ] Release all keys on window focus loss diff --git a/opennow-streamer/reverse/protocol.md b/opennow-streamer/reverse/protocol.md deleted file mode 100644 index ffbcf06..0000000 --- a/opennow-streamer/reverse/protocol.md +++ /dev/null @@ -1,315 +0,0 @@ -# GeForce NOW WebRTC & RTP Protocol - Reverse Engineering Documentation - -## 1. Authentication Flow - -### OAuth 2.0 with PKCE -``` -Endpoint: https://login.nvidia.com/authorize -Token: https://login.nvidia.com/token -Client ID: ZU7sPN-miLujMD95LfOQ453IB0AtjM8sMyvgJ9wCXEQ -Scopes: openid consent email tk_client age -``` - -### Request Parameters -```json -{ - "response_type": "code", - "device_id": "sha256(hostname + username + 'opennow-streamer')", - "scope": "openid consent email tk_client age", - "client_id": "ZU7sPN-miLujMD95LfOQ453IB0AtjM8sMyvgJ9wCXEQ", - "redirect_uri": "http://localhost:{port}", - "code_challenge": "sha256_base64(verifier)", - "code_challenge_method": "S256", - "idp_id": "PDiAhv2kJTFeQ7WOPqiQ2tRZ7lGhR2X11dXvM4TZSxg" -} -``` - -### Token Response -```json -{ - "access_token": "...", - "refresh_token": "...", - "id_token": "...", - "expires_in": 86400 -} -``` - -### Authorization Header -``` -Authorization: GFNJWT {token} -``` - ---- - -## 2. CloudMatch Session API - -### Base URL -``` -https://{zone}.cloudmatchbeta.nvidiagrid.net/v2/session -``` - -### Required Headers -``` -Authorization: GFNJWT {token} -Content-Type: application/json -nv-client-id: {uuid} -nv-client-type: NATIVE -nv-client-version: 2.0.80.173 -nv-client-streamer: NVIDIA-CLASSIC -nv-device-os: WINDOWS -nv-device-type: DESKTOP -x-device-id: {device-id} -Origin: https://play.geforcenow.com -``` - -### Create Session (POST /v2/session) -```json -{ - "sessionRequestData": { - "appId": "string", - "internalTitle": "Game Title", - "clientIdentification": "GFN-PC", - "deviceHashId": "{uuid}", - "clientVersion": "30.0", - "clientPlatformName": "windows", - "clientRequestMonitorSettings": [{ - "widthInPixels": 1920, - "heightInPixels": 1080, - "framesPerSecond": 60 - }], - "metaData": [ - {"key": "GSStreamerType", "value": "WebRTC"}, - {"key": "wssignaling", "value": "1"} - ], - "requestedStreamingFeatures": { - "reflex": false, - "trueHdr": false - } - } -} -``` - -### Session Response -```json -{ - "session": { - "sessionId": "string", - "status": 2, - "gpuType": "RTX_A5000", - "connectionInfo": [{ - "ip": "server_ip", - "port": 47998, - "usage": 14 - }] - }, - "requestStatus": { - "statusCode": 1, - "serverId": "NP-AMS-08" - } -} -``` - -### Session States -- **1**: Setting up / Launching -- **2**: Ready for streaming -- **3**: Already streaming -- **6**: Initialization pending - ---- - -## 3. WebSocket Signaling - -### Connection -``` -URL: wss://{server_ip}:443/nvst/sign_in?peer_id={peer_name}&version=2 -Subprotocol: x-nv-sessionid.{sessionId} -``` - -### Peer Info Message -```json -{ - "ackid": 1, - "peer_info": { - "id": 2, - "name": "peer-{random}", - "browser": "Chrome", - "browserVersion": "131", - "connected": true, - "peerRole": 0, - "resolution": "1920x1080", - "version": 2 - } -} -``` - -### Heartbeat (every 5s) -```json -{"hb": 1} -``` - -### Acknowledgment -```json -{"ack": 1} -``` - -### SDP Offer (from server) -```json -{ - "ackid": 2, - "peer_msg": { - "from": 1, - "to": 2, - "msg": "{\"type\":\"offer\",\"sdp\":\"v=0\\r\\no=...\"}" - } -} -``` - -### SDP Answer (to server) -```json -{ - "ackid": 3, - "peer_msg": { - "from": 2, - "to": 1, - "msg": "{\"type\":\"answer\",\"sdp\":\"...\",\"nvstSdp\":\"v=0\\r\\n...\"}" - } -} -``` - -### ICE Candidate -```json -{ - "ackid": 4, - "peer_msg": { - "from": 2, - "to": 1, - "msg": "{\"candidate\":\"candidate:...\",\"sdpMid\":\"...\",\"sdpMLineIndex\":0}" - } -} -``` - ---- - -## 4. SDP (Session Description Protocol) - -### ICE-Lite Detection -``` -a=ice-lite -``` -When server is ice-lite: -- Client MUST respond with `a=setup:active` -- Client initiates DTLS ClientHello - -### nvstSdp Attributes - -**FEC Settings:** -``` -a=vqos.fec.rateDropWindow:10 -a=vqos.fec.minRequiredFecPackets:2 -a=vqos.fec.repairMinPercent:5 -a=vqos.fec.repairMaxPercent:35 -``` - -**Dynamic Quality Control:** -``` -a=vqos.dfc.enable:1 -a=vqos.dfc.decodeFpsAdjPercent:85 -a=vqos.dfc.targetDownCooldownMs:250 -a=vqos.dfc.minTargetFps:100 -``` - -**Bitrate Control:** -``` -a=video.initialBitrateKbps:25000 -a=vqos.bw.maximumBitrateKbps:50000 -a=vqos.bw.minimumBitrateKbps:5000 -a=bwe.useOwdCongestionControl:1 -``` - -**NACK Settings:** -``` -a=video.enableRtpNack:1 -a=video.rtpNackQueueLength:1024 -a=video.rtpNackQueueMaxPackets:512 -``` - ---- - -## 5. RTP Protocol - -### Video Payload Types -- **96**: H.264 -- **127**: H.265/HEVC -- **98**: AV1 - -### Audio Payload Types -- **111**: Opus (stereo) -- **100**: Multiopus (up to 8 channels) - -### Clock Rates -- Video: 90000 Hz -- Audio: 48000 Hz - ---- - -## 6. RTCP Feedback - -### PLI (Picture Loss Indication) -```rust -let pli = PictureLossIndication { - sender_ssrc: 0, - media_ssrc: VIDEO_SSRC, -}; -peer_connection.write_rtcp(&[Box::new(pli)]).await? -``` - -### NACK (Negative Acknowledgment) -``` -Bitmask of missing sequence numbers -Requests retransmission of specific packets -``` - ---- - -## 7. Data Channels - -### Channel Names -| Channel | Ordered | Reliable | Purpose | -|---------|---------|----------|---------| -| input_channel_v1 | Yes | Yes | Keyboard, handshake | -| input_channel_partially_reliable | No | No (8ms) | Mouse movement | -| cursor_channel | Yes | Yes | Cursor updates | -| control_channel | Yes | Yes | Control messages | - -### Handshake Protocol -``` -Server → Client: [0x0e, major, minor, flags] -Client → Server: Echo same bytes -``` - ---- - -## 8. DTLS/TLS Security - -### Handshake States -1. ice-gathering -2. ice-connected -3. dtls-connecting -4. dtls-connected -5. peer-connected - -### Certificate Handling -- Self-signed certificates accepted -- `danger_accept_invalid_certs(true)` - ---- - -## 9. Comparison - -| Feature | Web Client | OpenNow | Official Client | -|---------|-----------|---------|-----------------| -| WebRTC | Browser | webrtc-rs | libwebrtc | -| Signaling | JavaScript | Tokio WS | Native C++ | -| ICE | Browser | Manual | libwebrtc | -| DTLS | Browser | webrtc-rs | Native | -| Data Channels | Browser | webrtc-rs | Native | diff --git a/opennow-streamer/reverse/rendering.md b/opennow-streamer/reverse/rendering.md deleted file mode 100644 index 019fdfa..0000000 --- a/opennow-streamer/reverse/rendering.md +++ /dev/null @@ -1,314 +0,0 @@ -# GeForce NOW Video Rendering & Shaders - Reverse Engineering Documentation - -## 1. GPU Architecture - -### Framework Selection -- **Windows**: wgpu with DirectX 12 backend (exclusive fullscreen support) -- **macOS**: wgpu with Metal backend -- **Linux**: wgpu with Vulkan backend - -### Backend Priority -```rust -#[cfg(target_os = "windows")] -let backends = wgpu::Backends::DX12; // Forced for exclusive fullscreen - -#[cfg(target_os = "macos")] -let backends = wgpu::Backends::METAL; - -#[cfg(target_os = "linux")] -let backends = wgpu::Backends::VULKAN; -``` - ---- - -## 2. WGSL Shader Architecture - -### Video Shader (YUV420P) -3 separate texture planes: -- Y plane: R8Unorm (full resolution) -- U plane: R8Unorm (half resolution) -- V plane: R8Unorm (half resolution) - -```wgsl -@group(0) @binding(0) var y_texture: texture_2d; -@group(0) @binding(1) var u_texture: texture_2d; -@group(0) @binding(2) var v_texture: texture_2d; -@group(0) @binding(3) var tex_sampler: sampler; - -@fragment -fn fs_main(@location(0) tex_coord: vec2) -> @location(0) vec4 { - let y_raw = textureSample(y_texture, tex_sampler, tex_coord).r; - let u_raw = textureSample(u_texture, tex_sampler, tex_coord).r; - let v_raw = textureSample(v_texture, tex_sampler, tex_coord).r; - - // BT.709 limited range to full range - let y = (y_raw - 0.0625) * 1.1644; - let u = (u_raw - 0.5) * 1.1384; - let v = (v_raw - 0.5) * 1.1384; - - // BT.709 color matrix - let r = y + 1.5748 * v; - let g = y - 0.1873 * u - 0.4681 * v; - let b = y + 1.8556 * u; - - return vec4(clamp(r, 0.0, 1.0), clamp(g, 0.0, 1.0), clamp(b, 0.0, 1.0), 1.0); -} -``` - -### NV12 Shader (Semi-planar) -2 planes with interleaved UV: -- Y plane: R8Unorm (full resolution) -- UV plane: Rg8Unorm (half resolution, interleaved) - -```wgsl -@group(0) @binding(0) var y_texture: texture_2d; -@group(0) @binding(1) var uv_texture: texture_2d; -@group(0) @binding(2) var tex_sampler: sampler; - -@fragment -fn fs_main(@location(0) tex_coord: vec2) -> @location(0) vec4 { - let y_raw = textureSample(y_texture, tex_sampler, tex_coord).r; - let uv = textureSample(uv_texture, tex_sampler, tex_coord).rg; - - let y = (y_raw - 0.0625) * 1.1644; - let u = (uv.r - 0.5) * 1.1384; - let v = (uv.g - 0.5) * 1.1384; - - let r = y + 1.5748 * v; - let g = y - 0.1873 * u - 0.4681 * v; - let b = y + 1.8556 * u; - - return vec4(clamp(r, 0.0, 1.0), clamp(g, 0.0, 1.0), clamp(b, 0.0, 1.0), 1.0); -} -``` - ---- - -## 3. BT.709 Color Space Conversion - -### Limited Range to Full Range -``` -Y' = (Y - 16/255) × (255/219) = (Y - 0.0625) × 1.1644 -U' = (U - 128/255) × (255/224) = (U - 0.5) × 1.1384 -V' = (V - 128/255) × (255/224) = (V - 0.5) × 1.1384 -``` - -### BT.709 Matrix -``` -R = Y' + 1.5748 × V' -G = Y' - 0.1873 × U' - 0.4681 × V' -B = Y' + 1.8556 × U' -``` - -### Matrix Form -``` -[R] [1.0000 0.0000 1.5748] [Y'] -[G] = [1.0000 -0.1873 -0.4681] [U'] -[B] [1.0000 1.8556 0.0000] [V'] -``` - -### CPU Fallback (Integer Math) -```rust -let r = (y + ((359 * v) >> 8)).clamp(0, 255) as u8; -let g = (y - ((88 * u + 183 * v) >> 8)).clamp(0, 255) as u8; -let b = (y + ((454 * u) >> 8)).clamp(0, 255) as u8; -``` - ---- - -## 4. Texture Formats - -### YUV420P Memory Layout (1920×1080) -``` -Y plane: 1920 × 1080 = 2,073,600 bytes -U plane: 960 × 540 = 518,400 bytes -V plane: 960 × 540 = 518,400 bytes -Total: 3,110,400 bytes (2.97 MB/frame) -``` - -### NV12 Memory Layout (1920×1080) -``` -Y plane: 1920 × 1080 = 2,073,600 bytes -UV plane: 1920 × 540 = 1,036,800 bytes (interleaved) -Total: 3,110,400 bytes (same size) -``` - ---- - -## 5. Present Mode Configuration - -### Latency Optimization -```rust -let present_mode = if caps.contains(&wgpu::PresentMode::Immediate) { - wgpu::PresentMode::Immediate // Best latency -} else if caps.contains(&wgpu::PresentMode::Mailbox) { - wgpu::PresentMode::Mailbox // Intermediate -} else { - wgpu::PresentMode::Fifo // VSync (fallback) -}; - -// Minimum frame latency -config.desired_maximum_frame_latency = 1; -``` - -### Present Mode Hierarchy -1. **Immediate**: No vsync, submit immediately (lowest latency) -2. **Mailbox**: Non-blocking buffer swap (intermediate) -3. **Fifo**: VSync blocking (highest latency) - ---- - -## 6. Exclusive Fullscreen (Windows) - -### DWM Bypass -- Bypasses Desktop Window Manager compositor -- Enables higher refresh rates (120Hz+) -- Lower input latency - -### Implementation -```rust -// Find video mode with highest refresh rate -let modes = monitor.video_modes(); -let best_mode = modes - .filter(|m| m.size().width >= width && m.size().height >= height) - .max_by_key(|m| m.refresh_rate_millihertz()); - -window.set_fullscreen(Some(Fullscreen::Exclusive(best_mode))); -``` - ---- - -## 7. macOS ProMotion Support - -### Frame Rate Configuration -```rust -struct CAFrameRateRange { - minimum: 120.0, - maximum: 120.0, - preferred: 120.0, // Force 120Hz -} -``` - -### High-Performance Mode -- `NSActivityUserInitiated`: Prevents App Nap -- `NSActivityLatencyCritical`: Low-latency scheduling -- Disables auto-termination - ---- - -## 8. Render Pipeline Stages - -### Order of Operations -1. **Swapchain Error Recovery**: Handle Outdated/Lost surface -2. **Video Frame Update**: Upload YUV/NV12 planes to GPU -3. **Video Render Pass**: Execute shader on full-screen quad -4. **egui UI Render Pass**: Render overlay UI -5. **Present**: Display to screen - ---- - -## 9. Hardware Decoder Integration - -### Decoder Priority (Windows/Linux) -1. NVIDIA CUVID (H.264, H.265, AV1) -2. Intel QSV (H.264, H.265, AV1) -3. D3D11VA (Windows) -4. VAAPI (Linux) -5. Software decoder (fallback) - -### Decoder Priority (macOS) -1. VideoToolbox (native) -2. Software decoder (fallback) - -### Output Formats -- **NV12**: Direct from VideoToolbox, CUVID, QSV (preferred) -- **YUV420P**: Converted via FFmpeg if needed - ---- - -## 10. Zero-Latency Frame Delivery - -### SharedFrame Structure -```rust -pub struct SharedFrame { - frame: Mutex>, - frame_count: AtomicU64, - last_read_count: AtomicU64, -} -``` - -### Design Principles -- Decoder writes latest frame to SharedFrame -- Renderer reads via take() (zero copy) -- No frame buffering = always most recent -- Atomic frame counter detects new frames - ---- - -## 11. Comparison - -| Feature | Web Client | OpenNow | Official Client | -|---------|-----------|---------|-----------------| -| Rendering API | WebGL/WebGPU | wgpu (Rust) | DirectX 12/Vulkan | -| YUV Conversion | GPU shader | WGSL shader | HLSL shader | -| Color Space | BT.709 | BT.709 | BT.709 | -| Texture Format | YUV420P/NV12 | YUV420P/NV12 | NV12 | -| Present Mode | Vsync | Immediate | Exclusive fullscreen | -| Latency | 40-80ms | <20ms | 10-30ms | -| CPU Load | ~5-10% | ~5% | ~3-5% | - ---- - -## 12. Performance Metrics - -### GPU Memory (1440p) -``` -Per-frame textures: - Y plane: 2560 × 1440 = 3,686,400 bytes - U plane: 1280 × 720 = 921,600 bytes - V plane: 1280 × 720 = 921,600 bytes - Total: 5,529,600 bytes (~5.3 MB) - -Triple buffering: ~15.9 MB total -``` - -### Frame Timing (1440p120) -``` -Decode time: 8-12ms (hardware accelerated) -GPU shader: <1ms -Render pass: <2ms -Total: <15ms per frame -``` - ---- - -## 13. Why BT.709? - -- **HD Content Standard**: Streams are 720p+ -- **Color Accuracy**: Better flesh tones -- **Industry Standard**: Used by all streaming services -- **Apple/NVIDIA Alignment**: Both prefer BT.709 - ---- - -## 14. NV12 Optimization Benefits - -1. **No interleaving**: Single memory layout -2. **Fewer textures**: 2 instead of 3 -3. **GPU efficiency**: Direct from hardware decoders -4. **Bandwidth**: Fewer texture fetch operations - ---- - -## 15. HDR Considerations - -### Current Status -- Not implemented in OpenNow -- Surface format: Linear (non-sRGB) -- No HDR10 texture formats -- No BT.2020 color space - -### Future Requirements -- SCRGB (Extended Dynamic Range) -- BT.2020 color primaries -- SMPTE ST.2084 tone-mapping diff --git a/opennow-streamer/reverse/session.md b/opennow-streamer/reverse/session.md deleted file mode 100644 index 022bd5f..0000000 --- a/opennow-streamer/reverse/session.md +++ /dev/null @@ -1,396 +0,0 @@ -# GeForce NOW Session & API Management - Reverse Engineering Documentation - -## 1. Authentication Flow - -### OAuth 2.0 with PKCE -``` -Endpoint: https://login.nvidia.com/authorize -Token: https://login.nvidia.com/token -Client ID: ZU7sPN-miLujMD95LfOQ453IB0AtjM8sMyvgJ9wCXEQ -Scopes: openid consent email tk_client age -IDP ID: PDiAhv2kJTFeQ7WOPqiQ2tRZ7lGhR2X11dXvM4TZSxg -``` - -### PKCE Parameters -```json -{ - "response_type": "code", - "device_id": "sha256(hostname + username + 'opennow-streamer')", - "scope": "openid consent email tk_client age", - "client_id": "ZU7sPN-miLujMD95LfOQ453IB0AtjM8sMyvgJ9wCXEQ", - "redirect_uri": "http://localhost:{port}", - "code_challenge": "sha256_base64(verifier)", - "code_challenge_method": "S256", - "idp_id": "PDiAhv2kJTFeQ7WOPqiQ2tRZ7lGhR2X11dXvM4TZSxg" -} -``` - -### Token Response -```json -{ - "access_token": "...", - "refresh_token": "...", - "id_token": "jwt_token", - "expires_in": 86400 -} -``` - -### Authorization Headers -``` -Native: Authorization: GFNJWT {token} -Partner: Authorization: GFNPartnerJWT auth={token} -OAuth: Authorization: Bearer {token} -``` - ---- - -## 2. Service URLs API - -### Endpoint -``` -GET https://pcs.geforcenow.com/v1/serviceUrls -``` - -### Response -Array of login providers with: -- `idp_id`: Identity provider ID -- `streaming_service_url`: Region-specific base URL - -### Alliance Partners -- KDD, TWM, BPC/bro.game, etc. -- Custom streaming URLs from serviceUrls response - ---- - -## 3. CloudMatch Session API - -### Base URL -``` -https://{zone}.cloudmatchbeta.nvidiagrid.net/v2/session -``` - -### Required Headers -``` -User-Agent: Mozilla/5.0 ... NVIDIACEFClient/HEAD/debb5919f6 GFN-PC/2.0.80.173 -Authorization: GFNJWT {token} -Content-Type: application/json -Origin: https://play.geforcenow.com -nv-client-id: {uuid} -nv-client-type: NATIVE -nv-client-version: 2.0.80.173 -nv-client-streamer: NVIDIA-CLASSIC -nv-device-os: WINDOWS -nv-device-type: DESKTOP -x-device-id: {uuid} -``` - ---- - -## 4. Create Session - -### POST /v2/session -```json -{ - "sessionRequestData": { - "appId": "string", - "internalTitle": "Game Name", - "clientIdentification": "GFN-PC", - "deviceHashId": "{uuid}", - "clientVersion": "30.0", - "clientPlatformName": "windows", - "clientRequestMonitorSettings": [{ - "widthInPixels": 1920, - "heightInPixels": 1080, - "framesPerSecond": 60 - }], - "metaData": [ - {"key": "GSStreamerType", "value": "WebRTC"}, - {"key": "wssignaling", "value": "1"} - ], - "requestedStreamingFeatures": { - "reflex": false, - "trueHdr": false - } - } -} -``` - -### Session Response -```json -{ - "session": { - "sessionId": "string", - "status": 2, - "gpuType": "RTX_A5000", - "connectionInfo": [{ - "ip": "server_ip", - "port": 443, - "usage": 14, - "protocol": 1, - "resourcePath": "/nvst/" - }], - "iceServerConfiguration": { - "iceServers": [ - {"urls": "turn:server:port", "username": "...", "credential": "..."} - ] - } - }, - "requestStatus": { - "statusCode": 1, - "serverId": "NP-AMS-08" - } -} -``` - ---- - -## 5. Session States - -| Status | Description | -|--------|-------------| -| 1 | Launching / Setting up | -| 2 | Ready for streaming | -| 3 | Actively streaming | -| 6 | Initialization pending | - ---- - -## 6. Poll Session - -### GET /v2/session/{sessionId} -Same response format as create session. - ---- - -## 7. Stop Session - -### DELETE /v2/session/{sessionId} -Terminates the session. - ---- - -## 8. Resume Session - -### PUT /v2/session/{sessionId} -```json -{ - "action": 2, - "data": "RESUME", - "sessionRequestData": { ... } -} -``` - ---- - -## 9. Error Codes - -### CloudMatch Status Codes -| Code | Description | -|------|-------------| -| 1 | Success | -| 2 | Forbidden | -| 3 | Timeout | -| 4 | Internal Error | -| 11 | Session Limit Exceeded | -| 14 | Auth Failure | -| 16 | Token Expired | -| 25 | Service Unavailable | -| 50 | Device Limit Reached | -| 51 | Zone At Capacity | -| 86 | Insufficient Playability | - -### Unified Error Codes (i64) -``` -15859712: Success -3237093643: Session Limit Exceeded -3237093648: Token Expired -3237093657: Service Unavailable -3237093682: Device Session Limit -3237093715: Max Session Limit -3237093718: Insufficient Playability -``` - ---- - -## 10. WebSocket Signaling - -### Connection URL -``` -wss://{server_ip}:443/nvst/sign_in?peer_id={peer_name}&version=2 - -Subprotocol: x-nv-sessionid.{sessionId} -``` - -### Peer Info Message -```json -{ - "ackid": 1, - "peer_info": { - "id": 2, - "name": "peer-{random}", - "browser": "Chrome", - "browserVersion": "131", - "connected": true, - "peerRole": 0, - "resolution": "1920x1080", - "version": 2 - } -} -``` - -### Heartbeat (Every 5s) -```json -{"hb": 1} -``` - -### Acknowledgment -```json -{"ack": 1} -``` - -### SDP Offer (from server) -```json -{ - "ackid": 2, - "peer_msg": { - "from": 1, - "to": 2, - "msg": "{\"type\":\"offer\",\"sdp\":\"v=0\\r\\no=...\"}" - } -} -``` - -### SDP Answer (to server) -```json -{ - "ackid": 3, - "peer_msg": { - "from": 2, - "to": 1, - "msg": "{\"type\":\"answer\",\"sdp\":\"...\",\"nvstSdp\":\"v=0\\r\\n...\"}" - } -} -``` - -### ICE Candidate -```json -{ - "ackid": 4, - "peer_msg": { - "from": 2, - "to": 1, - "msg": "{\"candidate\":\"candidate:...\",\"sdpMid\":\"...\",\"sdpMLineIndex\":0}" - } -} -``` - ---- - -## 11. nvstSdp Parameters - -### FEC Settings -``` -a=vqos.fec.rateDropWindow:10 -a=vqos.fec.minRequiredFecPackets:2 -a=vqos.fec.repairMinPercent:5 -a=vqos.fec.repairMaxPercent:35 -``` - -### Dynamic Quality Control -``` -a=vqos.dfc.enable:1 -a=vqos.dfc.decodeFpsAdjPercent:85 -a=vqos.dfc.targetDownCooldownMs:250 -a=vqos.dfc.minTargetFps:100 -``` - -### Bitrate Control -``` -a=video.initialBitrateKbps:25000 -a=vqos.bw.maximumBitrateKbps:50000 -a=vqos.bw.minimumBitrateKbps:5000 -a=bwe.useOwdCongestionControl:1 -``` - -### NACK Settings -``` -a=video.enableRtpNack:1 -a=video.rtpNackQueueLength:1024 -a=video.rtpNackQueueMaxPackets:512 -``` - ---- - -## 12. Game Library API - -### GraphQL Endpoint -``` -POST https://games.geforce.com/graphql - -Persisted Query Hash: f8e26265a5db5c20e1334a6872cf04b6e3970507697f6ae55a6ddefa5420daf0 -``` - -### Public Games List -``` -GET https://static.nvidiagrid.net/supported-public-game-list/locales/gfnpc-en-US.json -``` - -### Game Images -``` -https://cdn.cloudflare.steamstatic.com/steam/apps/{steam_id}/library_600x900.jpg -``` - ---- - -## 13. Subscription API - -### Endpoint -``` -GET https://mes.geforcenow.com/v4/subscriptions -?serviceName=gfn_pc&languageCode=en_US&vpcId={vpc_id}&userId={user_id} -``` - ---- - -## 14. Server Info - -### GET /v2/serverInfo -```json -{ - "requestStatus": { - "serverId": "NP-AMS-08" - }, - "metaData": [ - {"key": "region_name", "value": "https://region.cloudmatchbeta.nvidiagrid.net/"} - ] -} -``` - ---- - -## 15. Client Type Headers - -### Native Client -``` -nv-client-type: NATIVE -nv-client-streamer: NVIDIA-CLASSIC -``` - -### Browser/WebRTC -``` -nv-client-type: BROWSER -nv-client-streamer: WEBRTC -``` - ---- - -## 16. Comparison - -| Feature | Web Client | Official Client | OpenNow | -|---------|-----------|-----------------|---------| -| Session API | /v2/session POST | /v2/session POST | /v2/session POST | -| Auth Header | GFNJWT | GFNJWT | GFNJWT | -| WS Signaling | Custom JS | Native C++ | Tokio WebSocket | -| Session Polling | Callback-based | Polling loop | Async polling | -| Heartbeat | Every 5s | Every 5s | Every 5s | -| Alliance Partners | Full support | Full support | Full support | diff --git a/opennow-streamer/reverse/statistics.md b/opennow-streamer/reverse/statistics.md deleted file mode 100644 index 197f3f5..0000000 --- a/opennow-streamer/reverse/statistics.md +++ /dev/null @@ -1,347 +0,0 @@ -# GeForce NOW Statistics & QoS - Reverse Engineering Documentation - -## 1. Statistics Structure - -### StreamStats (OpenNow) -```rust -pub struct StreamStats { - pub resolution: String, - pub fps: f32, - pub render_fps: f32, - pub target_fps: u32, - pub bitrate_mbps: f32, - pub latency_ms: f32, - pub decode_time_ms: f32, - pub render_time_ms: f32, - pub input_latency_ms: f32, - pub codec: String, - pub gpu_type: String, - pub server_region: String, - pub packet_loss: f32, - pub jitter_ms: f32, - pub frames_received: u64, - pub frames_decoded: u64, - pub frames_dropped: u64, - pub frames_rendered: u64, -} -``` - -### DecodeStats -```rust -pub struct DecodeStats { - pub decode_time_ms: f32, - pub frame_produced: bool, - pub needs_keyframe: bool, -} -``` - ---- - -## 2. QoS SDP Parameters - -### FEC (Forward Error Correction) -``` -a=vqos.fec.rateDropWindow:10 -a=vqos.fec.minRequiredFecPackets:2 -a=vqos.fec.repairMinPercent:5 -a=vqos.fec.repairPercent:5 -a=vqos.fec.repairMaxPercent:35 -``` - -### DFC (Dynamic FPS Control) -``` -a=vqos.dfc.enable:1 -a=vqos.dfc.decodeFpsAdjPercent:85 -a=vqos.dfc.targetDownCooldownMs:250 -a=vqos.dfc.dfcAlgoVersion:2 -a=vqos.dfc.minTargetFps:100 (or 60 for lower fps) -``` - -### DRC (Dynamic Resolution Control) -``` -a=vqos.drc.minQpHeadroom:20 -a=vqos.drc.lowerQpThreshold:100 -a=vqos.drc.upperQpThreshold:200 -a=vqos.drc.minAdaptiveQpThreshold:180 -a=vqos.drc.iirFilterFactor:100 -``` - -### Bitrate Control -``` -a=vqos.bw.maximumBitrateKbps:{max_bitrate} -a=vqos.bw.minimumBitrateKbps:{max_bitrate / 10} -a=video.initialBitrateKbps:{max_bitrate / 2} -a=video.initialPeakBitrateKbps:{max_bitrate / 2} -``` - -### BWE (Bandwidth Estimation) -``` -a=bwe.useOwdCongestionControl:1 -a=bwe.iirFilterFactor:8 -a=vqos.drc.bitrateIirFilterFactor:18 -``` - -### NACK (Retransmission) -``` -a=video.enableRtpNack:1 -a=video.rtpNackQueueLength:1024 -a=video.rtpNackQueueMaxPackets:512 -a=video.rtpNackMaxPacketCount:25 -``` - -### Packet Pacing -``` -a=packetPacing.minNumPacketsPerGroup:15 -a=packetPacing.numGroups:3 (or 5 for 60fps) -a=packetPacing.maxDelayUs:1000 -a=packetPacing.minNumPacketsFrame:10 -``` - ---- - -## 3. RTCP Statistics Collection - -### Inbound RTP Video Stats -``` -packetsReceived - Total packets received -packetsLost - Total packets lost -bytesReceived - Total bytes received -framesReceived - Total frames received -framesDecoded - Total frames decoded -framesDropped - Total frames dropped -pliCount - Picture Loss Indication count -jitter - Network jitter -jitterBufferDelay - Jitter buffer delay (ms) -totalInterFrameDelay - Total inter-frame delay -totalDecodeTime - Total decode time -frameHeight/Width - Frame dimensions -``` - -### Connection Stats -``` -currentRoundTripTime - RTT (ms) -availableOutgoingBitrate -availableIncomingBitrate -``` - -### Audio Stats -``` -audioLevel -concealedSamples -jitterBufferDelay -totalSamplesDuration -``` - ---- - -## 4. Frame Timing Metrics - -### Decode Time Tracking -- Measured from packet receive to decode completion -- Tracked per-frame in DecodeStats -- Average calculated over 1-second intervals - -### Latency Calculations -``` -Pipeline Latency = Sum(decode_times) / frame_count -Input Latency = Time from event creation to transmission -Network Latency = RTT / 2 (approximation) -Total Latency = Network + Decode + Render -``` - ---- - -## 5. Bitrate Adaptation - -### Calculation -```rust -bitrate_mbps = (bytes_received * 8) / (elapsed_seconds * 1_000_000) -``` - -### Server-Side Adaptation Triggers -- Decode time exceeds threshold -- Frame drop rate increases -- Packet loss percentage increases -- QP (Quantization Parameter) feedback - -### Packet Loss Calculation -``` -PacketLoss% = (packetsLost * 100) / (packetsLost + packetsReceived) -``` - ---- - -## 6. OSD (On-Screen Display) - -### Display Locations -- BottomLeft (default) -- BottomRight -- TopLeft -- TopRight - -### Display Information -``` -Resolution & FPS: "1920x1080 @ 60 fps" -Codec & Bitrate: "H.264 • 25.5 Mbps" -Latency: Color-coded (Green <30ms, Yellow 30-60ms, Red >60ms) -Packet Loss: Only shown if >0% (Yellow <1%, Red >=1%) -Decode & Render: "Decode: 5.2 ms • Render: 1.8 ms" -Frame Stats: "Frames: 1204 rx, 1198 dec, 6 drop" -GPU & Region: "RTX 4090 • us-east-1" -``` - ---- - -## 7. Telemetry Binary Format - -### Audio Stats (Type 4) -``` -Float64: audioLevel -Uint32: concealedSamples -Uint32: concealmentEvents -Uint32: insertedSamplesForDeceleration -Float64: jitterBufferDelay -Uint32: jitterBufferEmittedCount -Uint32: removedSamplesForAcceleration -Uint32: silentConcealedSamples -Float64: totalSamplesReceived -Float64: totalSamplesDuration -Float64: timestamp -``` - -### Video Stats (Type 3) -``` -Uint32: framesDecoded -Uint32: framesDropped -Uint32: frameHeight -Uint32: frameWidth -Uint32: framesReceived -Float64: jitterBufferDelay -Uint32: jitterBufferEmittedCount -Float64: timestamp -``` - -### Inbound RTP Stats (Type 2) -``` -Uint32: packetsReceived -Uint32: bytesReceived -Uint32: packetsLost -Float64: lastPacketReceivedTimestamp -Float64: jitter -Float64: timestamp -``` - ---- - -## 8. Quality Adjustment Parameters - -### QP (Quantization Parameter) Thresholds -``` -a=vqos.drc.minQpHeadroom:20 -a=vqos.drc.lowerQpThreshold:100 -a=vqos.drc.upperQpThreshold:200 -a=vqos.drc.minAdaptiveQpThreshold:180 -a=vqos.drc.qpMaxResThresholdAdj:4 -a=vqos.grc.qpMaxResThresholdAdj:4 -``` - -### Decode Time Thresholds -``` -a=vqos.resControl.cpmRtc.decodeTimeThresholdMs:9 -a=vqos.resControl.cpmRtc.badNwSkipFramesCount:600 -``` - ---- - -## 9. PLI (Picture Loss Indication) - -### Trigger Conditions -- 10 consecutive packets without decoded frame -- After 5+ failures, sent every 20 packets - -### Implementation -```rust -let pli = PictureLossIndication { - sender_ssrc: 0, - media_ssrc: video_ssrc, -}; -peer_connection.write_rtcp(&[Box::new(pli)]).await? -``` - ---- - -## 10. High FPS Optimizations (120+) - -### SDP Parameters -``` -a=video.encoderFeatureSetting:47 -a=video.encoderPreset:6 -a=video.fbcDynamicFpsGrabTimeoutMs:6 -a=bwe.iirFilterFactor:8 -``` - -### 240+ FPS -``` -a=video.enableNextCaptureMode:1 -a=vqos.maxStreamFpsEstimate:240 -a=video.videoSplitEncodeStripsPerFrame:3 -a=video.fbcDynamicFpsGrabTimeoutMs:18 -``` - ---- - -## 11. Comparison - -| Feature | Web Client | Official Client | OpenNow | -|---------|-----------|-----------------|---------| -| Stats Collection | Full WebRTC getStats() | Native C++ | Basic StreamStats | -| Adaptive Bitrate | Server-side | Server-side | Not implemented | -| Adaptive Resolution | DRC algorithm | DRC/DFC hybrid | Not implemented | -| Adaptive FPS | DFC for high-FPS | DFC | Not implemented | -| Telemetry | Binary format | GEAR events | Basic logging | -| RTCP Stats | Full RFC 3550 | Native RTCP | Not implemented | -| BWE Algorithm | OWD congestion | Advanced | Not implemented | -| FEC | SDP configured | Dynamic | SDP configured | -| NACK | Full support | Full support | SDP configured | - ---- - -## 12. Telemetry Events - -### Web Client Events -``` -TelemetryHandlerChanged -WorkerProblem -WebWorkerProblem -VideoPaused -MissingInboundRtpVideo -InboundVideoStats -TURN Server Details -Worker Thread Creation Failed -``` - ---- - -## 13. Control Channel Messages - -### finAck (Network Test Result) -```json -{ - "finAck": { - "downlinkBandwidth": , - "packetLoss": , - "latency": - } -} -``` - -### fin (Graceful Shutdown) -```json -{ - "fin": { - "sessionId": "", - "packetsLost": , - "packetsReceived": - } -} -``` diff --git a/opennow-streamer/reverse/video.md b/opennow-streamer/reverse/video.md deleted file mode 100644 index 4ba18d0..0000000 --- a/opennow-streamer/reverse/video.md +++ /dev/null @@ -1,276 +0,0 @@ -# GeForce NOW Video Handling - Reverse Engineering Documentation - -## 1. Video Codec Support - -### Supported Codecs -- **H.264/AVC** (primary, widest compatibility) -- **H.265/HEVC** (better compression, dynamic payload type negotiation) -- **AV1** (newest codec, RTX 40+ only, requires CUVID or QSV) - -### Codec Registration (WebRTC) -From `src/webrtc/peer.rs`: -- H.264: Standard registered via `register_default_codecs()` -- H.265: Custom registration with MIME type `"video/H265"`, clock rate 90kHz, payload type 0 (dynamic) -- AV1: Custom registration with MIME type `"video/AV1"`, clock rate 90kHz, payload type 0 (dynamic) - ---- - -## 2. RTP Packet Structure & Depacketization - -### RTP Header Format (RFC 3550) -``` -0 1 2 3 -0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -|V=2|P|X| CC |M| PT | sequence number | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| timestamp | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| synchronization source (SSRC) identifier | -+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ -``` - -Key fields: -- **V (Version)**: 2 -- **M (Marker)**: 1 on last packet of frame (critical for frame boundaries) -- **PT (Payload Type)**: 96 (H.264), 127 (H.265), 98 (AV1) -- **Timestamp**: 90 kHz clock for video - -### H.264 RTP Payload Types (RFC 6184) - -**Single NAL Unit (PT 1-23):** -- Payload is raw NAL unit without start code -- Decoder adds start code: `0x00 0x00 0x00 0x01` - -**STAP-A (Single-Time Aggregation Packet, PT 24):** -``` -[0x18] + [Size1:2B BE] + [NAL1] + [Size2:2B BE] + [NAL2] + ... -``` - -**FU-A (Fragmentation Unit, PT 28):** -``` -[0x7C] + [FU Header: S|E|R|Type] + [Fragment Payload] -FU Header: S=1 (start), E=1 (end), R=0 (reserved) -``` - -### H.265/HEVC RTP Payload (RFC 7798) - -**NAL Unit Header: 2 bytes** -``` -Byte 0: F(1) | Type(6) | LayerId_hi(1) -Byte 1: LayerId_lo(5) | TId_plus1(3) -``` - -**AP (Aggregation Packet, Type 48):** -``` -[Header:2B] + [Size1:2B BE] + [NAL1] + [Size2:2B BE] + [NAL2] + ... -``` - -**FU (Fragmentation Unit, Type 49):** -``` -[Header:2B] + [FU Header: S|E|Reserved|Type] + [Fragment Payload] -``` - -### AV1 RTP Payload (RFC 9000) - -**Aggregation Header (1 byte):** -``` -Z(1) | Y(1) | W(2) | N(1) | Reserved(3) -Z: Continuation of previous OBU fragment -Y: Last OBU fragment or complete OBU -W: Number of OBU elements -N: First packet of coded video sequence -``` - -**OBU Types:** -- 1: SEQUENCE_HEADER (critical, must precede picture data) -- 4: TILE_GROUP (contains picture data) -- 6: FRAME (complete frame) - ---- - -## 3. NAL Unit Types - -### H.264 NAL Unit Types -``` -1: Slice (Non-IDR) - P-frame/B-frame -5: IDR Slice - Keyframe -6: SEI (Supplemental Enhancement Information) -7: SPS (Sequence Parameter Set) -8: PPS (Picture Parameter Set) -24: STAP-A (aggregation) -28: FU-A (fragmentation) -``` - -### H.265/HEVC NAL Unit Types -``` -19: IDR_W_RADL - Keyframe -20: IDR_N_LP - Keyframe -32: VPS (Video Parameter Set) -33: SPS (Sequence Parameter Set) -34: PPS (Picture Parameter Set) -48: AP (Aggregation Packet) -49: FU (Fragmentation Unit) -``` - -### SPS/PPS Caching Strategy -- H.264: Type 7 (SPS) and Type 8 (PPS) cached, prepended to IDR frames (type 5) -- H.265: Types 32/33/34 (VPS/SPS/PPS) cached, prepended to IDR frames (types 19-20) -- AV1: SEQUENCE_HEADER (type 1) cached, prepended to frames missing it - ---- - -## 4. Color Space & Pixel Formats - -### YUV420P (Planar) -``` -Layout: -Y Plane: height * stride (full resolution) -U Plane: (height/2) * (stride/2) -V Plane: (height/2) * (stride/2) -``` - -### NV12 (Semi-planar) -``` -Layout: -Y Plane: height * stride_y (full resolution) -UV Plane: (height/2) * stride_uv (interleaved U,V pairs) -``` - -### YUV to RGB Conversion (BT.709) - -**Limited Range to Full Range:** -``` -y = (y_raw - 0.0625) * 1.1644 // (Y - 16/255) * (255/219) -u = (u_raw - 0.5) * 1.1384 // (U - 128/255) * (255/224) -v = (v_raw - 0.5) * 1.1384 // (V - 128/255) * (255/224) -``` - -**BT.709 Matrix:** -``` -R = Y + 1.5748 * V -G = Y - 0.1873 * U - 0.4681 * V -B = Y + 1.8556 * U -``` - -**Integer Math (Fast CPU Fallback):** -``` -R = (y + ((359 * v) >> 8)).clamp(0, 255) -G = (y - ((88 * u + 183 * v) >> 8)).clamp(0, 255) -B = (y + ((454 * u) >> 8)).clamp(0, 255) -``` - ---- - -## 5. Hardware Acceleration - -### Decoder Priority Order - -**Windows:** -1. h264_cuvid / hevc_cuvid / av1_cuvid (NVIDIA CUDA) -2. h264_qsv / hevc_qsv / av1_qsv (Intel QuickSync) -3. h264_d3d11va / hevc_d3d11va (DirectX 11) -4. Software fallback - -**macOS:** -- VideoToolbox (native macOS, NV12 output) - -**Linux:** -1. CUVID (NVIDIA) -2. VAAPI (AMD/Intel) -3. Software fallback - ---- - -## 6. Frame Timing & Synchronization - -### RTP Timestamp Calculation -``` -90 kHz clock rate: -- 60 FPS = 1500 RTP ticks per frame (90000/60) -- 120 FPS = 750 RTP ticks per frame (90000/120) -- 240 FPS = 375 RTP ticks per frame (90000/240) -``` - -### Picture Loss Indication (PLI) -From `src/webrtc/peer.rs`: -```rust -let pli = PictureLossIndication { - sender_ssrc: 0, - media_ssrc: video_ssrc, -}; -peer_connection.write_rtcp(&[Box::new(pli)]).await? -``` - -**Trigger Conditions:** -- 10 consecutive packets without decoded frame -- Additional requests every 20 packets if still failing - ---- - -## 7. Streaming Parameters (SDP) - -### Video Quality Settings -``` -a=video.packetSize:1140 -a=video.maxFPS:120 -a=video.initialBitrateKbps:25000 -a=vqos.bw.maximumBitrateKbps:50000 -a=vqos.bw.minimumBitrateKbps:5000 -``` - -### NACK Configuration -``` -a=video.enableRtpNack:1 -a=video.rtpNackQueueLength:1024 -a=video.rtpNackQueueMaxPackets:512 -a=video.rtpNackMaxPacketCount:25 -``` - -### High FPS Optimizations (120+) -``` -a=video.encoderFeatureSetting:47 -a=video.encoderPreset:6 -a=video.fbcDynamicFpsGrabTimeoutMs:6 -a=bwe.iirFilterFactor:8 -``` - -### 240+ FPS -``` -a=video.enableNextCaptureMode:1 -a=vqos.maxStreamFpsEstimate:240 -a=video.videoSplitEncodeStripsPerFrame:3 -``` - ---- - -## 8. Decode Process Flow - -``` -RTP Packet Received - ↓ -Depacketize (H.264/H.265/AV1) - ↓ -Decode Async (FFmpeg + Hardware) - ↓ -Extract Planes (YUV420P or NV12) - ↓ -Upload to GPU Textures - ↓ -GPU Shader (YUV→RGB) - ↓ -Present to Screen -``` - ---- - -## 9. Comparison - -| Feature | Web Client | OpenNow | Official Client | -|---------|-----------|---------|-----------------| -| RTP Parsing | libwebrtc | Custom Rust | libwebrtc | -| H.265 Support | Yes | Yes | Yes | -| AV1 Support | Yes | Yes (RTX 40+) | Yes | -| Hardware Decode | Browser | CUVID/QSV/VT | NVDEC | -| Color Space | BT.709 | BT.709 | BT.709 | -| Frame Format | Varies | YUV420P/NV12 | NV12 | diff --git a/opennow-streamer/reverse/win.md b/opennow-streamer/reverse/win.md deleted file mode 100644 index 0dfd907..0000000 --- a/opennow-streamer/reverse/win.md +++ /dev/null @@ -1,745 +0,0 @@ -# Windows DXVA Video Decoder Research - -This document details the video decoding architecture used by NVIDIA GeForce NOW on Windows, based on reverse engineering of the official client and implementation research. - -## Table of Contents - -1. [Architecture Overview](#architecture-overview) -2. [Supported Codecs](#supported-codecs) -3. [DXVA2 API Usage](#dxva2-api-usage) -4. [H.264 (AVC) Decoder](#h264-avc-decoder) -5. [H.265 (HEVC) Decoder](#h265-hevc-decoder) -6. [AV1 Decoder](#av1-decoder) -7. [Reference Frame Management](#reference-frame-management) -8. [Picture Order Count (POC)](#picture-order-count-poc) -9. [Bitstream Formatting](#bitstream-formatting) -10. [Performance Optimizations](#performance-optimizations) - ---- - -## Architecture Overview - -### Official GFN Client Components - -The GeForce NOW client uses several key DLLs for video decoding: - -| DLL | Purpose | Size | -|-----|---------|------| -| `Geronimo.dll` | Main video streaming/decoding engine with DXVADecoder | 28 MB | -| `Bifrost2.dll` | Network streaming protocol, codec negotiation | 21 MB | -| `NvPixels.dll` | NVIDIA pixel processing library | 2.6 MB | -| `gfnmp4mft.dll` | GFN MP4 Media Foundation Transform | 1.3 MB | -| `gfnspfbc.dll` | Sparse Pixel Block Compression | 2.5 MB | - -### Decoder Initialization Flow - -From `geronimo.log`: - -``` -1. ConfigureStreamerVideoSettings is called -2. Create DX11 decoder asynchronously -3. Initialize DX11AsyncVideoFrameRenderer -4. Set adaptive queue and timestamp rendering -5. Query DXVADecoder for codec capabilities -6. Select optimal codec based on resolution/FPS -``` - -### Key Classes Identified - -- `DXVADecoder` - Main DXVA2 hardware decoder wrapper -- `DX11AsyncVideoFrameRenderer` - Async frame rendering with Vsync tracking -- `AsyncVideoFrameRenderer` - Timestamp and queue management - ---- - -## Supported Codecs - -### Codec Capabilities (from DXVADecoder logs) - -| Codec | Profile | Max Resolution SDR | Max Resolution HDR | Max FPS | -|-------|---------|-------------------|-------------------|---------| -| H.264 | High | 3840x2160 | N/A | 120 | -| HEVC | Main | 3840x2160 | 3840x2160 | 120 | -| HEVC | Main10 | 3840x2160 | 7680x4320 | 120 | -| AV1 | Main | 3840x2160 | 3840x2160 | 60 | - -### Codec Selection Priority - -For high frame rate (240+ FPS): HEVC > H.264 -For HDR content: HEVC Main10 (10-bit) -For compatibility: H.264 (widest hardware support) - -### Decoder Quality Scores - -From client logs: -- Decoder Score: 0.9999 (excellent) -- GPU Performance: 0.9039 -- Overall Stream Quality: 0.9039 - ---- - -## DXVA2 API Usage - -### D3D11 Video API Interfaces - -The client uses these COM interfaces for hardware decoding: - -```cpp -ID3D11Device // Base D3D11 device -ID3D11VideoDevice // Video decoding device -ID3D11VideoContext // Video decoding context -ID3D11VideoDecoder // Decoder instance -ID3D11VideoDecoderOutputView // Output surface view -``` - -### Device Creation - -```cpp -// Flags for video decoding support -UINT flags = D3D11_CREATE_DEVICE_VIDEO_SUPPORT | D3D11_CREATE_DEVICE_BGRA_SUPPORT; - -D3D11CreateDevice( - nullptr, // Default adapter - D3D_DRIVER_TYPE_HARDWARE, - nullptr, - flags, - featureLevels, // 12.1, 12.0, 11.1, 11.0 - D3D11_SDK_VERSION, - &device, - &featureLevel, - &context -); - -// Enable multithread protection (required for async decode) -ID3D11Multithread* mt; -device->QueryInterface(&mt); -mt->SetMultithreadProtected(TRUE); -``` - -### Decoder Profile GUIDs - -```cpp -// H.264/AVC -D3D11_DECODER_PROFILE_H264_VLD_NOFGT = {0x1b81be68-0xa0c7-11d3-b984-00c04f2e73c5} - -// HEVC/H.265 -D3D11_DECODER_PROFILE_HEVC_VLD_MAIN = {0x5b11d51b-2f4c-4452-bcc3-09f2a1160cc0} -D3D11_DECODER_PROFILE_HEVC_VLD_MAIN10 = {0x107af0e0-ef1a-4d19-aba8-67a163073d13} - -// AV1 -D3D11_DECODER_PROFILE_AV1_VLD_PROFILE0 = {0xb8be4ccb-cf53-46ba-8d59-d6b8a6da5d2a} -``` - -### Output Texture Array (RTArray) - -The client creates a texture array for zero-copy decoding: - -```cpp -D3D11_TEXTURE2D_DESC textureDesc = { - .Width = width, - .Height = height, - .MipLevels = 1, - .ArraySize = 25, // 25 surfaces for high bitrate 4K - .Format = DXGI_FORMAT_NV12, // or P010 for HDR - .SampleDesc = { 1, 0 }, - .Usage = D3D11_USAGE_DEFAULT, - .BindFlags = D3D11_BIND_DECODER, -}; -``` - -### ConfigBitstreamRaw Settings - -The decoder configuration's `ConfigBitstreamRaw` field determines bitstream format: - -| Value | Format | Description | -|-------|--------|-------------| -| 1 | Annex-B | Include start codes (0x000001) | -| 2 | Raw NAL | No start codes, raw NAL units | - -The client prefers `ConfigBitstreamRaw=2` (short slice format) when available. - ---- - -## H.264 (AVC) Decoder - -### NAL Unit Types (ITU-T H.264 Table 7-1) - -``` -0 - Unspecified -1 - Coded slice of a non-IDR picture (P/B frame) -2 - Coded slice data partition A -3 - Coded slice data partition B -4 - Coded slice data partition C -5 - Coded slice of an IDR picture (I frame, keyframe) -6 - SEI (Supplemental Enhancement Information) -7 - SPS (Sequence Parameter Set) -8 - PPS (Picture Parameter Set) -9 - AUD (Access Unit Delimiter) -10 - End of sequence -11 - End of stream -12 - Filler data -13-23 - Reserved -24-31 - Unspecified -``` - -### SPS (Sequence Parameter Set) Structure - -Critical fields for DXVA: - -```c -struct H264_SPS { - uint8_t profile_idc; // 66=Baseline, 77=Main, 100=High - uint8_t level_idc; // 30-52 (3.0 to 5.2) - uint8_t sps_id; // 0-31 - uint8_t chroma_format_idc; // 0=mono, 1=4:2:0, 2=4:2:2, 3=4:4:4 - uint8_t bit_depth_luma; // 8-14 - uint8_t bit_depth_chroma; // 8-14 - - // POC calculation parameters - uint8_t log2_max_frame_num; // 4-16 - uint8_t pic_order_cnt_type; // 0, 1, or 2 - uint8_t log2_max_pic_order_cnt_lsb; // 4-16 (if type=0) - bool delta_pic_order_always_zero; // (if type=1) - int32_t offset_for_non_ref_pic; // (if type=1) - int32_t offset_for_top_to_bottom_field; // (if type=1) - uint8_t num_ref_frames_in_poc_cycle; // (if type=1) - int32_t offset_for_ref_frame[256]; // (if type=1) - - uint8_t max_num_ref_frames; // Max reference frames in DPB - uint16_t pic_width_in_mbs; // Width / 16 - uint16_t pic_height_in_mbs; // Height / 16 - bool frame_mbs_only; // No interlaced support if true - bool mb_adaptive_frame_field; // MBAFF - bool direct_8x8_inference; - - // Cropping - bool frame_cropping; - uint16_t crop_left, crop_right, crop_top, crop_bottom; -}; -``` - -### PPS (Picture Parameter Set) Structure - -```c -struct H264_PPS { - uint8_t pps_id; // 0-255 - uint8_t sps_id; // Reference to SPS - bool entropy_coding_mode; // 0=CAVLC, 1=CABAC - bool bottom_field_pic_order; - uint8_t num_slice_groups; - uint8_t num_ref_idx_l0_active; // Default L0 ref count - uint8_t num_ref_idx_l1_active; // Default L1 ref count - bool weighted_pred; - uint8_t weighted_bipred_idc; // 0=none, 1=explicit, 2=implicit - int8_t pic_init_qp; // -26 to +25 - int8_t pic_init_qs; - int8_t chroma_qp_index_offset; - bool deblocking_filter_control; - bool constrained_intra_pred; - bool redundant_pic_cnt_present; - bool transform_8x8_mode; - int8_t second_chroma_qp_offset; -}; -``` - -### DXVA_PicParams_H264 Structure - -This is the critical structure sent to the hardware decoder: - -```c -// Size: approximately 296 bytes -typedef struct _DXVA_PicParams_H264 { - USHORT wFrameWidthInMbsMinus1; - USHORT wFrameHeightInMbsMinus1; - - DXVA_PicEntry_H264 CurrPic; // Current picture - UCHAR num_ref_frames; - - // Bitfield union for flags - union { - struct { - USHORT field_pic_flag : 1; - USHORT MbsffFrameFlag : 1; - USHORT residual_colour_transform_flag : 1; - USHORT sp_for_switch_flag : 1; - USHORT chroma_format_idc : 2; - USHORT RefPicFlag : 1; - USHORT constrained_intra_pred_flag : 1; - USHORT weighted_pred_flag : 1; - USHORT weighted_bipred_idc : 2; - USHORT MbsConsecutiveFlag : 1; - USHORT frame_mbs_only_flag : 1; - USHORT transform_8x8_mode_flag : 1; - USHORT MinLumaBipredSize8x8Flag : 1; - USHORT IntraPicFlag : 1; - }; - USHORT wBitFields; - }; - - UCHAR bit_depth_luma_minus8; - UCHAR bit_depth_chroma_minus8; - USHORT Reserved16Bits; - UINT StatusReportFeedbackNumber; - - // Reference frame list (16 entries) - DXVA_PicEntry_H264 RefFrameList[16]; - INT CurrFieldOrderCnt[2]; // Top/Bottom field POC - INT FieldOrderCntList[16][2]; // POC for each reference - - CHAR pic_init_qp_minus26; - CHAR chroma_qp_index_offset; - CHAR second_chroma_qp_index_offset; - UCHAR ContinuationFlag; - CHAR pic_init_qs_minus26; - UCHAR num_ref_idx_l0_active_minus1; - UCHAR num_ref_idx_l1_active_minus1; - UCHAR Reserved8BitsA; - - USHORT FrameNumList[16]; // frame_num for each ref - UINT UsedForReferenceFlags; // Bitmask: which refs are used - USHORT NonExistingFrameFlags; - USHORT frame_num; - - UCHAR log2_max_frame_num_minus4; - UCHAR pic_order_cnt_type; - UCHAR log2_max_pic_order_cnt_lsb_minus4; - UCHAR delta_pic_order_always_zero_flag; - UCHAR direct_8x8_inference_flag; - UCHAR entropy_coding_mode_flag; - UCHAR pic_order_present_flag; - UCHAR num_slice_groups_minus1; - - UCHAR slice_group_map_type; - UCHAR deblocking_filter_control_present_flag; - UCHAR redundant_pic_cnt_present_flag; - UCHAR Reserved8BitsB; - USHORT slice_group_change_rate_minus1; - - // Slice group map (optional, for FMO) - UCHAR SliceGroupMap[810]; -} DXVA_PicParams_H264; - -// Picture entry format -typedef struct _DXVA_PicEntry_H264 { - union { - struct { - UCHAR Index7Bits : 7; // Surface index (0-127) - UCHAR AssociatedFlag : 1; // 1=bottom field, 0=top/frame - }; - UCHAR bPicEntry; - }; -} DXVA_PicEntry_H264; -``` - -### H.264 POC Calculation - -H.264 has THREE different POC calculation methods (unlike HEVC which has one): - -#### Type 0 (Most Common) - -```c -// Uses pic_order_cnt_lsb from slice header -int MaxPicOrderCntLsb = 1 << log2_max_pic_order_cnt_lsb; - -if (pic_order_cnt_lsb < prevPicOrderCntLsb && - (prevPicOrderCntLsb - pic_order_cnt_lsb) >= MaxPicOrderCntLsb / 2) { - PicOrderCntMsb = prevPicOrderCntMsb + MaxPicOrderCntLsb; -} else if (pic_order_cnt_lsb > prevPicOrderCntLsb && - (pic_order_cnt_lsb - prevPicOrderCntLsb) > MaxPicOrderCntLsb / 2) { - PicOrderCntMsb = prevPicOrderCntMsb - MaxPicOrderCntLsb; -} else { - PicOrderCntMsb = prevPicOrderCntMsb; -} - -TopFieldOrderCnt = PicOrderCntMsb + pic_order_cnt_lsb; -BottomFieldOrderCnt = TopFieldOrderCnt + delta_pic_order_cnt_bottom; -``` - -#### Type 1 (Explicit Offsets) - -```c -// Uses delta_pic_order_cnt[0] and delta_pic_order_cnt[1] -if (frame_num < prevFrameNum) { - FrameNumOffset = prevFrameNumOffset + MaxFrameNum; -} else { - FrameNumOffset = prevFrameNumOffset; -} - -int absFrameNum = FrameNumOffset + frame_num; -int expectedPicOrderCnt = 0; - -if (num_ref_frames_in_pic_order_cnt_cycle != 0) { - int picOrderCntCycleCnt = (absFrameNum - 1) / num_ref_frames_in_pic_order_cnt_cycle; - int frameNumInPicOrderCntCycle = (absFrameNum - 1) % num_ref_frames_in_pic_order_cnt_cycle; - - expectedPicOrderCnt = picOrderCntCycleCnt * ExpectedDeltaPerPicOrderCntCycle; - for (int i = 0; i <= frameNumInPicOrderCntCycle; i++) { - expectedPicOrderCnt += offset_for_ref_frame[i]; - } -} - -if (!nal_ref_idc) { - expectedPicOrderCnt += offset_for_non_ref_pic; -} - -TopFieldOrderCnt = expectedPicOrderCnt + delta_pic_order_cnt[0]; -BottomFieldOrderCnt = TopFieldOrderCnt + offset_for_top_to_bottom_field + delta_pic_order_cnt[1]; -``` - -#### Type 2 (Simple Sequential) - -```c -// Simplest - POC derived directly from frame_num -if (frame_num < prevFrameNum) { - FrameNumOffset = prevFrameNumOffset + MaxFrameNum; -} else { - FrameNumOffset = prevFrameNumOffset; -} - -int tempPicOrderCnt; -if (IDR) { - tempPicOrderCnt = 0; -} else if (!nal_ref_idc) { - tempPicOrderCnt = 2 * (FrameNumOffset + frame_num) - 1; -} else { - tempPicOrderCnt = 2 * (FrameNumOffset + frame_num); -} - -TopFieldOrderCnt = tempPicOrderCnt; -BottomFieldOrderCnt = tempPicOrderCnt; -``` - -### H.264 Slice Header - -Key fields for DXVA: - -```c -struct H264_SliceHeader { - uint32_t first_mb_in_slice; // Macroblock address - uint8_t slice_type; // 0=P, 1=B, 2=I, 3=SP, 4=SI - uint8_t pps_id; - uint16_t frame_num; // For reference management - - // Field/frame info (if !frame_mbs_only_flag) - bool field_pic_flag; - bool bottom_field_flag; - - // IDR specific - uint16_t idr_pic_id; // For IDR pictures - - // POC info (depends on pic_order_cnt_type) - uint16_t pic_order_cnt_lsb; // Type 0 - int32_t delta_pic_order_cnt_bottom; // Type 0, if present - int32_t delta_pic_order_cnt[2]; // Type 1 - - // Reference picture management - uint8_t num_ref_idx_l0_active; - uint8_t num_ref_idx_l1_active; - - // Reference picture list modification - bool ref_pic_list_modification_flag_l0; - bool ref_pic_list_modification_flag_l1; - - // Decoded reference picture marking - bool no_output_of_prior_pics_flag; // IDR only - bool long_term_reference_flag; // IDR only - bool adaptive_ref_pic_marking_mode_flag; -}; -``` - -### DXVA_Slice_H264_Short Structure - -```c -typedef struct _DXVA_Slice_H264_Short { - UINT BSNALunitDataLocation; // Offset in bitstream buffer - UINT SliceBytesInBuffer; // Size of this slice - USHORT wBadSliceChopping; // 0 = not chopped -} DXVA_Slice_H264_Short; -``` - ---- - -## H.265 (HEVC) Decoder - -### NAL Unit Types (ITU-T H.265 Table 7-1) - -``` -0-9 - VCL NAL units (slices) - 0 - TRAIL_N (trailing, non-reference) - 1 - TRAIL_R (trailing, reference) - 19 - IDR_W_RADL (IDR with RADL) - 20 - IDR_N_LP (IDR without leading pictures) - 21 - CRA_NUT (Clean Random Access) - -32-40 - Non-VCL NAL units - 32 - VPS_NUT (Video Parameter Set) - 33 - SPS_NUT (Sequence Parameter Set) - 34 - PPS_NUT (Picture Parameter Set) - 35 - AUD_NUT (Access Unit Delimiter) - 39 - PREFIX_SEI_NUT - 40 - SUFFIX_SEI_NUT -``` - -### HEVC VPS/SPS/PPS - -See `hevc_parser.rs` for complete structure definitions. - -### DXVA_PicParams_HEVC Structure - -```c -// Size: 232 bytes (packed) -typedef struct _DXVA_PicParams_HEVC { - USHORT PicWidthInMinCbsY; - USHORT PicHeightInMinCbsY; - - // Bitfield: chroma_format_idc, bit_depth, log2_max_poc_lsb, etc. - USHORT wFormatAndSequenceInfoFlags; - - DXVA_PicEntry_HEVC CurrPic; - UCHAR sps_max_dec_pic_buffering_minus1; - UCHAR log2_min_luma_coding_block_size_minus3; - // ... more SPS parameters ... - - UINT dwCodingParamToolFlags; // Tool flags bitfield - UINT dwCodingSettingPicturePropertyFlags; - - CHAR pps_cb_qp_offset; - CHAR pps_cr_qp_offset; - UCHAR num_tile_columns_minus1; - UCHAR num_tile_rows_minus1; - USHORT column_width_minus1[19]; - USHORT row_height_minus1[21]; - - INT CurrPicOrderCntVal; - DXVA_PicEntry_HEVC RefPicList[15]; - INT PicOrderCntValList[15]; - - UCHAR RefPicSetStCurrBefore[8]; - UCHAR RefPicSetStCurrAfter[8]; - UCHAR RefPicSetLtCurr[8]; - - UINT StatusReportFeedbackNumber; -} DXVA_PicParams_HEVC; -``` - -### HEVC POC Calculation (Single Method) - -Unlike H.264, HEVC uses only one POC calculation method: - -```c -// From ITU-T H.265 section 8.3.1 -int MaxPicOrderCntLsb = 1 << log2_max_pic_order_cnt_lsb; - -if (IDR) { - PicOrderCntMsb = 0; - PicOrderCntVal = 0; -} else { - if (pic_order_cnt_lsb < prevPicOrderCntLsb && - (prevPicOrderCntLsb - pic_order_cnt_lsb) >= MaxPicOrderCntLsb / 2) { - PicOrderCntMsb = prevPicOrderCntMsb + MaxPicOrderCntLsb; - } else if (pic_order_cnt_lsb > prevPicOrderCntLsb && - (pic_order_cnt_lsb - prevPicOrderCntLsb) > MaxPicOrderCntLsb / 2) { - PicOrderCntMsb = prevPicOrderCntMsb - MaxPicOrderCntLsb; - } else { - PicOrderCntMsb = prevPicOrderCntMsb; - } - - PicOrderCntVal = PicOrderCntMsb + pic_order_cnt_lsb; -} -``` - ---- - -## AV1 Decoder - -### AV1 OBU Types - -``` -0 - Reserved -1 - Sequence Header -2 - Temporal Delimiter -3 - Frame Header -4 - Tile Group -5 - Metadata -6 - Frame (combined header + tile group) -7 - Redundant Frame Header -8-14 - Reserved -15 - Padding -``` - -AV1 uses the `DXVA_PicParams_AV1` structure (approximately 400 bytes). - ---- - -## Reference Frame Management - -### Decoded Picture Buffer (DPB) - -The DPB stores decoded pictures that may be used as references: - -```c -struct DPB_Entry { - uint8_t surface_index; // Texture array index - int32_t poc; // Picture Order Count - uint16_t frame_num; // H.264: frame_num from slice - bool is_reference; // Can be used as reference - bool is_long_term; // Long-term reference (persistent) - bool is_output; // Has been output for display -}; -``` - -### DPB Size Limits - -| Codec | Level | Max DPB Frames | -|-------|-------|----------------| -| H.264 | 4.0 | 8 (1080p) | -| H.264 | 4.1 | 4 (1080p) or 8 (720p) | -| H.264 | 5.1 | 16 (1080p) | -| HEVC | 4.0 | 6 (1080p) | -| HEVC | 5.1 | 16 (4K) | - -### Surface Allocation Strategy - -```c -// Recommended surface count formula -surface_count = dpb_size + 3; // DPB + decode + display + margin - -// GFN client uses: -// 25 surfaces for high bitrate 4K streams -// 20 surfaces for 1080p streams -``` - -### Avoiding Surface Reuse Conflicts - -When allocating a surface for decoding, ensure it's not currently in the DPB: - -```c -uint32_t get_free_surface() { - for (int i = 0; i < surface_count; i++) { - if (!is_in_dpb(i)) { - return i; - } - } - // Evict oldest entry if all surfaces in use - evict_oldest_dpb_entry(); - return get_free_surface(); -} -``` - ---- - -## Bitstream Formatting - -### Annex-B Format - -Start codes delimit NAL units: - -``` -0x00 0x00 0x01 - 3-byte start code -0x00 0x00 0x00 0x01 - 4-byte start code (before SPS/PPS/IDR) -``` - -### Emulation Prevention - -To prevent false start codes in payload: - -``` -0x00 0x00 0x00 → 0x00 0x00 0x03 0x00 -0x00 0x00 0x01 → 0x00 0x00 0x03 0x01 -0x00 0x00 0x02 → 0x00 0x00 0x03 0x02 -0x00 0x00 0x03 → 0x00 0x00 0x03 0x03 -``` - -Parser must remove `0x03` emulation prevention bytes before parsing. - -### Buffer Alignment - -DXVA requires 128-byte alignment for bitstream buffers: - -```c -size_t aligned_size = (raw_size + 127) & ~127; -// Pad with zeros to reach aligned size -``` - ---- - -## Performance Optimizations - -### Zero-Copy Rendering - -The GFN client achieves zero-copy by: - -1. Creating output textures with `D3D11_BIND_DECODER` flag -2. Using texture arrays (RTArray) for all surfaces -3. Passing texture directly to renderer (no CPU staging) -4. Using `D3D11Texture2D` directly with wgpu/DirectX - -### Async Frame Rendering - -From client logs, `DX11AsyncVideoFrameRenderer` provides: -- Render-on-demand capability -- Vsync tracking (monitors missed Vsyncs) -- Timestamp rendering for latency measurement -- Adaptive queue management - -### Frame Timing - -The client tracks: -- Decode time (packet receive to decode complete) -- Present time (decode complete to display) -- Vsync alignment -- Queue depth - -### Multi-threaded Decoding - -```c -// Enable multithread protection -ID3D11Multithread* mt; -device->QueryInterface(&mt); -mt->SetMultithreadProtected(TRUE); -``` - -This allows: -- Decode thread: Submits decode commands -- Render thread: Reads decoded textures -- No explicit synchronization needed (D3D11 handles it) - ---- - -## Implementation Checklist - -### H.264 Decoder Implementation - -- [ ] NAL unit parser (types 1-8) -- [ ] SPS parser with all fields -- [ ] PPS parser with all fields -- [ ] Slice header parser -- [ ] POC calculation (all 3 types) -- [ ] DXVA_PicParams_H264 structure -- [ ] DXVA_Slice_H264_Short structure -- [ ] Reference frame list construction -- [ ] DPB management with frame_num -- [ ] Long-term reference support -- [ ] CABAC/CAVLC handling (implicit) - -### HEVC Decoder (Implemented) - -- [x] NAL unit parser -- [x] VPS/SPS/PPS parsing -- [x] Slice header parsing -- [x] POC calculation -- [x] DXVA_PicParams_HEVC structure -- [x] DXVA_Slice_HEVC_Short structure -- [x] Reference picture set management -- [x] DPB with POC-based ordering -- [x] Zero-copy output - ---- - -## References - -- ITU-T H.264 (ISO/IEC 14496-10) - AVC Specification -- ITU-T H.265 (ISO/IEC 23008-2) - HEVC Specification -- Microsoft DXVA 2.0 Specification -- NVIDIA GeForce NOW Client (reverse engineered logs) -- FFmpeg libavcodec (reference implementation) diff --git a/opennow-streamer/scripts/bundle-gstreamer.ps1 b/opennow-streamer/scripts/bundle-gstreamer.ps1 deleted file mode 100644 index c5bd3de..0000000 --- a/opennow-streamer/scripts/bundle-gstreamer.ps1 +++ /dev/null @@ -1,177 +0,0 @@ -# Bundle GStreamer for OpenNow Streamer -# This script copies the required GStreamer DLLs to the app's directory -# -# Prerequisites: -# 1. Install GStreamer MSVC runtime from https://gstreamer.freedesktop.org/download/ -# - Choose the MSVC 64-bit runtime installer -# - Install with default options -# -# Usage: -# .\bundle-gstreamer.ps1 -OutputDir "target\release" -# .\bundle-gstreamer.ps1 -OutputDir "target\debug" -Minimal - -param( - [Parameter(Mandatory=$true)] - [string]$OutputDir, - - [switch]$Minimal, # Only copy essential plugins for H.264 decoding - - [string]$GStreamerRoot = $env:GSTREAMER_1_0_ROOT_MSVC_X86_64 -) - -$ErrorActionPreference = "Stop" - -# Find GStreamer installation -if (-not $GStreamerRoot) { - # Try common installation paths - $possiblePaths = @( - "C:\gstreamer\1.0\msvc_x86_64", - "C:\Program Files\gstreamer\1.0\msvc_x86_64", - "C:\Program Files (x86)\gstreamer\1.0\msvc_x86_64" - ) - - foreach ($path in $possiblePaths) { - if (Test-Path $path) { - $GStreamerRoot = $path - break - } - } -} - -if (-not $GStreamerRoot -or -not (Test-Path $GStreamerRoot)) { - Write-Error @" -GStreamer not found! Please install GStreamer MSVC runtime: -1. Download from: https://gstreamer.freedesktop.org/download/ -2. Choose 'MSVC 64-bit (VS 2019, Release CRT)' runtime installer -3. Run installer with default options -4. Re-run this script - -Or set GSTREAMER_1_0_ROOT_MSVC_X86_64 environment variable to your GStreamer installation path. -"@ - exit 1 -} - -Write-Host "Using GStreamer from: $GStreamerRoot" -ForegroundColor Cyan - -# Create output directories -$gstOutputDir = Join-Path $OutputDir "gstreamer" -$gstBinDir = Join-Path $gstOutputDir "bin" -$gstPluginDir = Join-Path $gstOutputDir "lib\gstreamer-1.0" - -New-Item -ItemType Directory -Force -Path $gstBinDir | Out-Null -New-Item -ItemType Directory -Force -Path $gstPluginDir | Out-Null - -# Core GStreamer DLLs (always required) -$coreDlls = @( - "gstreamer-1.0-0.dll", - "gstbase-1.0-0.dll", - "gstvideo-1.0-0.dll", - "gstapp-1.0-0.dll", - "gstpbutils-1.0-0.dll", - "gsttag-1.0-0.dll", - "gstaudio-1.0-0.dll", - "gstrtp-1.0-0.dll", - "gstcodecparsers-1.0-0.dll", - # GLib dependencies - "glib-2.0-0.dll", - "gobject-2.0-0.dll", - "gmodule-2.0-0.dll", - "gio-2.0-0.dll", - "intl-8.dll", - "ffi-8.dll", - "pcre2-8-0.dll", - "z-1.dll", - # Other dependencies - "orc-0.4-0.dll" -) - -# Essential plugins for H.264/HEVC decoding -$essentialPlugins = @( - # Core plugins - "gstcoreelements.dll", - "gstcoretracers.dll", - "gstvideoparsersbad.dll", # h264parse, h265parse - "gstvideoconvertscale.dll", # videoconvert, videoscale - # D3D11 hardware decoding (Windows) - "gstd3d11.dll", # d3d11h264dec, d3d11h265dec, d3d11download - # Software fallback - "gstlibav.dll", # avdec_h264, avdec_h265 (FFmpeg-based) - # App source/sink - "gstapp.dll", # appsrc, appsink - # Video format handling - "gstvideorate.dll", - "gstautodetect.dll", - "gsttypefindfunctions.dll" -) - -# Additional plugins for full functionality -$additionalPlugins = @( - "gstplayback.dll", # playbin, decodebin - "gstaudioparsers.dll", - "gstaudioconvert.dll", - "gstaudioresample.dll", - "gstvolume.dll", - "gstopus.dll", # Opus audio codec - "gstrtp.dll", # RTP support - "gstrtpmanager.dll", - "gstrtsp.dll", - "gstwasapi.dll", # Windows audio - "gstwasapi2.dll" -) - -# Copy core DLLs -Write-Host "`nCopying core DLLs..." -ForegroundColor Yellow -$srcBinDir = Join-Path $GStreamerRoot "bin" - -foreach ($dll in $coreDlls) { - $src = Join-Path $srcBinDir $dll - if (Test-Path $src) { - Copy-Item $src $gstBinDir -Force - Write-Host " Copied: $dll" -ForegroundColor Green - } else { - Write-Warning " Missing: $dll" - } -} - -# Copy plugins -Write-Host "`nCopying plugins..." -ForegroundColor Yellow -$srcPluginDir = Join-Path $GStreamerRoot "lib\gstreamer-1.0" - -$pluginsToCopy = if ($Minimal) { $essentialPlugins } else { $essentialPlugins + $additionalPlugins } - -foreach ($plugin in $pluginsToCopy) { - $src = Join-Path $srcPluginDir $plugin - if (Test-Path $src) { - Copy-Item $src $gstPluginDir -Force - Write-Host " Copied: $plugin" -ForegroundColor Green - } else { - Write-Warning " Missing: $plugin" - } -} - -# Calculate total size -$totalSize = 0 -Get-ChildItem -Recurse $gstOutputDir | ForEach-Object { $totalSize += $_.Length } -$sizeMB = [math]::Round($totalSize / 1MB, 2) - -Write-Host "`nGStreamer bundle created successfully!" -ForegroundColor Cyan -Write-Host "Location: $gstOutputDir" -ForegroundColor Cyan -Write-Host "Total size: $sizeMB MB" -ForegroundColor Cyan - -# Verify essential components -Write-Host "`nVerifying essential components..." -ForegroundColor Yellow -$d3d11Plugin = Join-Path $gstPluginDir "gstd3d11.dll" -if (Test-Path $d3d11Plugin) { - Write-Host " D3D11 hardware decoder: OK" -ForegroundColor Green -} else { - Write-Warning " D3D11 hardware decoder: MISSING - H.264 will use software decoding" -} - -$libavPlugin = Join-Path $gstPluginDir "gstlibav.dll" -if (Test-Path $libavPlugin) { - Write-Host " Software decoder (libav): OK" -ForegroundColor Green -} else { - Write-Warning " Software decoder (libav): MISSING" -} - -Write-Host "`nDone! The app will automatically detect the bundled GStreamer." -ForegroundColor Green diff --git a/opennow-streamer/scripts/dev.ps1 b/opennow-streamer/scripts/dev.ps1 deleted file mode 100644 index f417cf6..0000000 --- a/opennow-streamer/scripts/dev.ps1 +++ /dev/null @@ -1 +0,0 @@ -$env:GSTREAMER_1_0_ROOT_MSVC_X86_64="C:\Program Files\gstreamer\1.0\msvc_x86_64"; $env:PATH="C:\Program Files\gstreamer\1.0\msvc_x86_64\bin;$env:PATH"; cargo run diff --git a/opennow-streamer/scripts/run.ps1 b/opennow-streamer/scripts/run.ps1 deleted file mode 100644 index 3640b5d..0000000 --- a/opennow-streamer/scripts/run.ps1 +++ /dev/null @@ -1 +0,0 @@ -$env:GSTREAMER_1_0_ROOT_MSVC_X86_64="C:\Program Files\gstreamer\1.0\msvc_x86_64"; $env:PATH="C:\Program Files\gstreamer\1.0\msvc_x86_64\bin;$env:PATH"; cargo run --release diff --git a/opennow-streamer/scripts/test-decoders.ps1 b/opennow-streamer/scripts/test-decoders.ps1 deleted file mode 100644 index 7b22313..0000000 --- a/opennow-streamer/scripts/test-decoders.ps1 +++ /dev/null @@ -1,148 +0,0 @@ -# Test Video Decoders - Checks available decoders and measures latency -# Usage: .\test-decoders.ps1 - -$gstRoot = "C:\Program Files\gstreamer\1.0\msvc_x86_64" -$env:GSTREAMER_1_0_ROOT_MSVC_X86_64 = $gstRoot -$env:PATH = "$gstRoot\bin;$env:PATH" - -Write-Host "========================================" -ForegroundColor Cyan -Write-Host " OpenNow Decoder Latency Test" -ForegroundColor Cyan -Write-Host "========================================" -ForegroundColor Cyan -Write-Host "" - -# Check GStreamer -Write-Host "Checking GStreamer installation..." -ForegroundColor Yellow -$gstInspect = "$gstRoot\bin\gst-inspect-1.0.exe" -$gstLaunch = "$gstRoot\bin\gst-launch-1.0.exe" -$gstDir = "$gstRoot\bin" - -if (Test-Path $gstDir) { - # Check if runtime is installed (not just dev files) - if (Test-Path $gstInspect) { - $version = & $gstLaunch --version 2>$null | Select-Object -First 1 - Write-Host " GStreamer Runtime found: $version" -ForegroundColor Green - Write-Host "" - - # Check D3D11 decoders - Write-Host "Available Hardware Decoders:" -ForegroundColor Yellow - - $decoders = @( - @{ Name = "d3d11h264dec"; Desc = "D3D11 H.264 HW decoder (NVIDIA/AMD/Intel)"; Latency = "Low"; Priority = 2 }, - @{ Name = "d3d11h265dec"; Desc = "D3D11 HEVC HW decoder (NVIDIA/AMD/Intel)"; Latency = "Low"; Priority = 2 }, - @{ Name = "nvh264dec"; Desc = "NVIDIA NVDEC H.264"; Latency = "Very Low*"; Priority = 1 }, - @{ Name = "nvh265dec"; Desc = "NVIDIA NVDEC HEVC"; Latency = "Very Low*"; Priority = 1 }, - @{ Name = "qsvh264dec"; Desc = "Intel QuickSync H.264"; Latency = "Low"; Priority = 3 }, - @{ Name = "qsvh265dec"; Desc = "Intel QuickSync HEVC"; Latency = "Low"; Priority = 3 }, - @{ Name = "avdec_h264"; Desc = "FFmpeg H.264 (Software)"; Latency = "Medium"; Priority = 10 }, - @{ Name = "avdec_h265"; Desc = "FFmpeg HEVC (Software)"; Latency = "Medium-High"; Priority = 10 } - ) - - $availableDecoders = @() - foreach ($dec in $decoders) { - $null = & $gstInspect $dec.Name 2>$null - if ($LASTEXITCODE -eq 0) { - Write-Host " [OK] $($dec.Name) - $($dec.Desc) [$($dec.Latency)]" -ForegroundColor Green - $availableDecoders += $dec - } else { - Write-Host " [--] $($dec.Name) - $($dec.Desc)" -ForegroundColor DarkGray - } - } - - Write-Host "" - Write-Host " * NVDEC benchmark shows higher init time but lowest sustained latency" -ForegroundColor DarkGray - Write-Host "" - - Write-Host "========================================" -ForegroundColor Cyan - Write-Host " Latency Analysis" -ForegroundColor Cyan - Write-Host "========================================" -ForegroundColor Cyan - Write-Host "" - - # Check which HW decoder is available and provide recommendations - $hasD3d11H264 = $availableDecoders | Where-Object { $_.Name -eq "d3d11h264dec" } - $hasNvdec = $availableDecoders | Where-Object { $_.Name -eq "nvh264dec" } - $hasNvdecH265 = $availableDecoders | Where-Object { $_.Name -eq "nvh265dec" } - $hasQsv = $availableDecoders | Where-Object { $_.Name -eq "qsvh264dec" } - - Write-Host "Decoder latency breakdown (typical values):" -ForegroundColor White - Write-Host "" - Write-Host " Hardware Decoders (GPU accelerated):" -ForegroundColor Yellow - Write-Host " NVIDIA NVDEC: 0.5-1.5ms decode + GPU memory (best for NVIDIA)" -ForegroundColor Gray - Write-Host " D3D11 Decoder: 1-3ms decode + CPU copy (universal fallback)" -ForegroundColor Gray - Write-Host " Intel QSV: 1-2ms decode (Intel integrated GPU)" -ForegroundColor Gray - Write-Host "" - Write-Host " Software Decoder:" -ForegroundColor Yellow - Write-Host " FFmpeg avdec: 5-15ms @ 1080p, 15-40ms @ 4K (CPU bound)" -ForegroundColor Gray - Write-Host "" - - Write-Host "========================================" -ForegroundColor Cyan - Write-Host " OpenNow Configuration" -ForegroundColor Cyan - Write-Host "========================================" -ForegroundColor Cyan - Write-Host "" - - Write-Host "Current OpenNow decoder selection:" -ForegroundColor White - Write-Host "" - - if ($hasNvdec -or $hasD3d11H264) { - Write-Host " H.264 streams: GStreamer with D3D11 hardware decode" -ForegroundColor Green - if ($hasNvdec) { - Write-Host " -> d3d11h264dec (NVDEC backend on NVIDIA GPUs)" -ForegroundColor Gray - } else { - Write-Host " -> d3d11h264dec (generic D3D11VA)" -ForegroundColor Gray - } - } else { - Write-Host " H.264 streams: Software decode (avdec_h264)" -ForegroundColor Yellow - Write-Host " -> Higher CPU usage and latency" -ForegroundColor Gray - } - - Write-Host "" - Write-Host " HEVC streams: Native DXVA decoder (built-in)" -ForegroundColor Green - Write-Host " -> Direct D3D11 Video API, lowest latency" -ForegroundColor Gray - Write-Host " -> Zero-copy GPU textures" -ForegroundColor Gray - - Write-Host "" - Write-Host "========================================" -ForegroundColor Cyan - Write-Host " Low Latency Tips" -ForegroundColor Cyan - Write-Host "========================================" -ForegroundColor Cyan - Write-Host "" - Write-Host " 1. Use HEVC codec when possible (better native support)" -ForegroundColor White - Write-Host " 2. Disable VSync in GPU control panel" -ForegroundColor White - Write-Host " 3. Use 'Balanced' stream quality (lower bitrate = faster decode)" -ForegroundColor White - Write-Host " 4. Connect via Ethernet, not WiFi" -ForegroundColor White - Write-Host " 5. Choose nearest server region" -ForegroundColor White - Write-Host "" - - # Show expected total latency - Write-Host "Expected end-to-end latency breakdown:" -ForegroundColor Yellow - Write-Host " Network RTT: 10-50ms (depends on server distance)" -ForegroundColor Gray - Write-Host " Server encode: 5-10ms" -ForegroundColor Gray - Write-Host " Decode: 1-3ms (HW) or 10-20ms (SW)" -ForegroundColor Gray - Write-Host " Render: 1-2ms" -ForegroundColor Gray - Write-Host " Display: 0-16ms (VSync) or <1ms (no VSync)" -ForegroundColor Gray - Write-Host " ----------------------------------------" -ForegroundColor Gray - Write-Host " Total: ~25-80ms typical with HW decode" -ForegroundColor White - - } else { - Write-Host " GStreamer directory found but RUNTIME not installed!" -ForegroundColor Red - Write-Host "" - Write-Host " You have the development files but not the runtime." -ForegroundColor Yellow - Write-Host "" - Write-Host " Please download and install BOTH packages from:" -ForegroundColor Yellow - Write-Host " https://gstreamer.freedesktop.org/download/" -ForegroundColor Cyan - Write-Host "" - Write-Host " 1. Runtime installer (MSVC 64-bit): gstreamer-1.0-msvc-x86_64-X.XX.X.msi" -ForegroundColor White - Write-Host " 2. Development installer (MSVC 64-bit): gstreamer-1.0-devel-msvc-x86_64-X.XX.X.msi" -ForegroundColor White - Write-Host "" - Write-Host " You currently have: Development files only" -ForegroundColor DarkGray - Write-Host " Missing: gst-inspect-1.0.exe, gst-launch-1.0.exe, etc." -ForegroundColor DarkGray - } -} else { - Write-Host " GStreamer not found at: $gstRoot" -ForegroundColor Red - Write-Host "" - Write-Host " Please install GStreamer MSVC runtime from:" -ForegroundColor Yellow - Write-Host " https://gstreamer.freedesktop.org/download/" -ForegroundColor Cyan -} - -Write-Host "" -Write-Host "========================================" -ForegroundColor Cyan -Write-Host " Done" -ForegroundColor Cyan -Write-Host "========================================" -ForegroundColor Cyan diff --git a/opennow-streamer/src/api/cloudmatch.rs b/opennow-streamer/src/api/cloudmatch.rs deleted file mode 100644 index 28e0f3d..0000000 --- a/opennow-streamer/src/api/cloudmatch.rs +++ /dev/null @@ -1,1056 +0,0 @@ -//! CloudMatch Session API -//! -//! Create and manage GFN streaming sessions. - -use anyhow::{Context, Result}; -use log::{debug, error, info, warn}; - -use super::error_codes::SessionError; -use super::GfnApiClient; -use crate::app::session::*; -use crate::app::Settings; -use crate::auth; -use crate::utils::generate_uuid; - -/// GFN client version -const GFN_CLIENT_VERSION: &str = "2.0.80.173"; - -/// User-Agent for native client -const GFN_USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 NVIDIACEFClient/HEAD/debb5919f6 GFN-PC/2.0.80.173"; - -/// Build CloudMatch zone URL -fn cloudmatch_zone_url(zone: &str) -> String { - format!("https://{}.cloudmatchbeta.nvidiagrid.net", zone) -} - -impl GfnApiClient { - /// Request a new streaming session using browser-compatible format - pub async fn create_session( - &self, - app_id: &str, - game_title: &str, - settings: &Settings, - zone: &str, - account_linked: bool, - ) -> Result { - let token = self.token().context("No access token")?; - - let device_id = generate_uuid(); - let client_id = generate_uuid(); - let sub_session_id = generate_uuid(); - - let (width, height) = settings.resolution_tuple(); - - // Get timezone offset in milliseconds - let timezone_offset_ms = chrono::Local::now().offset().local_minus_utc() as i64 * 1000; - - // Build browser-compatible request - let request = CloudMatchRequest { - session_request_data: SessionRequestData { - app_id: app_id.to_string(), // STRING format - internal_title: Some(game_title.to_string()), - available_supported_controllers: vec![], - network_test_session_id: None, - parent_session_id: None, - client_identification: "GFN-PC".to_string(), - device_hash_id: device_id.clone(), - client_version: "30.0".to_string(), - sdk_version: "1.0".to_string(), - streamer_version: 1, // NUMBER format - client_platform_name: "windows".to_string(), - client_request_monitor_settings: vec![MonitorSettings { - width_in_pixels: width, - height_in_pixels: height, - frames_per_second: settings.fps, - sdr_hdr_mode: if settings.hdr_enabled { 1 } else { 0 }, - display_data: DisplayData { - // HDR luminance values (typical HDR display capabilities) - desired_content_max_luminance: if settings.hdr_enabled { 1000 } else { 0 }, - desired_content_min_luminance: 0, - desired_content_max_frame_average_luminance: if settings.hdr_enabled { - 500 - } else { - 0 - }, - }, - dpi: 100, - }], - use_ops: true, - audio_mode: 2, // 5.1 surround - meta_data: vec![ - MetaDataEntry { - key: "SubSessionId".to_string(), - value: sub_session_id, - }, - MetaDataEntry { - key: "wssignaling".to_string(), - value: "1".to_string(), - }, - MetaDataEntry { - key: "GSStreamerType".to_string(), - value: "WebRTC".to_string(), - }, - MetaDataEntry { - key: "networkType".to_string(), - value: "Unknown".to_string(), - }, - MetaDataEntry { - key: "ClientImeSupport".to_string(), - value: "0".to_string(), - }, - MetaDataEntry { - key: "clientPhysicalResolution".to_string(), - value: format!( - "{{\"horizontalPixels\":{},\"verticalPixels\":{}}}", - width, height - ), - }, - MetaDataEntry { - key: "surroundAudioInfo".to_string(), - value: "2".to_string(), - }, - ], - sdr_hdr_mode: if settings.hdr_enabled { 1 } else { 0 }, - client_display_hdr_capabilities: if settings.hdr_enabled { - Some(HdrCapabilities { - version: 1, - hdr_edr_supported_flags_in_uint32: 1, // HDR10 support flag - static_metadata_descriptor_id: 0, - }) - } else { - None - }, - surround_audio_info: 0, - remote_controllers_bitmap: 0, - client_timezone_offset: timezone_offset_ms, - enhanced_stream_mode: 1, - app_launch_mode: 1, - secure_rtsp_supported: false, - partner_custom_data: Some("".to_string()), - account_linked, - enable_persisting_in_game_settings: true, - user_age: 26, - requested_streaming_features: Some(StreamingFeatures { - reflex: settings.fps >= 120, // Enable Reflex for high refresh rate - bit_depth: if settings.hdr_enabled { - 10 - } else { - settings.color_quality.bit_depth() - }, - cloud_gsync: false, - enabled_l4s: false, - mouse_movement_flags: 0, - true_hdr: settings.hdr_enabled, - supported_hid_devices: 0, - profile: 0, - fallback_to_logical_resolution: false, - hid_devices: None, - chroma_format: settings.color_quality.chroma_format(), - prefilter_mode: 0, - prefilter_sharpness: 0, - prefilter_noise_reduction: 0, - hud_streaming_mode: 0, - // Color space values from NVIDIA GFN client: - // SDR colorSpace = 2 (BT.709 / YCBCR_LIMITED_BT709) - // HDR colorSpace = 4 (BT.2020 / YCBCR_LIMITED_BT2020) - sdr_color_space: 2, - hdr_color_space: if settings.hdr_enabled { 4 } else { 0 }, - }), - }, - }; - - // Check if we're using an Alliance Partner - let streaming_base_url = auth::get_streaming_base_url(); - let is_alliance_partner = !streaming_base_url.contains("cloudmatchbeta.nvidiagrid.net"); - - // Build session URL - let url = if is_alliance_partner { - let base = streaming_base_url.trim_end_matches('/'); - format!( - "{}/v2/session?keyboardLayout=en-US&languageCode=en_US", - base - ) - } else { - format!( - "{}/v2/session?keyboardLayout=en-US&languageCode=en_US", - cloudmatch_zone_url(zone) - ) - }; - - info!("Creating session at: {}", url); - debug!("App ID: {}, Title: {}", app_id, game_title); - - let response = self - .client - .post(&url) - .header("User-Agent", GFN_USER_AGENT) - .header("Authorization", format!("GFNJWT {}", token)) - .header("Content-Type", "application/json") - .header("Origin", "https://play.geforcenow.com") - .header("Referer", "https://play.geforcenow.com/") - // NV-* headers - .header("nv-browser-type", "CHROME") - .header("nv-client-id", &client_id) - .header("nv-client-streamer", "NVIDIA-CLASSIC") - .header("nv-client-type", "NATIVE") - .header("nv-client-version", GFN_CLIENT_VERSION) - .header("nv-device-make", "UNKNOWN") - .header("nv-device-model", "UNKNOWN") - .header("nv-device-os", "WINDOWS") - .header("nv-device-type", "DESKTOP") - .header("x-device-id", &device_id) - .json(&request) - .send() - .await - .context("Session request failed")?; - - let status = response.status(); - let response_text = response.text().await.context("Failed to read response")?; - - debug!( - "CloudMatch response ({} bytes): {}", - response_text.len(), - &response_text[..response_text.len().min(500)] - ); - - if !status.is_success() { - // Parse error response for user-friendly message - let session_error = SessionError::from_response(status.as_u16(), &response_text); - error!( - "CloudMatch session error: {} - {} (code: {}, unified: {:?})", - session_error.title, - session_error.description, - session_error.gfn_error_code, - session_error.unified_error_code - ); - - return Err(anyhow::anyhow!( - "{}: {}", - session_error.title, - session_error.description - )); - } - - let api_response: CloudMatchResponse = - serde_json::from_str(&response_text).context("Failed to parse CloudMatch response")?; - - if api_response.request_status.status_code != 1 { - // Parse error for user-friendly message - let session_error = SessionError::from_response(200, &response_text); - error!( - "CloudMatch API error: {} - {} (statusCode: {}, unified: {})", - session_error.title, - session_error.description, - api_response.request_status.status_code, - api_response.request_status.unified_error_code - ); - - return Err(anyhow::anyhow!( - "{}: {}", - session_error.title, - session_error.description - )); - } - - let session_data = api_response.session; - info!( - "Session allocated: {} (status: {})", - session_data.session_id, session_data.status - ); - - // Determine session state - let state = Self::parse_session_state(&session_data); - - // Extract connection info - let server_ip = session_data.streaming_server_ip().unwrap_or_default(); - let signaling_path = session_data.signaling_url(); - - // Build full signaling URL - let signaling_url = signaling_path - .map(|path| { - if path.starts_with("wss://") || path.starts_with("rtsps://") { - // Already a full URL - Self::build_signaling_url(&path, &server_ip) - } else if path.starts_with('/') { - // Path like /nvst/ - format!("wss://{}:443{}", server_ip, path) - } else { - format!("wss://{}:443/nvst/", server_ip) - } - }) - .or_else(|| { - if !server_ip.is_empty() { - Some(format!("wss://{}:443/nvst/", server_ip)) - } else { - None - } - }); - - info!( - "Stream server: {}, signaling: {:?}", - server_ip, signaling_url - ); - - // Extract ICE servers and media info before moving other fields - let ice_servers = session_data.ice_servers(); - let media_connection_info = session_data.media_connection_info(); - - // Debug: log connection info - if let Some(ref conns) = session_data.connection_info { - for conn in conns { - info!( - "ConnectionInfo: ip={:?} port={} usage={} protocol={}", - conn.ip, conn.port, conn.usage, conn.protocol - ); - } - } else { - info!("No connection_info in session response"); - } - info!("Media connection info: {:?}", media_connection_info); - - // Extract ads info if present - let ads_required = session_data.session_ads_required; - let ads_info = session_data.session_ads.as_ref().map(|ads| SessionAdsInfo { - video_url: ads - .get("videoUrl") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()), - duration_secs: ads.get("duration").and_then(|v| v.as_u64()).unwrap_or(120) as u32, - completion_url: ads - .get("completionUrl") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()), - raw_config: Some(ads.clone()), - }); - - // If ads are required and we're in queue, switch to WatchingAds state - let final_state = if ads_required { - info!("Session requires ads (free tier user)"); - match state { - SessionState::InQueue { .. } | SessionState::Launching => { - let duration = ads_info.as_ref().map(|a| a.duration_secs).unwrap_or(120); - SessionState::WatchingAds { - remaining_secs: duration, - total_secs: duration, - } - } - other => other, - } - } else { - state - }; - - Ok(SessionInfo { - session_id: session_data.session_id, - server_ip, - zone: zone.to_string(), - state: final_state, - gpu_type: session_data.gpu_type, - signaling_url, - ice_servers, - media_connection_info, - ads_required, - ads_info, - }) - } - - /// Build signaling WebSocket URL from raw path/URL - fn build_signaling_url(raw: &str, server_ip: &str) -> String { - if raw.starts_with("rtsps://") || raw.starts_with("rtsp://") { - // Extract hostname from RTSP URL - let host = raw - .strip_prefix("rtsps://") - .or_else(|| raw.strip_prefix("rtsp://")) - .and_then(|s| s.split(':').next()) - .filter(|h| !h.is_empty() && !h.starts_with('.')); - - if let Some(h) = host { - format!("wss://{}/nvst/", h) - } else { - // Malformed URL, use server IP - format!("wss://{}:443/nvst/", server_ip) - } - } else if raw.starts_with("wss://") { - raw.to_string() - } else if raw.starts_with('/') { - format!("wss://{}:443{}", server_ip, raw) - } else { - format!("wss://{}:443/nvst/", server_ip) - } - } - - /// Poll session status until ready - pub async fn poll_session( - &self, - session_id: &str, - zone: &str, - server_ip: Option<&str>, - ) -> Result { - let token = self.token().context("No access token")?; - - let device_id = generate_uuid(); - let client_id = generate_uuid(); - - // Check if we're using an Alliance Partner - let streaming_base_url = auth::get_streaming_base_url(); - let is_alliance_partner = !streaming_base_url.contains("cloudmatchbeta.nvidiagrid.net"); - - // Build polling URL - prefer server IP if available - let poll_base = if is_alliance_partner { - streaming_base_url.trim_end_matches('/').to_string() - } else if let Some(ip) = server_ip { - format!("https://{}", ip) - } else { - cloudmatch_zone_url(zone) - }; - - let url = format!("{}/v2/session/{}", poll_base, session_id); - - debug!("Polling session at: {}", url); - - let response = self - .client - .get(&url) - .header("User-Agent", GFN_USER_AGENT) - .header("Authorization", format!("GFNJWT {}", token)) - .header("Content-Type", "application/json") - // NV-* headers - .header("nv-client-id", &client_id) - .header("nv-client-streamer", "NVIDIA-CLASSIC") - .header("nv-client-type", "NATIVE") - .header("nv-client-version", GFN_CLIENT_VERSION) - .header("nv-device-os", "WINDOWS") - .header("nv-device-type", "DESKTOP") - .header("x-device-id", &device_id) - .send() - .await - .context("Poll request failed")?; - - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - return Err(anyhow::anyhow!("Poll failed: {} - {}", status, body)); - } - - let response_text = response - .text() - .await - .context("Failed to read poll response")?; - - let poll_response: CloudMatchResponse = - serde_json::from_str(&response_text).context("Failed to parse poll response")?; - - if poll_response.request_status.status_code != 1 { - // Parse error for user-friendly message - let session_error = SessionError::from_response(200, &response_text); - error!( - "Poll failed: {} - {} (statusCode: {}, unified: {})", - session_error.title, - session_error.description, - poll_response.request_status.status_code, - poll_response.request_status.unified_error_code - ); - return Err(anyhow::anyhow!( - "Poll failed: {} - {}", - session_error.title, - session_error.description - )); - } - - let session_data = poll_response.session; - let state = Self::parse_session_state(&session_data); - - let server_ip = session_data.streaming_server_ip().unwrap_or_default(); - let signaling_path = session_data.signaling_url(); - - // Build full signaling URL - let signaling_url = signaling_path - .map(|path| Self::build_signaling_url(&path, &server_ip)) - .or_else(|| { - if !server_ip.is_empty() { - Some(format!("wss://{}:443/nvst/", server_ip)) - } else { - None - } - }); - - // Extract ICE servers and media info before moving other fields - let ice_servers = session_data.ice_servers(); - let media_connection_info = session_data.media_connection_info(); - - // Debug: log connection info in poll response - if let Some(ref conns) = session_data.connection_info { - for conn in conns { - info!( - "Poll ConnectionInfo: ip={:?} port={} usage={} protocol={}", - conn.ip, conn.port, conn.usage, conn.protocol - ); - } - } - if media_connection_info.is_some() { - info!("Poll media connection info: {:?}", media_connection_info); - } - - // Extract ads info if present - let ads_required = session_data.session_ads_required; - let ads_info = session_data.session_ads.as_ref().map(|ads| SessionAdsInfo { - video_url: ads - .get("videoUrl") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()), - duration_secs: ads.get("duration").and_then(|v| v.as_u64()).unwrap_or(120) as u32, - completion_url: ads - .get("completionUrl") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()), - raw_config: Some(ads.clone()), - }); - - // If ads are required and we're in queue, switch to WatchingAds state - let final_state = if ads_required { - match state { - SessionState::InQueue { .. } | SessionState::Launching => { - let duration = ads_info.as_ref().map(|a| a.duration_secs).unwrap_or(120); - SessionState::WatchingAds { - remaining_secs: duration, - total_secs: duration, - } - } - other => other, - } - } else { - state - }; - - Ok(SessionInfo { - session_id: session_data.session_id, - server_ip, - zone: zone.to_string(), - state: final_state, - gpu_type: session_data.gpu_type, - signaling_url, - ice_servers, - media_connection_info, - ads_required, - ads_info, - }) - } - - /// Stop a streaming session - pub async fn stop_session( - &self, - session_id: &str, - zone: &str, - server_ip: Option<&str>, - ) -> Result<()> { - let token = self.token().context("No access token")?; - - let device_id = generate_uuid(); - - // Check if we're using an Alliance Partner - let streaming_base_url = auth::get_streaming_base_url(); - let is_alliance_partner = !streaming_base_url.contains("cloudmatchbeta.nvidiagrid.net"); - - // Build delete URL - let delete_base = if is_alliance_partner { - streaming_base_url.trim_end_matches('/').to_string() - } else if let Some(ip) = server_ip { - format!("https://{}", ip) - } else { - cloudmatch_zone_url(zone) - }; - - let url = format!("{}/v2/session/{}", delete_base, session_id); - - info!("Stopping session at: {}", url); - - let response = self - .client - .delete(&url) - .header("User-Agent", GFN_USER_AGENT) - .header("Authorization", format!("GFNJWT {}", token)) - .header("Content-Type", "application/json") - .header("x-device-id", &device_id) - .send() - .await - .context("Stop session request failed")?; - - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - warn!("Session stop returned: {} - {}", status, body); - } - - info!("Session stopped: {}", session_id); - Ok(()) - } - - /// Parse session state from CloudMatch response - fn parse_session_state(session_data: &CloudMatchSession) -> SessionState { - // Status 2 = ready for streaming - if session_data.status == 2 { - return SessionState::Ready; - } - - // Status 3 = already streaming - if session_data.status == 3 { - return SessionState::Streaming; - } - - // Check seat setup info for detailed states - if let Some(ref seat_info) = session_data.seat_setup_info { - match seat_info.seat_setup_step { - 0 => return SessionState::Connecting, - 1 => { - // In queue - show position - return SessionState::InQueue { - position: seat_info.queue_position.max(0) as u32, - eta_secs: (seat_info.seat_setup_eta / 1000).max(0) as u32, - }; - } - 5 => return SessionState::CleaningUp, - 6 => return SessionState::WaitingForStorage, - _ => { - // Other steps = general launching/configuring - if seat_info.seat_setup_step > 0 { - return SessionState::Launching; - } - } - } - } - - // Status 1 = setting up - if session_data.status == 1 { - return SessionState::Launching; - } - - // Error states - if session_data.status <= 0 || session_data.error_code != 0 { - return SessionState::Error(format!( - "Error code: {} (status: {})", - session_data.error_code, session_data.status - )); - } - - SessionState::Launching - } - - /// Get active sessions - /// Returns list of sessions with status 2 (Ready) or 3 (Streaming) - pub async fn get_active_sessions(&self) -> Result> { - let token = self.token().context("No access token")?; - - let device_id = generate_uuid(); - let client_id = generate_uuid(); - - // Get streaming base URL - let streaming_base_url = auth::get_streaming_base_url(); - let session_url = format!("{}/v2/session", streaming_base_url.trim_end_matches('/')); - - info!("Checking for active sessions at: {}", session_url); - - let response = self - .client - .get(&session_url) - .header("User-Agent", GFN_USER_AGENT) - .header("Authorization", format!("GFNJWT {}", token)) - .header("Content-Type", "application/json") - .header("nv-client-id", &client_id) - .header("nv-client-streamer", "NVIDIA-CLASSIC") - .header("nv-client-type", "NATIVE") - .header("nv-client-version", GFN_CLIENT_VERSION) - .header("nv-device-os", "WINDOWS") - .header("nv-device-type", "DESKTOP") - .header("x-device-id", &device_id) - .send() - .await - .context("Failed to get sessions")?; - - let status = response.status(); - let response_text = response.text().await.context("Failed to read response")?; - - if !status.is_success() { - warn!( - "Get sessions failed: {} - {}", - status, - &response_text[..response_text.len().min(200)] - ); - return Ok(vec![]); - } - - info!("Active sessions response: {}", response_text); - - let sessions_response: GetSessionsResponse = - serde_json::from_str(&response_text).context("Failed to parse sessions response")?; - - if sessions_response.request_status.status_code != 1 { - warn!( - "Get sessions API error: {:?}", - sessions_response.request_status.status_description - ); - return Ok(vec![]); - } - - info!( - "Found {} session(s) from API", - sessions_response.sessions.len() - ); - - let active_sessions: Vec = sessions_response - .sessions - .into_iter() - .filter(|s| { - debug!("Session {} has status {}", s.session_id, s.status); - s.status == 2 || s.status == 3 - }) - .map(|s| { - let app_id = s - .session_request_data - .as_ref() - .map(|d| d.get_app_id()) - .unwrap_or(0); - - let server_ip = s.session_control_info.as_ref().and_then(|c| c.ip.clone()); - - debug!( - "Session {} control info: {:?}", - s.session_id, s.session_control_info - ); - debug!( - "Session {} server_ip extracted: {:?}", - s.session_id, server_ip - ); - - let signaling_url = s - .connection_info - .as_ref() - .and_then(|conns| conns.iter().find(|c| c.usage == 14)) - .and_then(|conn| conn.ip.as_ref().map(|ip| format!("wss://{}:443/nvst/", ip))) - .or_else(|| { - server_ip - .as_ref() - .map(|ip| format!("wss://{}:443/nvst/", ip)) - }); - - let (resolution, fps) = s - .monitor_settings - .as_ref() - .and_then(|ms| ms.first()) - .map(|m| { - ( - Some(format!( - "{}x{}", - m.width_in_pixels.unwrap_or(0), - m.height_in_pixels.unwrap_or(0) - )), - m.frames_per_second, - ) - }) - .unwrap_or((None, None)); - - ActiveSessionInfo { - session_id: s.session_id, - app_id, - gpu_type: s.gpu_type, - status: s.status, - server_ip, - signaling_url, - resolution, - fps, - } - }) - .collect(); - - info!("Found {} active session(s)", active_sessions.len()); - Ok(active_sessions) - } - - /// Claim/Resume an existing session - /// Required before connecting to an existing session - pub async fn claim_session( - &self, - session_id: &str, - server_ip: &str, - app_id: &str, - settings: &Settings, - ) -> Result { - let token = self.token().context("No access token")?; - - let device_id = generate_uuid(); - let client_id = generate_uuid(); - let sub_session_id = generate_uuid(); - - let (width, height) = settings.resolution_tuple(); - - let timezone_offset_ms = chrono::Local::now().offset().local_minus_utc() as i64 * 1000; - - let claim_url = format!( - "https://{}/v2/session/{}?keyboardLayout=en-US&languageCode=en_US", - server_ip, session_id - ); - - info!("Claiming session: {} at {}", session_id, claim_url); - - // Build HDR capabilities JSON if enabled - // Based on NVIDIA GFN client analysis: - // - hdrEdrSupportedFlagsInUint32: 1 = HDR10 supported - // - staticMetadataDescriptorId: 0 = Type 1 (Static Metadata) - let hdr_capabilities = if settings.hdr_enabled { - serde_json::json!({ - "version": 1, - "hdrEdrSupportedFlagsInUint32": 3, // 1=HDR10, 2=EDR, 3=both - "staticMetadataDescriptorId": 0, - "displayData": { - "maxLuminance": 1000, - "minLuminance": 0.01, - "maxFrameAverageLuminance": 500 - } - }) - } else { - serde_json::Value::Null - }; - - // HDR colorspace values from NVIDIA GFN client: - // SDR colorSpace = 2 (BT.709) - // HDR colorSpace = 4 (BT.2020) - let sdr_color_space = 2; // BT.709 / YCBCR_LIMITED_BT709 - let hdr_color_space = 4; // BT.2020 / YCBCR_LIMITED_BT2020 - - let resume_payload = serde_json::json!({ - "action": 2, - "data": "RESUME", - "sessionRequestData": { - "audioMode": 2, - "remoteControllersBitmap": 0, - "sdrHdrMode": if settings.hdr_enabled { 1 } else { 0 }, - "networkTestSessionId": null, - "availableSupportedControllers": [], - "clientVersion": "30.0", - "deviceHashId": device_id, - "internalTitle": null, - "clientPlatformName": "windows", - "metaData": [ - {"key": "SubSessionId", "value": sub_session_id}, - {"key": "wssignaling", "value": "1"}, - {"key": "GSStreamerType", "value": "WebRTC"}, - {"key": "networkType", "value": "Unknown"}, - {"key": "ClientImeSupport", "value": "0"}, - {"key": "clientPhysicalResolution", "value": format!("{{\"horizontalPixels\":{},\"verticalPixels\":{}}}", width, height)}, - {"key": "surroundAudioInfo", "value": "2"} - ], - "surroundAudioInfo": 0, - "clientTimezoneOffset": timezone_offset_ms, - "clientIdentification": "GFN-PC", - "parentSessionId": null, - "appId": app_id, - "streamerVersion": 1, - "clientRequestMonitorSettings": [{ - "widthInPixels": width, - "heightInPixels": height, - "framesPerSecond": settings.fps, - "sdrHdrMode": if settings.hdr_enabled { 1 } else { 0 }, - "displayData": { - "desiredContentMaxLuminance": if settings.hdr_enabled { 1000 } else { 0 }, - "desiredContentMinLuminance": 0, - "desiredContentMaxFrameAverageLuminance": if settings.hdr_enabled { 500 } else { 0 } - }, - "dpi": 0 - }], - "appLaunchMode": 1, - "sdkVersion": "1.0", - "enhancedStreamMode": 1, - "useOps": true, - "clientDisplayHdrCapabilities": if settings.hdr_enabled { hdr_capabilities } else { serde_json::Value::Null }, - "accountLinked": true, - "partnerCustomData": "", - "enablePersistingInGameSettings": true, - "secureRTSPSupported": false, - "userAge": 26, - "requestedStreamingFeatures": { - "reflex": settings.fps >= 120, - "bitDepth": if settings.hdr_enabled { 10 } else { 0 }, - "cloudGsync": false, - "enabledL4S": false, - "profile": 0, - "fallbackToLogicalResolution": false, - "chromaFormat": settings.color_quality.chroma_format(), - "prefilterMode": 0, - "hudStreamingMode": 0 - } - }, - "metaData": [] - }); - - let response = self - .client - .put(&claim_url) - .header("User-Agent", GFN_USER_AGENT) - .header("Authorization", format!("GFNJWT {}", token)) - .header("Content-Type", "application/json") - .header("Origin", "https://play.geforcenow.com") - .header("Referer", "https://play.geforcenow.com/") - .header("nv-client-id", &client_id) - .header("nv-client-streamer", "NVIDIA-CLASSIC") - .header("nv-client-type", "NATIVE") - .header("nv-client-version", GFN_CLIENT_VERSION) - .header("nv-device-os", "WINDOWS") - .header("nv-device-type", "DESKTOP") - .header("x-device-id", &device_id) - .json(&resume_payload) - .send() - .await - .context("Claim session request failed")?; - - let status = response.status(); - let response_text = response - .text() - .await - .context("Failed to read claim response")?; - - if !status.is_success() { - error!("Claim session failed response: {}", response_text); - return Err(anyhow::anyhow!( - "Claim session failed: {} - {}", - status, - &response_text[..response_text.len().min(1000)] - )); - } - - let api_response: CloudMatchResponse = - serde_json::from_str(&response_text).context("Failed to parse claim response")?; - - if api_response.request_status.status_code != 1 { - // Parse error for user-friendly message - let session_error = SessionError::from_response(200, &response_text); - error!( - "Claim failed: {} - {} (statusCode: {})", - session_error.title, - session_error.description, - api_response.request_status.status_code - ); - return Err(anyhow::anyhow!( - "Claim failed: {} - {}", - session_error.title, - session_error.description - )); - } - - info!("Session claimed! Polling until ready..."); - - let get_url = format!("https://{}/v2/session/{}", server_ip, session_id); - - for attempt in 1..=60 { - if attempt > 1 { - tokio::time::sleep(tokio::time::Duration::from_millis(1000)).await; - } - - let poll_response = self - .client - .get(&get_url) - .header("User-Agent", GFN_USER_AGENT) - .header("Authorization", format!("GFNJWT {}", token)) - .header("Content-Type", "application/json") - .header("nv-client-id", &client_id) - .header("nv-client-streamer", "NVIDIA-CLASSIC") - .header("nv-client-type", "NATIVE") - .header("nv-client-version", GFN_CLIENT_VERSION) - .header("nv-device-os", "WINDOWS") - .header("nv-device-type", "DESKTOP") - .header("x-device-id", &device_id) - .send() - .await - .context("Poll claim request failed")?; - - if !poll_response.status().is_success() { - continue; - } - - let poll_text = poll_response - .text() - .await - .context("Failed to read poll response")?; - - let poll_api_response: CloudMatchResponse = match serde_json::from_str(&poll_text) { - Ok(r) => r, - Err(_) => continue, - }; - - let session_data = poll_api_response.session; - debug!( - "Claim poll attempt {}: status {}", - attempt, session_data.status - ); - - if session_data.status == 2 || session_data.status == 3 { - info!("Session ready after claim! Status: {}", session_data.status); - - let state = Self::parse_session_state(&session_data); - let server_ip_final = session_data - .streaming_server_ip() - .unwrap_or_else(|| server_ip.to_string()); - let signaling_path = session_data.signaling_url(); - - let signaling_url = signaling_path - .map(|path| Self::build_signaling_url(&path, &server_ip_final)) - .or_else(|| Some(format!("wss://{}:443/nvst/", server_ip_final))); - - let ice_servers = session_data.ice_servers(); - let media_connection_info = session_data.media_connection_info(); - - // Extract ads info if present - let ads_required = session_data.session_ads_required; - let ads_info = session_data.session_ads.as_ref().map(|ads| SessionAdsInfo { - video_url: ads - .get("videoUrl") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()), - duration_secs: ads.get("duration").and_then(|v| v.as_u64()).unwrap_or(120) - as u32, - completion_url: ads - .get("completionUrl") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()), - raw_config: Some(ads.clone()), - }); - - // If ads are required and we're in queue, switch to WatchingAds state - let final_state = if ads_required { - match state { - SessionState::InQueue { .. } | SessionState::Launching => { - let duration = - ads_info.as_ref().map(|a| a.duration_secs).unwrap_or(120); - SessionState::WatchingAds { - remaining_secs: duration, - total_secs: duration, - } - } - other => other, - } - } else { - state - }; - - return Ok(SessionInfo { - session_id: session_data.session_id, - server_ip: server_ip_final, - zone: String::new(), - state: final_state, - gpu_type: session_data.gpu_type, - signaling_url, - ice_servers, - media_connection_info, - ads_required, - ads_info, - }); - } - - if session_data.status != 6 { - break; - } - } - - Err(anyhow::anyhow!( - "Session did not become ready after claiming" - )) - } -} diff --git a/opennow-streamer/src/api/error_codes.rs b/opennow-streamer/src/api/error_codes.rs deleted file mode 100644 index 27bb791..0000000 --- a/opennow-streamer/src/api/error_codes.rs +++ /dev/null @@ -1,793 +0,0 @@ -//! GFN CloudMatch Error Codes -//! -//! Error code mappings extracted from the official GFN web client. -//! These provide user-friendly error messages for session failures. - -use once_cell::sync::Lazy; -use std::collections::HashMap; - -/// GFN Session Error Codes from official client -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[repr(i64)] -pub enum GfnErrorCode { - // Success codes - Success = 15859712, - - // Client-side errors (3237085xxx - 3237093xxx) - InvalidOperation = 3237085186, - NetworkError = 3237089282, - GetActiveSessionServerError = 3237089283, - AuthTokenNotUpdated = 3237093377, - SessionFinishedState = 3237093378, - ResponseParseFailure = 3237093379, - InvalidServerResponse = 3237093381, - PutOrPostInProgress = 3237093382, - GridServerNotInitialized = 3237093383, - DOMExceptionInSessionControl = 3237093384, - InvalidAdStateTransition = 3237093386, - AuthTokenUpdateTimeout = 3237093387, - - // Server error codes (base 3237093632 + statusCode) - SessionServerErrorBegin = 3237093632, - RequestForbidden = 3237093634, // statusCode 2 - ServerInternalTimeout = 3237093635, // statusCode 3 - ServerInternalError = 3237093636, // statusCode 4 - ServerInvalidRequest = 3237093637, // statusCode 5 - ServerInvalidRequestVersion = 3237093638, // statusCode 6 - SessionListLimitExceeded = 3237093639, // statusCode 7 - InvalidRequestDataMalformed = 3237093640, // statusCode 8 - InvalidRequestDataMissing = 3237093641, // statusCode 9 - RequestLimitExceeded = 3237093642, // statusCode 10 - SessionLimitExceeded = 3237093643, // statusCode 11 - InvalidRequestVersionOutOfDate = 3237093644, // statusCode 12 - SessionEntitledTimeExceeded = 3237093645, // statusCode 13 - AuthFailure = 3237093646, // statusCode 14 - InvalidAuthenticationMalformed = 3237093647, // statusCode 15 - InvalidAuthenticationExpired = 3237093648, // statusCode 16 - InvalidAuthenticationNotFound = 3237093649, // statusCode 17 - EntitlementFailure = 3237093650, // statusCode 18 - InvalidAppIdNotAvailable = 3237093651, // statusCode 19 - InvalidAppIdNotFound = 3237093652, // statusCode 20 - InvalidSessionIdMalformed = 3237093653, // statusCode 21 - InvalidSessionIdNotFound = 3237093654, // statusCode 22 - EulaUnAccepted = 3237093655, // statusCode 23 - MaintenanceStatus = 3237093656, // statusCode 24 - ServiceUnAvailable = 3237093657, // statusCode 25 - SteamGuardRequired = 3237093658, // statusCode 26 - SteamLoginRequired = 3237093659, // statusCode 27 - SteamGuardInvalid = 3237093660, // statusCode 28 - SteamProfilePrivate = 3237093661, // statusCode 29 - InvalidCountryCode = 3237093662, // statusCode 30 - InvalidLanguageCode = 3237093663, // statusCode 31 - MissingCountryCode = 3237093664, // statusCode 32 - MissingLanguageCode = 3237093665, // statusCode 33 - SessionNotPaused = 3237093666, // statusCode 34 - EmailNotVerified = 3237093667, // statusCode 35 - InvalidAuthenticationUnsupportedProtocol = 3237093668, // statusCode 36 - InvalidAuthenticationUnknownToken = 3237093669, // statusCode 37 - InvalidAuthenticationCredentials = 3237093670, // statusCode 38 - SessionNotPlaying = 3237093671, // statusCode 39 - InvalidServiceResponse = 3237093672, // statusCode 40 - AppPatching = 3237093673, // statusCode 41 - GameNotFound = 3237093674, // statusCode 42 - NotEnoughCredits = 3237093675, // statusCode 43 - InvitationOnlyRegistration = 3237093676, // statusCode 44 - RegionNotSupportedForRegistration = 3237093677, // statusCode 45 - SessionTerminatedByAnotherClient = 3237093678, // statusCode 46 - DeviceIdAlreadyUsed = 3237093679, // statusCode 47 - ServiceNotExist = 3237093680, // statusCode 48 - SessionExpired = 3237093681, // statusCode 49 - SessionLimitPerDeviceReached = 3237093682, // statusCode 50 - ForwardingZoneOutOfCapacity = 3237093683, // statusCode 51 - RegionNotSupportedIndefinitely = 3237093684, // statusCode 52 - RegionBanned = 3237093685, // statusCode 53 - RegionOnHoldForFree = 3237093686, // statusCode 54 - RegionOnHoldForPaid = 3237093687, // statusCode 55 - AppMaintenanceStatus = 3237093688, // statusCode 56 - ResourcePoolNotConfigured = 3237093689, // statusCode 57 - InsufficientVmCapacity = 3237093690, // statusCode 58 - InsufficientRouteCapacity = 3237093691, // statusCode 59 - InsufficientScratchSpaceCapacity = 3237093692, // statusCode 60 - RequiredSeatInstanceTypeNotSupported = 3237093693, // statusCode 61 - ServerSessionQueueLengthExceeded = 3237093694, // statusCode 62 - RegionNotSupportedForStreaming = 3237093695, // statusCode 63 - SessionForwardRequestAllocationTimeExpired = 3237093696, // statusCode 64 - SessionForwardGameBinariesNotAvailable = 3237093697, // statusCode 65 - GameBinariesNotAvailableInRegion = 3237093698, // statusCode 66 - UekRetrievalFailed = 3237093699, // statusCode 67 - EntitlementFailureForResource = 3237093700, // statusCode 68 - SessionInQueueAbandoned = 3237093701, // statusCode 69 - MemberTerminated = 3237093702, // statusCode 70 - SessionRemovedFromQueueMaintenance = 3237093703, // statusCode 71 - ZoneMaintenanceStatus = 3237093704, // statusCode 72 - GuestModeCampaignDisabled = 3237093705, // statusCode 73 - RegionNotSupportedAnonymousAccess = 3237093706, // statusCode 74 - InstanceTypeNotSupportedInSingleRegion = 3237093707, // statusCode 75 - InvalidZoneForQueuedSession = 3237093710, // statusCode 78 - SessionWaitingAdsTimeExpired = 3237093711, // statusCode 79 - UserCancelledWatchingAds = 3237093712, // statusCode 80 - StreamingNotAllowedInLimitedMode = 3237093713, // statusCode 81 - ForwardRequestJPMFailed = 3237093714, // statusCode 82 - MaxSessionNumberLimitExceeded = 3237093715, // statusCode 83 - GuestModePartnerCapacityDisabled = 3237093716, // statusCode 84 - SessionRejectedNoCapacity = 3237093717, // statusCode 85 - SessionInsufficientPlayabilityLevel = 3237093718, // statusCode 86 - ForwardRequestLOFNFailed = 3237093719, // statusCode 87 - InvalidTransportRequest = 3237093720, // statusCode 88 - UserStorageNotAvailable = 3237093721, // statusCode 89 - GfnStorageNotAvailable = 3237093722, // statusCode 90 - SessionServerErrorEnd = 3237093887, - - // Session setup cancelled - SessionSetupCancelled = 15867905, - SessionSetupCancelledDuringQueuing = 15867906, - RequestCancelled = 15867907, - SystemSleepDuringSessionSetup = 15867909, - NoInternetDuringSessionSetup = 15868417, - - // Network errors (3237101xxx) - SocketError = 3237101580, - AddressResolveFailed = 3237101581, - ConnectFailed = 3237101582, - SslError = 3237101583, - ConnectionTimeout = 3237101584, - DataReceiveTimeout = 3237101585, - PeerNoResponse = 3237101586, - UnexpectedHttpRedirect = 3237101587, - DataSendFailure = 3237101588, - DataReceiveFailure = 3237101589, - CertificateRejected = 3237101590, - DataNotAllowed = 3237101591, - NetworkErrorUnknown = 3237101592, -} - -/// User-friendly error messages -static ERROR_MESSAGES: Lazy> = Lazy::new(|| { - let mut m = HashMap::new(); - - // (Title, Description) - m.insert(15859712, ("Success", "Session started successfully.")); - - // Client errors - m.insert( - 3237085186, - ( - "Invalid Operation", - "The requested operation is not valid at this time.", - ), - ); - m.insert( - 3237089282, - ( - "Network Error", - "A network error occurred. Please check your internet connection.", - ), - ); - m.insert( - 3237093377, - ( - "Authentication Required", - "Your session has expired. Please log in again.", - ), - ); - m.insert( - 3237093379, - ( - "Server Response Error", - "Failed to parse server response. Please try again.", - ), - ); - m.insert( - 3237093381, - ( - "Invalid Server Response", - "The server returned an invalid response.", - ), - ); - m.insert( - 3237093384, - ("Session Error", "An error occurred during session setup."), - ); - m.insert( - 3237093387, - ( - "Authentication Timeout", - "Authentication token update timed out. Please log in again.", - ), - ); - - // Server errors (most common ones with user-friendly messages) - m.insert( - 3237093634, - ("Access Forbidden", "Access to this service is forbidden."), - ); - m.insert( - 3237093635, - ("Server Timeout", "The server timed out. Please try again."), - ); - m.insert( - 3237093636, - ( - "Server Error", - "An internal server error occurred. Please try again later.", - ), - ); - m.insert(3237093637, ("Invalid Request", "The request was invalid.")); - m.insert( - 3237093639, - ( - "Too Many Sessions", - "You have too many active sessions. Please close some sessions and try again.", - ), - ); - m.insert(3237093643, ("Session Limit Exceeded", "You have reached your session limit. Another session may already be running on your account.")); - m.insert( - 3237093645, - ( - "Session Time Exceeded", - "Your session time has been exceeded.", - ), - ); - m.insert( - 3237093646, - ( - "Authentication Failed", - "Authentication failed. Please log in again.", - ), - ); - m.insert( - 3237093648, - ( - "Session Expired", - "Your authentication has expired. Please log in again.", - ), - ); - m.insert( - 3237093650, - ( - "Entitlement Error", - "You don't have access to this game or service.", - ), - ); - m.insert( - 3237093651, - ( - "Game Not Available", - "This game is not currently available.", - ), - ); - m.insert( - 3237093652, - ("Game Not Found", "This game was not found in the library."), - ); - m.insert( - 3237093655, - ( - "EULA Required", - "You must accept the End User License Agreement to continue.", - ), - ); - m.insert( - 3237093656, - ( - "Under Maintenance", - "GeForce NOW is currently under maintenance. Please try again later.", - ), - ); - m.insert( - 3237093657, - ( - "Service Unavailable", - "The service is temporarily unavailable. Please try again later.", - ), - ); - m.insert( - 3237093658, - ( - "Steam Guard Required", - "Steam Guard authentication is required. Please complete Steam Guard verification.", - ), - ); - m.insert( - 3237093659, - ( - "Steam Login Required", - "You need to link your Steam account to play this game.", - ), - ); - m.insert( - 3237093660, - ( - "Steam Guard Invalid", - "Steam Guard code is invalid. Please try again.", - ), - ); - m.insert( - 3237093661, - ( - "Steam Profile Private", - "Your Steam profile is private. Please make it public or friends-only.", - ), - ); - m.insert( - 3237093667, - ( - "Email Not Verified", - "Please verify your email address to continue.", - ), - ); - m.insert( - 3237093673, - ( - "Game Updating", - "This game is currently being updated. Please try again later.", - ), - ); - m.insert(3237093674, ("Game Not Found", "This game was not found.")); - m.insert( - 3237093675, - ( - "Insufficient Credits", - "You don't have enough credits for this session.", - ), - ); - m.insert( - 3237093678, - ( - "Session Taken Over", - "Your session was taken over by another device.", - ), - ); - m.insert(3237093681, ("Session Expired", "Your session has expired.")); - m.insert( - 3237093682, - ( - "Device Limit Reached", - "You have reached the session limit for this device.", - ), - ); - m.insert( - 3237093683, - ( - "Region At Capacity", - "Your region is currently at capacity. Please try again later.", - ), - ); - m.insert( - 3237093684, - ( - "Region Not Supported", - "GeForce NOW is not available in your region.", - ), - ); - m.insert( - 3237093685, - ( - "Region Banned", - "GeForce NOW is not available in your region.", - ), - ); - m.insert( - 3237093686, - ( - "Free Tier On Hold", - "Free tier is temporarily unavailable in your region.", - ), - ); - m.insert( - 3237093687, - ( - "Paid Tier On Hold", - "Paid tier is temporarily unavailable in your region.", - ), - ); - m.insert( - 3237093688, - ( - "Game Maintenance", - "This game is currently under maintenance.", - ), - ); - m.insert( - 3237093690, - ( - "No Capacity", - "No gaming rigs are available right now. Please try again later or join the queue.", - ), - ); - m.insert( - 3237093694, - ( - "Queue Full", - "The queue is currently full. Please try again later.", - ), - ); - m.insert( - 3237093695, - ( - "Region Not Supported", - "Streaming is not supported in your region.", - ), - ); - m.insert( - 3237093698, - ( - "Game Not Available", - "This game is not available in your region.", - ), - ); - m.insert( - 3237093701, - ("Queue Abandoned", "Your session in queue was abandoned."), - ); - m.insert( - 3237093702, - ("Account Terminated", "Your account has been terminated."), - ); - m.insert( - 3237093703, - ( - "Queue Maintenance", - "The queue was cleared due to maintenance.", - ), - ); - m.insert( - 3237093704, - ("Zone Maintenance", "This server zone is under maintenance."), - ); - m.insert(3237093711, ("Ads Timeout", "Session expired while waiting for ads. Free tier users must watch ads to play. Please start a new session.")); - m.insert( - 3237093712, - ( - "Ads Cancelled", - "Session cancelled because ads were skipped. Free tier users must watch ads to play.", - ), - ); - m.insert( - 3237093713, - ("Limited Mode", "Streaming is not allowed in limited mode."), - ); - m.insert( - 3237093715, - ("Session Limit", "Maximum number of sessions reached."), - ); - m.insert( - 3237093717, - ( - "No Capacity", - "No gaming rigs are available. Please try again later.", - ), - ); - m.insert(3237093718, ("Playability Level Issue", "Your account's playability level is insufficient. This may mean another session is already running, or there's a subscription issue.")); - m.insert( - 3237093721, - ("Storage Unavailable", "User storage is not available."), - ); - m.insert( - 3237093722, - ("Storage Error", "GFN storage is not available."), - ); - - // Cancellation - m.insert( - 15867905, - ("Session Cancelled", "Session setup was cancelled."), - ); - m.insert(15867906, ("Queue Cancelled", "You left the queue.")); - m.insert( - 15867907, - ("Request Cancelled", "The request was cancelled."), - ); - m.insert( - 15867909, - ( - "System Sleep", - "Session setup was interrupted by system sleep.", - ), - ); - m.insert( - 15868417, - ( - "No Internet", - "No internet connection during session setup.", - ), - ); - - // Network errors - m.insert( - 3237101580, - ( - "Socket Error", - "A socket error occurred. Please check your network.", - ), - ); - m.insert( - 3237101581, - ( - "DNS Error", - "Failed to resolve server address. Please check your network.", - ), - ); - m.insert( - 3237101582, - ( - "Connection Failed", - "Failed to connect to the server. Please check your network.", - ), - ); - m.insert( - 3237101583, - ("SSL Error", "A secure connection error occurred."), - ); - m.insert( - 3237101584, - ( - "Connection Timeout", - "Connection timed out. Please check your network.", - ), - ); - m.insert( - 3237101585, - ( - "Receive Timeout", - "Data receive timed out. Please check your network.", - ), - ); - m.insert( - 3237101586, - ("No Response", "Server not responding. Please try again."), - ); - m.insert( - 3237101590, - ("Certificate Error", "Server certificate was rejected."), - ); - - m -}); - -/// Parsed error information from CloudMatch response -#[derive(Debug, Clone)] -pub struct SessionError { - /// HTTP status code (e.g., 403) - pub http_status: u16, - /// CloudMatch status code from requestStatus.statusCode - pub status_code: i32, - /// Status description from requestStatus.statusDescription - pub status_description: Option, - /// Unified error code from requestStatus.unifiedErrorCode - pub unified_error_code: Option, - /// Session error code from session.errorCode - pub session_error_code: Option, - /// Computed GFN error code - pub gfn_error_code: i64, - /// User-friendly title - pub title: String, - /// User-friendly description - pub description: String, -} - -impl SessionError { - /// Parse error from CloudMatch response JSON - pub fn from_response(http_status: u16, response_body: &str) -> Self { - // Try to parse JSON - let json: serde_json::Value = - serde_json::from_str(response_body).unwrap_or(serde_json::Value::Null); - - // Extract fields - let status_code = json["requestStatus"]["statusCode"].as_i64().unwrap_or(0) as i32; - - let status_description = json["requestStatus"]["statusDescription"] - .as_str() - .map(|s| s.to_string()); - - let unified_error_code = json["requestStatus"]["unifiedErrorCode"].as_i64(); - - let session_error_code = json["session"]["errorCode"].as_i64().map(|c| c as i32); - - // Compute GFN error code using official client logic - let gfn_error_code = Self::compute_error_code(status_code, unified_error_code); - - // Get user-friendly message - let (title, description) = - Self::get_error_message(gfn_error_code, &status_description, http_status); - - SessionError { - http_status, - status_code, - status_description, - unified_error_code, - session_error_code, - gfn_error_code, - title, - description, - } - } - - /// Compute GFN error code from CloudMatch response (matching official client logic) - fn compute_error_code(status_code: i32, unified_error_code: Option) -> i64 { - // Base error code - let mut error_code: i64 = 3237093632; // SessionServerErrorBegin - - // Convert statusCode to error code - if status_code == 1 { - error_code = 15859712; // Success - } else if status_code > 0 && status_code < 255 { - error_code = 3237093632 + status_code as i64; - } - - // Use unifiedErrorCode if available and error_code is generic - if let Some(unified) = unified_error_code { - match error_code { - 3237093632 | 3237093636 | 3237093381 => { - error_code = unified; - } - _ => {} - } - } - - error_code - } - - /// Get user-friendly error message - fn get_error_message( - error_code: i64, - status_description: &Option, - http_status: u16, - ) -> (String, String) { - // Check for known error code - if let Some((title, desc)) = ERROR_MESSAGES.get(&error_code) { - return (title.to_string(), desc.to_string()); - } - - // Parse status description for known patterns - if let Some(desc) = status_description { - let desc_upper = desc.to_uppercase(); - - if desc_upper.contains("INSUFFICIENT_PLAYABILITY") { - return ( - "Session Already Active".to_string(), - "Another session is already running on your account. Please close it first or wait for it to timeout.".to_string() - ); - } - - if desc_upper.contains("SESSION_LIMIT") { - return ( - "Session Limit Exceeded".to_string(), - "You have reached your maximum number of concurrent sessions.".to_string(), - ); - } - - if desc_upper.contains("MAINTENANCE") { - return ( - "Under Maintenance".to_string(), - "The service is currently under maintenance. Please try again later." - .to_string(), - ); - } - - if desc_upper.contains("CAPACITY") || desc_upper.contains("QUEUE") { - return ( - "No Capacity Available".to_string(), - "All gaming rigs are currently in use. Please try again later.".to_string(), - ); - } - - if desc_upper.contains("AUTH") || desc_upper.contains("TOKEN") { - return ( - "Authentication Error".to_string(), - "Please log in again.".to_string(), - ); - } - - if desc_upper.contains("ENTITLEMENT") { - return ( - "Access Denied".to_string(), - "You don't have access to this game or service.".to_string(), - ); - } - } - - // Fallback based on HTTP status - match http_status { - 401 => ( - "Unauthorized".to_string(), - "Please log in again.".to_string(), - ), - 403 => ( - "Access Denied".to_string(), - "Access to this resource was denied.".to_string(), - ), - 404 => ( - "Not Found".to_string(), - "The requested resource was not found.".to_string(), - ), - 429 => ( - "Too Many Requests".to_string(), - "Please wait a moment and try again.".to_string(), - ), - 500..=599 => ( - "Server Error".to_string(), - "A server error occurred. Please try again later.".to_string(), - ), - _ => ( - "Error".to_string(), - format!("An error occurred (HTTP {}).", http_status), - ), - } - } - - /// Check if this error indicates another session is running - pub fn is_session_conflict(&self) -> bool { - matches!( - self.gfn_error_code, - 3237093643 | // SessionLimitExceeded - 3237093682 | // SessionLimitPerDeviceReached - 3237093715 | // MaxSessionNumberLimitExceeded - 3237093718 // SessionInsufficientPlayabilityLevel - ) || self - .status_description - .as_ref() - .map(|d| d.to_uppercase().contains("INSUFFICIENT_PLAYABILITY")) - .unwrap_or(false) - } - - /// Check if this is a temporary error that might resolve with retry - pub fn is_retryable(&self) -> bool { - matches!( - self.gfn_error_code, - 3237089282 | // NetworkError - 3237093635 | // ServerInternalTimeout - 3237093636 | // ServerInternalError - 3237093683 | // ForwardingZoneOutOfCapacity - 3237093690 | // InsufficientVmCapacity - 3237093717 | // SessionRejectedNoCapacity - 3237101584 | // ConnectionTimeout - 3237101585 | // DataReceiveTimeout - 3237101586 // PeerNoResponse - ) - } - - /// Check if user needs to log in again - pub fn needs_reauth(&self) -> bool { - matches!( - self.gfn_error_code, - 3237093377 | // AuthTokenNotUpdated - 3237093387 | // AuthTokenUpdateTimeout - 3237093646 | // AuthFailure - 3237093647 | // InvalidAuthenticationMalformed - 3237093648 | // InvalidAuthenticationExpired - 3237093649 | // InvalidAuthenticationNotFound - 3237093668 | // InvalidAuthenticationUnsupportedProtocol - 3237093669 | // InvalidAuthenticationUnknownToken - 3237093670 // InvalidAuthenticationCredentials - ) || self.http_status == 401 - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_parse_insufficient_playability() { - let response = r#"{"session":{"sessionId":"test","errorCode":1},"requestStatus":{"statusCode":86,"statusDescription":"INSUFFICIENT_PLAYABILITY_LEVEL 8192C105","unifiedErrorCode":-2121088763}}"#; - - let error = SessionError::from_response(403, response); - - assert_eq!(error.status_code, 86); - assert_eq!(error.gfn_error_code, 3237093718); // 3237093632 + 86 - assert!(error.is_session_conflict()); - assert_eq!(error.title, "Session Already Active"); - } - - #[test] - fn test_parse_session_limit() { - let response = - r#"{"requestStatus":{"statusCode":11,"statusDescription":"SESSION_LIMIT_EXCEEDED"}}"#; - - let error = SessionError::from_response(403, response); - - assert_eq!(error.gfn_error_code, 3237093643); // 3237093632 + 11 - assert!(error.is_session_conflict()); - } -} diff --git a/opennow-streamer/src/api/games.rs b/opennow-streamer/src/api/games.rs deleted file mode 100644 index d256e17..0000000 --- a/opennow-streamer/src/api/games.rs +++ /dev/null @@ -1,746 +0,0 @@ -//! Games Library API -//! -//! Fetch and search GFN game catalog using GraphQL. - -use anyhow::{Result, Context}; -use log::{info, debug, warn, error}; -use serde::Deserialize; -use std::time::{SystemTime, UNIX_EPOCH}; - -use crate::app::{GameInfo, GameSection, GameVariant}; -use crate::auth; -use super::GfnApiClient; - -/// GraphQL endpoint -const GRAPHQL_URL: &str = "https://games.geforce.com/graphql"; - -/// Persisted query hash for panels (MAIN, LIBRARY, etc.) -const PANELS_QUERY_HASH: &str = "f8e26265a5db5c20e1334a6872cf04b6e3970507697f6ae55a6ddefa5420daf0"; - -/// Persisted query hash for app metadata -const APP_METADATA_QUERY_HASH: &str = "39187e85b6dcf60b7279a5f233288b0a8b69a8b1dbcfb5b25555afdcb988f0d7"; - -/// GFN CEF User-Agent -const GFN_USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 NVIDIACEFClient/HEAD/debb5919f6 GFN-PC/2.0.80.173"; - -/// Default VPC ID for general access (from GFN config) -const DEFAULT_VPC_ID: &str = "GFN-PC"; - -/// Default locale -const DEFAULT_LOCALE: &str = "en_US"; - -/// LCARS Client ID -const LCARS_CLIENT_ID: &str = "ec7e38d4-03af-4b58-b131-cfb0495903ab"; - -/// GFN client version -const GFN_CLIENT_VERSION: &str = "2.0.80.173"; - -// ============================================ -// GraphQL Response Types (matching Tauri client) -// ============================================ - -#[derive(Debug, Deserialize)] -struct GraphQLResponse { - data: Option, - errors: Option>, -} - -#[derive(Debug, Deserialize)] -struct AppMetaDataResponse { - data: Option, - errors: Option>, -} - -#[derive(Debug, Deserialize)] -struct AppsData { - apps: ItemsData, -} - -#[derive(Debug, Deserialize)] -struct ItemsData { - items: Vec, -} - -#[derive(Debug, Deserialize)] -struct GraphQLError { - message: String, -} - -#[derive(Debug, Deserialize)] -struct PanelsData { - panels: Vec, -} - -#[derive(Debug, Deserialize)] -struct Panel { - #[allow(dead_code)] - id: Option, - name: String, - #[serde(default)] - sections: Vec, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct PanelSection { - #[serde(default)] - id: Option, - #[serde(default)] - title: Option, - #[serde(default)] - items: Vec, -} - -/// Panel items are tagged by __typename -#[derive(Debug, Deserialize)] -#[serde(tag = "__typename")] -enum PanelItem { - GameItem { app: AppData }, - #[serde(other)] - Other, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct AppData { - id: String, - title: String, - #[serde(default)] - description: Option, - #[serde(default)] - long_description: Option, - #[serde(default)] - images: Option, - #[serde(default)] - variants: Option>, - #[serde(default)] - gfn: Option, -} - -/// Image URLs from GraphQL -#[derive(Debug, Deserialize)] -struct AppImages { - #[serde(rename = "GAME_BOX_ART")] - game_box_art: Option, - #[serde(rename = "TV_BANNER")] - tv_banner: Option, - #[serde(rename = "HERO_IMAGE")] - hero_image: Option, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct AppVariant { - id: String, - app_store: String, - #[serde(default)] - supported_controls: Option>, - #[serde(default)] - gfn: Option, -} - -#[derive(Debug, Deserialize)] -struct VariantGfnStatus { - #[serde(default)] - status: Option, - #[serde(default)] - library: Option, -} - -#[derive(Debug, Deserialize)] -struct VariantLibraryStatus { - #[serde(default)] - selected: Option, - #[serde(default)] - installed: Option, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct AppGfnStatus { - #[serde(default)] - playability_state: Option, - #[serde(default)] - play_type: Option, - #[serde(default)] - minimum_membership_tier_label: Option, - #[serde(default)] - catalog_sku_strings: Option, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -struct CatalogSkuStrings { - #[serde(default)] - sku_based_playability_text: Option, - #[serde(default)] - sku_based_tag: Option, -} - -// ============================================ -// Raw game data from static JSON -// ============================================ - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct RawGameInfo { - /// Game ID (numeric in public list) - #[serde(default)] - id: Option, - /// Game title - #[serde(default)] - title: Option, - /// Publisher name - #[serde(default)] - publisher: Option, - /// Store type (Steam, Epic, etc.) - #[serde(default)] - store: Option, - /// Steam URL (contains app ID) - #[serde(default)] - steam_url: Option, - /// Epic URL - #[serde(default)] - epic_url: Option, - /// Status (AVAILABLE, etc.) - #[serde(default)] - status: Option, - /// Genres - #[serde(default)] - genres: Vec, -} - -/// Generate a random huId for GraphQL requests -fn generate_hu_id() -> String { - let timestamp = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_nanos(); - format!("{:x}", timestamp) -} - -/// Optimize image URL with webp format and size -fn optimize_image_url(url: &str, width: u32) -> String { - if url.contains("img.nvidiagrid.net") { - format!("{};f=webp;w={}", url, width) - } else { - url.to_string() - } -} - -impl GfnApiClient { - /// Fetch panels using persisted query (GET request) - /// This is the correct way to fetch from GFN API - async fn fetch_panels(&self, panel_names: &[&str], vpc_id: &str) -> Result> { - let token = self.token() - .context("No access token for panel fetch")?; - - let variables = serde_json::json!({ - "vpcId": vpc_id, - "locale": DEFAULT_LOCALE, - "panelNames": panel_names, - }); - - let extensions = serde_json::json!({ - "persistedQuery": { - "sha256Hash": PANELS_QUERY_HASH - } - }); - - // Build request type based on panel names - let request_type = if panel_names.contains(&"LIBRARY") { - "panels/Library" - } else { - "panels/MainV2" - }; - - let variables_str = serde_json::to_string(&variables) - .context("Failed to serialize variables")?; - let extensions_str = serde_json::to_string(&extensions) - .context("Failed to serialize extensions")?; - - let hu_id = generate_hu_id(); - - // Build URL with all required parameters - let url = format!( - "{}?requestType={}&extensions={}&huId={}&variables={}", - GRAPHQL_URL, - urlencoding::encode(request_type), - urlencoding::encode(&extensions_str), - urlencoding::encode(&hu_id), - urlencoding::encode(&variables_str) - ); - - debug!("Fetching panels from: {}", url); - - let response = self.client - .get(&url) - .header("User-Agent", GFN_USER_AGENT) - .header("Accept", "application/json, text/plain, */*") - .header("Content-Type", "application/graphql") - .header("Origin", "https://play.geforcenow.com") - .header("Referer", "https://play.geforcenow.com/") - .header("Authorization", format!("GFNJWT {}", token)) - // GFN client headers (native client) - .header("nv-client-id", LCARS_CLIENT_ID) - .header("nv-client-type", "NATIVE") - .header("nv-client-version", GFN_CLIENT_VERSION) - .header("nv-client-streamer", "NVIDIA-CLASSIC") - .header("nv-device-os", "WINDOWS") - .header("nv-device-type", "DESKTOP") - .header("nv-device-make", "UNKNOWN") - .header("nv-device-model", "UNKNOWN") - .header("nv-browser-type", "CHROME") - .send() - .await - .context("Panel fetch request failed")?; - - let status = response.status(); - if !status.is_success() { - let body = response.text().await.unwrap_or_default(); - return Err(anyhow::anyhow!("Panel fetch failed: {} - {}", status, body)); - } - - let body_text = response.text().await - .context("Failed to read panel response")?; - - debug!("Panel response (first 500 chars): {}", &body_text[..body_text.len().min(500)]); - - let graphql_response: GraphQLResponse = serde_json::from_str(&body_text) - .context(format!("Failed to parse panel response: {}", &body_text[..body_text.len().min(200)]))?; - - if let Some(errors) = graphql_response.errors { - if !errors.is_empty() { - let error_msg = errors.iter().map(|e| e.message.clone()).collect::>().join(", "); - return Err(anyhow::anyhow!("GraphQL errors: {}", error_msg)); - } - } - - Ok(graphql_response.data - .map(|d| d.panels) - .unwrap_or_default()) - } - - /// Convert AppData to GameInfo - fn app_to_game_info(app: AppData) -> GameInfo { - // Build variants list from app variants - let variants: Vec = app.variants.as_ref() - .map(|vars| vars.iter().map(|v| GameVariant { - id: v.id.clone(), - store: v.app_store.clone(), - supported_controls: v.supported_controls.clone().unwrap_or_default(), - }).collect()) - .unwrap_or_default(); - - // Find selected variant index (the one marked as selected, or 0 for first) - let selected_variant_index = app.variants.as_ref() - .and_then(|vars| vars.iter().position(|v| { - v.gfn.as_ref() - .and_then(|g| g.library.as_ref()) - .and_then(|l| l.selected) - .unwrap_or(false) - })) - .unwrap_or(0); - - // Get the selected variant for current store/id - let selected_variant = variants.get(selected_variant_index); - - let store = selected_variant - .map(|v| v.store.clone()) - .unwrap_or_else(|| "Unknown".to_string()); - - // Use variant ID for launching (e.g., "102217611") - let variant_id = selected_variant - .map(|v| v.id.clone()) - .unwrap_or_default(); - - // Parse app_id from variant ID (may be numeric) - let app_id = variant_id.parse::().ok(); - - // Optimize image URLs (272px width for cards, webp format) - // Prefer GAME_BOX_ART over TV_BANNER for better quality box art - let image_url = app.images.as_ref() - .and_then(|i| i.game_box_art.as_ref().or(i.tv_banner.as_ref()).or(i.hero_image.as_ref())) - .map(|url| optimize_image_url(url, 272)); - - // Check if playType is INSTALL_TO_PLAY - let is_install_to_play = app.gfn.as_ref() - .and_then(|g| g.play_type.as_deref()) - .map(|t| t == "INSTALL_TO_PLAY") - .unwrap_or(false); - - GameInfo { - id: if variant_id.is_empty() { app.id.clone() } else { variant_id }, - title: app.title, - publisher: None, - image_url, - store, - app_id, - is_install_to_play, - play_type: app.gfn.as_ref().and_then(|g| g.play_type.clone()), - membership_tier_label: app.gfn.as_ref().and_then(|g| g.minimum_membership_tier_label.clone()), - playability_text: app.gfn.as_ref().and_then(|g| g.catalog_sku_strings.as_ref()).and_then(|s| s.sku_based_playability_text.clone()), - uuid: Some(app.id.clone()), - description: app.description.or(app.long_description), - variants, - selected_variant_index, - } - } - - /// Fetch games from MAIN panel (GraphQL with images) - pub async fn fetch_main_games(&self, vpc_id: Option<&str>) -> Result> { - // Use provided VPC ID or fetch dynamically from serverInfo - let vpc = match vpc_id { - Some(v) => v.to_string(), - None => { - let token = self.token().map(|s| s.as_str()); - super::get_vpc_id(&self.client, token).await - } - }; - - info!("Fetching main games from GraphQL (VPC: {})", vpc); - - let panels = self.fetch_panels(&["MAIN"], &vpc).await?; - - let mut games: Vec = Vec::new(); - - for panel in panels { - info!("Panel '{}' has {} sections", panel.name, panel.sections.len()); - for section in panel.sections { - debug!("Section has {} items", section.items.len()); - for item in section.items { - if let PanelItem::GameItem { app } = item { - debug!("Found game: {} with images: {:?}", app.title, app.images.is_some()); - games.push(Self::app_to_game_info(app)); - } - } - } - } - - info!("Fetched {} games from MAIN panel", games.len()); - Ok(games) - } - - /// Fetch games organized by section (Home view) - /// Returns sections with titles like "Trending", "Free to Play", etc. - pub async fn fetch_sectioned_games(&self, vpc_id: Option<&str>) -> Result> { - // Use provided VPC ID or fetch dynamically from serverInfo - let vpc = match vpc_id { - Some(v) => v.to_string(), - None => { - let token = self.token().map(|s| s.as_str()); - super::get_vpc_id(&self.client, token).await - } - }; - - info!("Fetching sectioned games from GraphQL (VPC: {})", vpc); - - let panels = self.fetch_panels(&["MAIN"], &vpc).await?; - - let mut sections: Vec = Vec::new(); - - for panel in panels { - info!("Panel '{}' has {} sections", panel.name, panel.sections.len()); - for section in panel.sections { - let section_title = section.title.clone().unwrap_or_else(|| "Games".to_string()); - debug!("Section '{}' has {} items", section_title, section.items.len()); - - let games: Vec = section.items - .into_iter() - .filter_map(|item| { - if let PanelItem::GameItem { app } = item { - Some(Self::app_to_game_info(app)) - } else { - None - } - }) - .collect(); - - if !games.is_empty() { - info!("Section '{}': {} games", section_title, games.len()); - sections.push(GameSection { - id: section.id, - title: section_title, - games, - }); - } - } - } - - info!("Fetched {} sections from MAIN panel", sections.len()); - Ok(sections) - } - - /// Fetch user's library (GraphQL) - pub async fn fetch_library(&self, vpc_id: Option<&str>) -> Result> { - // Use provided VPC ID or fetch dynamically from serverInfo - let vpc = match vpc_id { - Some(v) => v.to_string(), - None => { - let token = self.token().map(|s| s.as_str()); - super::get_vpc_id(&self.client, token).await - } - }; - - info!("Fetching library from GraphQL (VPC: {})", vpc); - - let panels = match self.fetch_panels(&["LIBRARY"], &vpc).await { - Ok(p) => p, - Err(e) => { - warn!("Library fetch failed: {}", e); - return Ok(Vec::new()); - } - }; - - let mut games: Vec = Vec::new(); - - for panel in panels { - if panel.name == "LIBRARY" { - for section in panel.sections { - for item in section.items { - if let PanelItem::GameItem { app } = item { - games.push(Self::app_to_game_info(app)); - } - } - } - } - } - - info!("Fetched {} games from LIBRARY panel", games.len()); - Ok(games) - } - - /// Fetch public games list (static JSON, no auth required) - /// Uses Steam CDN for game images when available - pub async fn fetch_public_games(&self) -> Result> { - let url = "https://static.nvidiagrid.net/supported-public-game-list/locales/gfnpc-en-US.json"; - - info!("Fetching public games from: {}", url); - - let response = self.client.get(url) - .header("User-Agent", GFN_USER_AGENT) - .send() - .await - .context("Failed to fetch games list")?; - - let text = response.text().await - .context("Failed to read games response")?; - - debug!("Fetched {} bytes of games data", text.len()); - - let raw_games: Vec = serde_json::from_str(&text) - .context("Failed to parse games JSON")?; - - let games: Vec = raw_games.into_iter() - .filter_map(|g| { - let title = g.title?; - - // Extract ID (can be number or string) - let id = match g.id { - Some(serde_json::Value::Number(n)) => n.to_string(), - Some(serde_json::Value::String(s)) => s, - _ => title.clone(), - }; - - // Extract Steam app ID from steamUrl - // Format: https://store.steampowered.com/app/123456 - let app_id = g.steam_url - .as_ref() - .and_then(|url| { - url.split("/app/") - .nth(1) - .and_then(|s| s.split('/').next()) - .and_then(|s| s.parse::().ok()) - }); - - // Skip games that aren't available - if g.status.as_deref() != Some("AVAILABLE") { - return None; - } - - // Generate image URL from Steam CDN if we have a Steam app ID - // Steam CDN provides public game art: header.jpg (460x215), library_600x900.jpg - let image_url = app_id.map(|steam_id| { - format!("https://cdn.cloudflare.steamstatic.com/steam/apps/{}/library_600x900.jpg", steam_id) - }); - - let store = g.store.unwrap_or_else(|| "Unknown".to_string()); - - Some(GameInfo { - id, - title, - publisher: g.publisher, - image_url, - store, - app_id, - is_install_to_play: false, - play_type: None, - membership_tier_label: None, - playability_text: None, - uuid: None, - description: None, - variants: Vec::new(), - selected_variant_index: 0, - }) - }) - .collect(); - - info!("Parsed {} games from public list", games.len()); - Ok(games) - } - /// Search games by title - pub fn search_games<'a>(games: &'a [GameInfo], query: &str) -> Vec<&'a GameInfo> { - let query_lower = query.to_lowercase(); - - games.iter() - .filter(|g| g.title.to_lowercase().contains(&query_lower)) - .collect() - } - - /// Fetch full details for a specific app (including playType) - pub async fn fetch_app_details(&self, app_id: &str) -> Result> { - let token = self.token() - .context("No access token for app details")?; - - // Get VPC ID - let vpc_id = super::get_vpc_id(&self.client, Some(token)).await; - - let variables = serde_json::json!({ - "vpcId": vpc_id, - "locale": DEFAULT_LOCALE, - "appIds": [app_id], - }); - - let extensions = serde_json::json!({ - "persistedQuery": { - "sha256Hash": APP_METADATA_QUERY_HASH - } - }); - - let variables_str = serde_json::to_string(&variables)?; - let extensions_str = serde_json::to_string(&extensions)?; - let hu_id = generate_hu_id(); - - let url = format!( - "{}?requestType=appMetaData&extensions={}&huId={}&variables={}", - GRAPHQL_URL, - urlencoding::encode(&extensions_str), - urlencoding::encode(&hu_id), - urlencoding::encode(&variables_str) - ); - - debug!("Fetching app details from: {}", url); - info!("Fetching app details for ID: {} (Variables: {})", app_id, variables_str); - - let response = self.client - .get(&url) - .header("User-Agent", GFN_USER_AGENT) - .header("Accept", "application/json") - .header("Content-Type", "application/graphql") - .header("Authorization", format!("GFNJWT {}", token)) - .header("nv-client-id", LCARS_CLIENT_ID) - .send() - .await - .context("App details request failed")?; - - if !response.status().is_success() { - let status = response.status(); - let error_body = response.text().await.unwrap_or_else(|_| "Could not read error body".to_string()); - error!("App details failed for {}: {} - Body: {}", app_id, status, error_body); - return Err(anyhow::anyhow!("App details failed: {} - {}", status, error_body)); - } - - let body = response.text().await?; - let response_data: AppMetaDataResponse = serde_json::from_str(&body) - .context("Failed to parse app details")?; - - if let Some(data) = response_data.data { - if let Some(app) = data.apps.items.into_iter().next() { - return Ok(Some(Self::app_to_game_info(app))); - } - } - - Ok(None) - } -} - - - -/// Fetch server info to get VPC ID for current provider -pub async fn fetch_server_info(access_token: Option<&str>) -> Result { - let base_url = auth::get_streaming_base_url(); - let url = format!("{}v2/serverInfo", base_url); - - info!("Fetching server info from: {}", url); - - let client = reqwest::Client::builder() - .user_agent(GFN_USER_AGENT) - .build()?; - - let mut request = client - .get(&url) - .header("Accept", "application/json") - .header("nv-client-id", LCARS_CLIENT_ID) - .header("nv-client-type", "BROWSER") - .header("nv-client-version", GFN_CLIENT_VERSION) - .header("nv-client-streamer", "WEBRTC") - .header("nv-device-os", "WINDOWS") - .header("nv-device-type", "DESKTOP"); - - if let Some(token) = access_token { - request = request.header("Authorization", format!("GFNJWT {}", token)); - } - - let response = request.send().await - .context("Server info request failed")?; - - if !response.status().is_success() { - return Err(anyhow::anyhow!("Server info failed: {}", response.status())); - } - - #[derive(Deserialize)] - #[serde(rename_all = "camelCase")] - struct ServerInfoResponse { - request_status: Option, - #[serde(default)] - meta_data: Vec, - } - - #[derive(Deserialize)] - #[serde(rename_all = "camelCase")] - struct RequestStatus { - server_id: Option, - } - - #[derive(Deserialize)] - struct MetaDataEntry { - key: String, - value: String, - } - - let server_response: ServerInfoResponse = response.json().await - .context("Failed to parse server info")?; - - let vpc_id = server_response.request_status - .and_then(|s| s.server_id) - .unwrap_or_else(|| DEFAULT_VPC_ID.to_string()); - - // Extract regions from metaData - let mut regions: Vec<(String, String)> = Vec::new(); - for meta in server_response.meta_data { - if meta.value.starts_with("https://") { - regions.push((meta.key, meta.value)); - } - } - - info!("Server info: VPC={}, {} regions", vpc_id, regions.len()); - - Ok(ServerInfo { vpc_id, regions }) -} - -/// Server info result -#[derive(Debug, Clone)] -pub struct ServerInfo { - pub vpc_id: String, - pub regions: Vec<(String, String)>, -} diff --git a/opennow-streamer/src/api/mod.rs b/opennow-streamer/src/api/mod.rs deleted file mode 100644 index b79e439..0000000 --- a/opennow-streamer/src/api/mod.rs +++ /dev/null @@ -1,479 +0,0 @@ -//! GFN API Client -//! -//! HTTP API interactions with GeForce NOW services. - -mod cloudmatch; -mod games; -pub mod error_codes; -pub mod queue; - -#[allow(unused_imports)] -pub use cloudmatch::*; -pub use games::*; -pub use error_codes::SessionError; -pub use queue::{QueueServerInfo, fetch_queue_servers, format_queue_eta, calculate_server_score, get_auto_selected_server, get_unique_regions, sort_servers}; - -use reqwest::Client; -use parking_lot::RwLock; -use log::{info, debug, warn}; -use serde::Deserialize; - -/// Cached VPC ID from serverInfo -static CACHED_VPC_ID: RwLock> = RwLock::new(None); - -/// Server info response from /v2/serverInfo endpoint -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct ServerInfoResponse { - request_status: Option, - #[serde(default)] - meta_data: Vec, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct ServerInfoRequestStatus { - server_id: Option, -} - -#[derive(Debug, Clone, Deserialize)] -struct ServerMetaData { - key: String, - value: String, -} - -/// Dynamic server region from serverInfo API -#[derive(Debug, Clone)] -pub struct DynamicServerRegion { - pub name: String, - pub url: String, -} - -/// Get the cached VPC ID or fetch it from serverInfo -pub async fn get_vpc_id(client: &Client, token: Option<&str>) -> String { - // Check cache first - { - let cached = CACHED_VPC_ID.read(); - if let Some(vpc_id) = cached.as_ref() { - return vpc_id.clone(); - } - } - - // Fetch from serverInfo endpoint - if let Some(vpc_id) = fetch_vpc_id_from_server_info(client, token).await { - // Cache it - *CACHED_VPC_ID.write() = Some(vpc_id.clone()); - return vpc_id; - } - - // Fallback to a common European VPC - "NP-AMS-08".to_string() -} - -/// Fetch VPC ID from the /v2/serverInfo endpoint -async fn fetch_vpc_id_from_server_info(client: &Client, token: Option<&str>) -> Option { - let url = "https://prod.cloudmatchbeta.nvidiagrid.net/v2/serverInfo"; - - info!("Fetching VPC ID from serverInfo: {}", url); - - let mut request = client - .get(url) - .header("Accept", "application/json") - .header("nv-client-id", "ec7e38d4-03af-4b58-b131-cfb0495903ab") - .header("nv-client-type", "NATIVE") - .header("nv-client-version", "2.0.80.173") - .header("nv-client-streamer", "NVIDIA-CLASSIC") - .header("nv-device-os", "WINDOWS") - .header("nv-device-type", "DESKTOP"); - - if let Some(t) = token { - request = request.header("Authorization", format!("GFNJWT {}", t)); - } - - let response = match request.send().await { - Ok(r) => r, - Err(e) => { - warn!("Failed to fetch serverInfo: {}", e); - return None; - } - }; - - if !response.status().is_success() { - warn!("serverInfo returned status: {}", response.status()); - return None; - } - - let body = match response.text().await { - Ok(b) => b, - Err(e) => { - warn!("Failed to read serverInfo body: {}", e); - return None; - } - }; - - debug!("serverInfo response: {}", &body[..body.len().min(500)]); - - let info: ServerInfoResponse = match serde_json::from_str(&body) { - Ok(i) => i, - Err(e) => { - warn!("Failed to parse serverInfo: {}", e); - return None; - } - }; - - let vpc_id = info.request_status - .and_then(|s| s.server_id); - - info!("Discovered VPC ID: {:?}", vpc_id); - vpc_id -} - -/// Clear the cached VPC ID (call on logout) -pub fn clear_vpc_cache() { - *CACHED_VPC_ID.write() = None; -} - -/// Fetch dynamic server regions from the /v2/serverInfo endpoint -/// Uses the selected provider's streaming URL (supports Alliance partners) -/// Returns regions discovered from metaData with their streaming URLs -pub async fn fetch_dynamic_regions(client: &Client, token: Option<&str>) -> Vec { - use crate::auth; - - // Get the base URL from the selected provider (Alliance partners have different URLs) - let base_url = auth::get_streaming_base_url(); - let url = format!("{}v2/serverInfo", base_url); - - info!("[serverInfo] Fetching dynamic regions from: {}", url); - - let mut request = client - .get(&url) - .header("Accept", "application/json") - .header("nv-client-id", "ec7e38d4-03af-4b58-b131-cfb0495903ab") - .header("nv-client-type", "BROWSER") - .header("nv-client-version", "2.0.80.173") - .header("nv-client-streamer", "WEBRTC") - .header("nv-device-os", "WINDOWS") - .header("nv-device-type", "DESKTOP"); - - if let Some(t) = token { - request = request.header("Authorization", format!("GFNJWT {}", t)); - } - - let response = match request.send().await { - Ok(r) => r, - Err(e) => { - warn!("[serverInfo] Failed to fetch: {}", e); - return Vec::new(); - } - }; - - if !response.status().is_success() { - warn!("[serverInfo] Returned status: {}", response.status()); - return Vec::new(); - } - - let body = match response.text().await { - Ok(b) => b, - Err(e) => { - warn!("[serverInfo] Failed to read body: {}", e); - return Vec::new(); - } - }; - - let info: ServerInfoResponse = match serde_json::from_str(&body) { - Ok(i) => i, - Err(e) => { - warn!("[serverInfo] Failed to parse: {}", e); - return Vec::new(); - } - }; - - // Extract regions from metaData - // Format: key="REGION NAME", value="https://region-url.domain.net" - // For NVIDIA: URLs contain "nvidiagrid.net" - // For Alliance partners: URLs may have different domains - let mut regions: Vec = Vec::new(); - - for meta in &info.meta_data { - // Skip special keys like "gfn-regions" - if meta.key == "gfn-regions" || meta.key.starts_with("gfn-") { - continue; - } - - // Include entries where value is a streaming URL (https://) - // Don't filter by domain - Alliance partners have different domains - if meta.value.starts_with("https://") { - regions.push(DynamicServerRegion { - name: meta.key.clone(), - url: meta.value.clone(), - }); - } - } - - info!("[serverInfo] Found {} zones from API", regions.len()); - - // Also cache the VPC ID if available - if let Some(vpc_id) = info.request_status.and_then(|s| s.server_id) { - info!("[serverInfo] Discovered VPC ID: {}", vpc_id); - *CACHED_VPC_ID.write() = Some(vpc_id); - } - - regions -} - -/// HTTP client wrapper for GFN APIs -pub struct GfnApiClient { - client: Client, - access_token: Option, -} - -impl GfnApiClient { - /// Create a new API client - pub fn new() -> Self { - let client = Client::builder() - .danger_accept_invalid_certs(true) // GFN servers may have self-signed certs - .gzip(true) - .build() - .expect("Failed to create HTTP client"); - - Self { - client, - access_token: None, - } - } - - /// Set the access token for authenticated requests - pub fn set_access_token(&mut self, token: String) { - self.access_token = Some(token); - } - - /// Get the HTTP client - pub fn client(&self) -> &Client { - &self.client - } - - /// Get the access token - pub fn token(&self) -> Option<&String> { - self.access_token.as_ref() - } -} - -impl Default for GfnApiClient { - fn default() -> Self { - Self::new() - } -} - -/// Common headers for GFN API requests -pub fn gfn_headers() -> Vec<(&'static str, &'static str)> { - vec![ - ("nv-browser-type", "CHROME"), - ("nv-client-streamer", "NVIDIA-CLASSIC"), - ("nv-client-type", "NATIVE"), - ("nv-client-version", "2.0.80.173"), - ("nv-device-os", "WINDOWS"), - ("nv-device-type", "DESKTOP"), - ] -} - -/// MES (Membership/Subscription) API URL -const MES_URL: &str = "https://mes.geforcenow.com/v4/subscriptions"; - -/// LCARS Client ID -const LCARS_CLIENT_ID: &str = "ec7e38d4-03af-4b58-b131-cfb0495903ab"; - -/// GFN client version -const GFN_CLIENT_VERSION: &str = "2.0.80.173"; - -/// Subscription response from MES API -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct SubscriptionResponse { - #[serde(default = "default_tier")] - membership_tier: String, - remaining_time_in_minutes: Option, - total_time_in_minutes: Option, - #[serde(default)] - sub_type: Option, // TIME_CAPPED or UNLIMITED - #[serde(default)] - addons: Vec, - features: Option, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct SubscriptionFeatures { - #[serde(default)] - resolutions: Vec, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct SubscriptionResolution { - height_in_pixels: u32, - width_in_pixels: u32, - frames_per_second: u32, - is_entitled: bool, -} - -fn default_tier() -> String { - "FREE".to_string() -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct SubscriptionAddon { - #[serde(rename = "type")] - addon_type: Option, - sub_type: Option, - #[serde(default)] - attributes: Vec, - status: Option, -} - -#[derive(Debug, Deserialize)] -struct AddonAttribute { - key: Option, - #[serde(rename = "textValue")] - text_value: Option, -} - -/// Fetch subscription info from MES API -pub async fn fetch_subscription(token: &str, user_id: &str) -> Result { - use crate::auth; - - let client = Client::builder() - .gzip(true) - .build() - .map_err(|e| format!("Failed to create HTTP client: {}", e))?; - - // For Alliance partners, we need to fetch VPC ID from their serverInfo first - // because the cached VPC ID might be stale or from NVIDIA's serverInfo - let provider = auth::get_selected_provider(); - let vpc_id = if provider.is_alliance_partner() { - // Fetch VPC ID from Alliance partner's serverInfo - info!("Fetching VPC ID for Alliance partner: {}", provider.login_provider_display_name); - let regions = fetch_dynamic_regions(&client, Some(token)).await; - - // The VPC ID gets cached by fetch_dynamic_regions, so try to read it - let cached = CACHED_VPC_ID.read(); - if let Some(vpc) = cached.as_ref() { - info!("Using Alliance VPC ID: {}", vpc); - vpc.clone() - } else { - // Fallback: try to extract from first region URL - if let Some(first_region) = regions.first() { - // Extract VPC-like ID from region name if possible - info!("Using Alliance region as VPC hint: {}", first_region.name); - first_region.name.clone() - } else { - return Err("Could not determine Alliance VPC ID".to_string()); - } - } - } else { - // For NVIDIA, use cached VPC ID or fallback - let cached = CACHED_VPC_ID.read(); - cached.as_ref().cloned().unwrap_or_else(|| "NP-AMS-08".to_string()) - }; - - let url = format!( - "{}?serviceName=gfn_pc&languageCode=en_US&vpcId={}&userId={}", - MES_URL, vpc_id, user_id - ); - - info!("Fetching subscription from: {}", url); - - let response = client - .get(&url) - .header("Authorization", format!("GFNJWT {}", token)) - .header("Accept", "application/json") - .header("nv-client-id", LCARS_CLIENT_ID) - .header("nv-client-type", "NATIVE") - .header("nv-client-version", GFN_CLIENT_VERSION) - .header("nv-client-streamer", "NVIDIA-CLASSIC") - .header("nv-device-os", "WINDOWS") - .header("nv-device-type", "DESKTOP") - .send() - .await - .map_err(|e| format!("Failed to fetch subscription: {}", e))?; - - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - return Err(format!("Subscription API failed with status {}: {}", status, body)); - } - - let body = response.text().await - .map_err(|e| format!("Failed to read subscription response: {}", e))?; - - debug!("Subscription response: {}", &body[..body.len().min(500)]); - - let sub: SubscriptionResponse = serde_json::from_str(&body) - .map_err(|e| format!("Failed to parse subscription: {}", e))?; - - // Convert minutes to hours - let remaining_hours = sub.remaining_time_in_minutes - .map(|m| m as f32 / 60.0) - .unwrap_or(0.0); - let total_hours = sub.total_time_in_minutes - .map(|m| m as f32 / 60.0) - .unwrap_or(0.0); - - // Check for persistent storage addon - let mut has_persistent_storage = false; - let mut storage_size_gb: Option = None; - - for addon in &sub.addons { - // Check for storage addon - API returns type="STORAGE", subType="PERMANENT_STORAGE", status="OK" - if addon.addon_type.as_deref() == Some("STORAGE") - && addon.sub_type.as_deref() == Some("PERMANENT_STORAGE") - && addon.status.as_deref() == Some("OK") - { - has_persistent_storage = true; - // Try to find storage size from attributes (key is "TOTAL_STORAGE_SIZE_IN_GB") - for attr in &addon.attributes { - if attr.key.as_deref() == Some("TOTAL_STORAGE_SIZE_IN_GB") { - if let Some(val) = attr.text_value.as_ref() { - storage_size_gb = val.parse().ok(); - } - } - } - } - } - - info!("Subscription: tier={}, hours={:.1}/{:.1}, storage={}, subType={:?}", - sub.membership_tier, remaining_hours, total_hours, has_persistent_storage, sub.sub_type); - - // Check if this is an unlimited subscription (no hour cap) - let is_unlimited = sub.sub_type.as_deref() == Some("UNLIMITED"); - - // Extract entitled resolutions - let mut entitled_resolutions = Vec::new(); - if let Some(features) = sub.features { - for res in features.resolutions { - // User requested to ignore entitlement check (include all resolutions) - entitled_resolutions.push(crate::app::types::EntitledResolution { - width: res.width_in_pixels, - height: res.height_in_pixels, - fps: res.frames_per_second, - }); - } - } - - // Sort to be nice (highest res/fps first) - entitled_resolutions.sort_by(|a, b| { - b.width.cmp(&a.width) - .then(b.height.cmp(&a.height)) - .then(b.fps.cmp(&a.fps)) - }); - - Ok(crate::app::SubscriptionInfo { - membership_tier: sub.membership_tier, - remaining_hours, - total_hours, - has_persistent_storage, - storage_size_gb, - is_unlimited, - entitled_resolutions, - }) -} diff --git a/opennow-streamer/src/api/queue.rs b/opennow-streamer/src/api/queue.rs deleted file mode 100644 index eeabca6..0000000 --- a/opennow-streamer/src/api/queue.rs +++ /dev/null @@ -1,327 +0,0 @@ -//! PrintedWaste Queue Times API -//! -//! Fetches queue time information from PrintedWaste API for GeForce NOW servers. - -use log::{info, warn, debug}; -use reqwest::Client; -use serde::Deserialize; -use std::collections::HashMap; - -/// Server mapping data from PrintedWaste -#[derive(Debug, Clone)] -pub struct ServerMapping { - pub title: String, - pub region: String, - pub is4080_server: bool, - pub is5080_server: bool, - pub nuked: bool, -} - -// Custom deserialization for the weird field names -impl ServerMapping { - fn from_raw(raw: RawServerMapping) -> Self { - Self { - title: raw.title, - region: raw.region, - is4080_server: raw.is_4080_server, - is5080_server: raw.is_5080_server, - nuked: raw.nuked, - } - } -} - -#[derive(Debug, Deserialize)] -struct RawServerMapping { - title: String, - region: String, - #[serde(default, rename = "is4080Server")] - is_4080_server: bool, - #[serde(default, rename = "is5080Server")] - is_5080_server: bool, - #[serde(default)] - nuked: bool, -} - -/// Queue data for a server from PrintedWaste -#[derive(Debug, Clone, Deserialize)] -pub struct QueueData { - #[serde(rename = "QueuePosition")] - pub queue_position: i32, - #[serde(rename = "Last Updated")] - pub last_updated: i64, - #[serde(rename = "Region")] - pub region: String, - /// ETA in milliseconds - #[serde(default)] - pub eta: Option, -} - -/// Response from PrintedWaste queue API -#[derive(Debug, Deserialize)] -pub struct QueueResponse { - #[serde(default)] - pub status: bool, - #[serde(default)] - pub errors: Vec, - pub data: HashMap, -} - -/// Response from PrintedWaste server mapping API -#[derive(Debug, Deserialize)] -struct RawMappingResponse { - #[serde(default)] - status: bool, - #[serde(default)] - errors: Vec, - data: HashMap, -} - -/// Combined server info for queue display -#[derive(Debug, Clone)] -pub struct QueueServerInfo { - pub server_id: String, - pub display_name: String, - pub region: String, - pub ping_ms: Option, - pub queue_position: i32, - pub eta_seconds: Option, - pub is_4080_server: bool, - pub is_5080_server: bool, - pub last_updated: i64, -} - -/// App version for User-Agent header -const APP_VERSION: &str = env!("CARGO_PKG_VERSION"); - -/// PrintedWaste API endpoints -const QUEUE_API_URL: &str = "https://api.printedwaste.com/gfn/queue/"; -const MAPPING_API_URL: &str = "https://remote.printedwaste.com/config/GFN_SERVERID_TO_REGION_MAPPING"; - -/// Fetch server mapping from PrintedWaste -pub async fn fetch_server_mapping(client: &Client) -> Result, String> { - let user_agent = format!("OpenNOW/{}", APP_VERSION); - - debug!("Fetching server mapping from PrintedWaste..."); - - let response = client - .get(MAPPING_API_URL) - .header("User-Agent", &user_agent) - .send() - .await - .map_err(|e| format!("Failed to fetch server mapping: {}", e))?; - - if !response.status().is_success() { - return Err(format!("Server mapping API returned status: {}", response.status())); - } - - let body = response.text().await - .map_err(|e| format!("Failed to read server mapping response: {}", e))?; - - let raw: RawMappingResponse = serde_json::from_str(&body) - .map_err(|e| format!("Failed to parse server mapping: {}", e))?; - - if !raw.errors.is_empty() { - return Err(format!("Server mapping API returned errors: {:?}", raw.errors)); - } - - // Convert raw mappings to our struct - let mappings: HashMap = raw.data - .into_iter() - .map(|(k, v)| (k, ServerMapping::from_raw(v))) - .collect(); - - info!("Fetched {} server mappings from PrintedWaste", mappings.len()); - Ok(mappings) -} - -/// Fetch queue data from PrintedWaste -pub async fn fetch_queue_data(client: &Client) -> Result { - let user_agent = format!("OpenNOW/{}", APP_VERSION); - - debug!("Fetching queue data from PrintedWaste..."); - - let response = client - .get(QUEUE_API_URL) - .header("User-Agent", &user_agent) - .send() - .await - .map_err(|e| format!("Failed to fetch queue data: {}", e))?; - - if !response.status().is_success() { - return Err(format!("Queue API returned status: {}", response.status())); - } - - let body = response.text().await - .map_err(|e| format!("Failed to read queue response: {}", e))?; - - let queue: QueueResponse = serde_json::from_str(&body) - .map_err(|e| format!("Failed to parse queue data: {}", e))?; - - if !queue.errors.is_empty() { - return Err(format!("Queue API returned errors: {:?}", queue.errors)); - } - - info!("Fetched queue data for {} servers from PrintedWaste", queue.data.len()); - Ok(queue) -} - -/// Fetch combined queue server info -pub async fn fetch_queue_servers(client: &Client) -> Result, String> { - // Fetch both mapping and queue data - let (mapping_result, queue_result) = tokio::join!( - fetch_server_mapping(client), - fetch_queue_data(client) - ); - - let mapping = mapping_result?; - let queue = queue_result?; - - let mut servers: Vec = Vec::new(); - let mut servers_missing_queue_data = 0; - - for (server_id, server_mapping) in &mapping { - // Skip nuked servers - if server_mapping.nuked { - continue; - } - - // Only include RTX 4080 or 5080 servers - if !server_mapping.is4080_server && !server_mapping.is5080_server { - continue; - } - - // Get queue data for this server - if let Some(queue_data) = queue.data.get(server_id) { - servers.push(QueueServerInfo { - server_id: server_id.clone(), - display_name: server_mapping.title.clone(), - // Use the simple region from queue API (e.g., "US", "EU") - // NOT the detailed region from mapping (e.g., "US Central") - region: queue_data.region.clone(), - ping_ms: None, // Will be filled in by caller if needed - queue_position: queue_data.queue_position, - eta_seconds: queue_data.eta.map(|ms| ms / 1000), // Convert ms to seconds - is_4080_server: server_mapping.is4080_server, - is_5080_server: server_mapping.is5080_server, - last_updated: queue_data.last_updated, - }); - } else { - servers_missing_queue_data += 1; - debug!("Server {} ({}) has no queue data", server_id, server_mapping.title); - } - } - - if servers_missing_queue_data > 0 { - warn!( - "{} servers from mapping are missing queue data (out of {} eligible servers)", - servers_missing_queue_data, - servers.len() + servers_missing_queue_data - ); - } - - // Sort by queue position (shortest first) - servers.sort_by(|a, b| a.queue_position.cmp(&b.queue_position)); - - Ok(servers) -} - -/// Format ETA in a human-readable format -pub fn format_queue_eta(eta_seconds: Option) -> String { - match eta_seconds { - None | Some(0) => "No wait".to_string(), - Some(secs) if secs < 0 => "No wait".to_string(), - Some(secs) if secs < 60 => format!("{}s", secs), - Some(secs) if secs < 3600 => { - let minutes = secs / 60; - format!("{}m", minutes) - } - Some(secs) if secs < 86400 => { - let hours = secs / 3600; - let minutes = (secs % 3600) / 60; - if minutes > 0 { - format!("{}h {}m", hours, minutes) - } else { - format!("{}h", hours) - } - } - Some(secs) => { - let days = secs / 86400; - let hours = (secs % 86400) / 3600; - if hours > 0 { - format!("{}d {}h", days, hours) - } else { - format!("{}d", days) - } - } - } -} - -/// Calculate server score for "best value" sorting -/// Lower score = better server -/// Balances ping (important for gameplay) with queue time -pub fn calculate_server_score(server: &QueueServerInfo) -> f64 { - // Ping weight: 1.0 per ms - // ETA weight: 0.5 per minute (capped at 100 to prevent extremely long queues from dominating) - let ping_score = server.ping_ms.unwrap_or(500) as f64; // High penalty for unknown ping - let eta_minutes = (server.eta_seconds.unwrap_or(0) as f64) / 60.0; - let eta_score = (eta_minutes * 0.5).min(100.0); // 0.5 per minute, capped at 100 - - ping_score + eta_score -} - -/// Get the best auto-selected server based on ping and queue time balance -pub fn get_auto_selected_server(servers: &[QueueServerInfo]) -> Option<&QueueServerInfo> { - if servers.is_empty() { - return None; - } - - servers.iter().min_by(|a, b| { - let score_a = calculate_server_score(a); - let score_b = calculate_server_score(b); - score_a.partial_cmp(&score_b).unwrap_or(std::cmp::Ordering::Equal) - }) -} - -/// Get unique regions from server list -pub fn get_unique_regions(servers: &[QueueServerInfo]) -> Vec { - let mut regions: Vec = servers - .iter() - .map(|s| s.region.clone()) - .collect::>() - .into_iter() - .collect(); - regions.sort(); - regions -} - -/// Sort servers by the specified mode -pub fn sort_servers(servers: &mut [QueueServerInfo], mode: crate::app::QueueSortMode) { - use crate::app::QueueSortMode; - - match mode { - QueueSortMode::BestValue => { - servers.sort_by(|a, b| { - let score_a = calculate_server_score(a); - let score_b = calculate_server_score(b); - score_a.partial_cmp(&score_b).unwrap_or(std::cmp::Ordering::Equal) - }); - } - QueueSortMode::QueueTime => { - servers.sort_by(|a, b| { - let eta_a = a.eta_seconds.unwrap_or(i64::MAX); - let eta_b = b.eta_seconds.unwrap_or(i64::MAX); - eta_a.cmp(&eta_b) - }); - } - QueueSortMode::Ping => { - servers.sort_by(|a, b| { - let ping_a = a.ping_ms.unwrap_or(u32::MAX); - let ping_b = b.ping_ms.unwrap_or(u32::MAX); - ping_a.cmp(&ping_b) - }); - } - QueueSortMode::Alphabetical => { - servers.sort_by(|a, b| a.display_name.cmp(&b.display_name)); - } - } -} diff --git a/opennow-streamer/src/app/cache.rs b/opennow-streamer/src/app/cache.rs deleted file mode 100644 index 1624353..0000000 --- a/opennow-streamer/src/app/cache.rs +++ /dev/null @@ -1,774 +0,0 @@ -//! App Data Cache Management -//! -//! Handles caching of games, library, subscription, sessions, and tokens. - -use log::{error, info, warn}; -use std::path::PathBuf; - -use super::{ - ActiveSessionInfo, GameInfo, GameSection, SessionInfo, SessionState, SubscriptionInfo, -}; -use crate::app::session::MediaConnectionInfo; -use crate::auth::AuthTokens; - -/// Get the application data directory -/// Creates directory if it doesn't exist -pub fn get_app_data_dir() -> Option { - use std::sync::OnceLock; - static APP_DATA_DIR: OnceLock> = OnceLock::new(); - - APP_DATA_DIR - .get_or_init(|| { - let data_dir = dirs::data_dir()?; - let app_dir = data_dir.join("opennow"); - - // Ensure directory exists - if let Err(e) = std::fs::create_dir_all(&app_dir) { - error!("Failed to create app data directory: {}", e); - } - - // Migration: copy auth.json from legacy locations if it doesn't exist in new location - let new_auth = app_dir.join("auth.json"); - if !new_auth.exists() { - // Try legacy opennow-streamer location (config_dir) - if let Some(config_dir) = dirs::config_dir() { - let legacy_path = config_dir.join("opennow-streamer").join("auth.json"); - if legacy_path.exists() { - if let Err(e) = std::fs::copy(&legacy_path, &new_auth) { - warn!("Failed to migrate auth.json from legacy location: {}", e); - } else { - info!( - "Migrated auth.json from {:?} to {:?}", - legacy_path, new_auth - ); - } - } - } - - // Try gfn-client location (config_dir) - if !new_auth.exists() { - if let Some(config_dir) = dirs::config_dir() { - let legacy_path = config_dir.join("gfn-client").join("auth.json"); - if legacy_path.exists() { - if let Err(e) = std::fs::copy(&legacy_path, &new_auth) { - warn!("Failed to migrate auth.json from gfn-client: {}", e); - } else { - info!( - "Migrated auth.json from {:?} to {:?}", - legacy_path, new_auth - ); - } - } - } - } - } - - Some(app_dir) - }) - .clone() -} - -// ============================================================ -// Auth Token Cache -// ============================================================ - -pub fn tokens_path() -> Option { - get_app_data_dir().map(|p| p.join("auth.json")) -} - -pub fn load_tokens() -> Option { - let path = tokens_path()?; - let content = std::fs::read_to_string(&path).ok()?; - let tokens: AuthTokens = serde_json::from_str(&content).ok()?; - - // If token is expired, try to refresh it - if tokens.is_expired() { - if tokens.can_refresh() { - info!("Token expired, attempting refresh..."); - // Try synchronous refresh using a blocking tokio runtime - match try_refresh_tokens_sync(&tokens) { - Some(new_tokens) => { - info!("Token refresh successful!"); - return Some(new_tokens); - } - None => { - warn!("Token refresh failed, clearing auth file"); - let _ = std::fs::remove_file(&path); - return None; - } - } - } else { - info!("Token expired and no refresh token available, clearing auth file"); - let _ = std::fs::remove_file(&path); - return None; - } - } - - Some(tokens) -} - -/// Attempt to refresh tokens synchronously (blocking) -/// Used when loading tokens at startup -fn try_refresh_tokens_sync(tokens: &AuthTokens) -> Option { - let refresh_token = tokens.refresh_token.as_ref()?; - - // Create a new tokio runtime for this blocking operation - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .ok()?; - - let refresh_token_clone = refresh_token.clone(); - let result = rt.block_on(async { crate::auth::refresh_token(&refresh_token_clone).await }); - - match result { - Ok(new_tokens) => { - // Save the new tokens - save_tokens(&new_tokens); - Some(new_tokens) - } - Err(e) => { - warn!("Token refresh failed: {}", e); - None - } - } -} - -pub fn save_tokens(tokens: &AuthTokens) { - if let Some(path) = tokens_path() { - if let Some(parent) = path.parent() { - let _ = std::fs::create_dir_all(parent); - } - if let Ok(json) = serde_json::to_string_pretty(tokens) { - if let Err(e) = std::fs::write(&path, &json) { - error!("Failed to save tokens: {}", e); - } else { - info!("Saved tokens to {:?}", path); - } - } - } -} - -pub fn clear_tokens() { - if let Some(path) = tokens_path() { - let _ = std::fs::remove_file(path); - info!("Cleared auth tokens"); - } -} - -// ============================================================ -// Login Provider Cache (for Alliance persistence) -// ============================================================ - -use crate::auth::LoginProvider; - -fn provider_cache_path() -> Option { - get_app_data_dir().map(|p| p.join("login_provider.json")) -} - -pub fn save_login_provider(provider: &LoginProvider) { - if let Some(path) = provider_cache_path() { - if let Some(parent) = path.parent() { - let _ = std::fs::create_dir_all(parent); - } - if let Ok(json) = serde_json::to_string_pretty(provider) { - if let Err(e) = std::fs::write(&path, &json) { - error!("Failed to save login provider: {}", e); - } else { - info!( - "Saved login provider: {}", - provider.login_provider_display_name - ); - } - } - } -} - -pub fn load_login_provider() -> Option { - let path = provider_cache_path()?; - let content = std::fs::read_to_string(&path).ok()?; - let provider: LoginProvider = serde_json::from_str(&content).ok()?; - info!( - "Loaded cached login provider: {}", - provider.login_provider_display_name - ); - Some(provider) -} - -pub fn clear_login_provider() { - if let Some(path) = provider_cache_path() { - let _ = std::fs::remove_file(path); - info!("Cleared cached login provider"); - } -} - -// ============================================================ -// Games Cache -// ============================================================ - -fn games_cache_path() -> Option { - get_app_data_dir().map(|p| p.join("games_cache.json")) -} - -pub fn save_games_cache(games: &[GameInfo]) { - if let Some(path) = games_cache_path() { - if let Some(parent) = path.parent() { - let _ = std::fs::create_dir_all(parent); - } - if let Ok(json) = serde_json::to_string(games) { - let _ = std::fs::write(path, json); - } - } -} - -pub fn load_games_cache() -> Option> { - let path = games_cache_path()?; - let content = std::fs::read_to_string(path).ok()?; - serde_json::from_str(&content).ok() -} - -pub fn clear_games_cache() { - if let Some(path) = games_cache_path() { - let _ = std::fs::remove_file(path); - } -} - -// ============================================================ -// Library Cache -// ============================================================ - -fn library_cache_path() -> Option { - get_app_data_dir().map(|p| p.join("library_cache.json")) -} - -pub fn save_library_cache(games: &[GameInfo]) { - if let Some(path) = library_cache_path() { - if let Some(parent) = path.parent() { - let _ = std::fs::create_dir_all(parent); - } - if let Ok(json) = serde_json::to_string(games) { - let _ = std::fs::write(path, json); - } - } -} - -pub fn load_library_cache() -> Option> { - let path = library_cache_path()?; - let content = std::fs::read_to_string(path).ok()?; - serde_json::from_str(&content).ok() -} - -// ============================================================ -// Game Sections Cache (Home tab) -// ============================================================ - -/// Serializable section for cache -#[derive(serde::Serialize, serde::Deserialize)] -struct CachedSection { - id: Option, - title: String, - games: Vec, -} - -fn sections_cache_path() -> Option { - get_app_data_dir().map(|p| p.join("sections_cache.json")) -} - -pub fn save_sections_cache(sections: &[GameSection]) { - if let Some(path) = sections_cache_path() { - if let Some(parent) = path.parent() { - let _ = std::fs::create_dir_all(parent); - } - let cached: Vec = sections - .iter() - .map(|s| CachedSection { - id: s.id.clone(), - title: s.title.clone(), - games: s.games.clone(), - }) - .collect(); - if let Ok(json) = serde_json::to_string(&cached) { - let _ = std::fs::write(path, json); - } - } -} - -pub fn load_sections_cache() -> Option> { - let path = sections_cache_path()?; - let content = std::fs::read_to_string(path).ok()?; - let cached: Vec = serde_json::from_str(&content).ok()?; - Some( - cached - .into_iter() - .map(|c| GameSection { - id: c.id, - title: c.title, - games: c.games, - }) - .collect(), - ) -} - -// ============================================================ -// Subscription Cache -// ============================================================ - -fn subscription_cache_path() -> Option { - get_app_data_dir().map(|p| p.join("subscription_cache.json")) -} - -pub fn save_subscription_cache(sub: &SubscriptionInfo) { - if let Some(path) = subscription_cache_path() { - if let Some(parent) = path.parent() { - let _ = std::fs::create_dir_all(parent); - } - let cache = serde_json::json!({ - "membership_tier": sub.membership_tier, - "remaining_hours": sub.remaining_hours, - "total_hours": sub.total_hours, - "has_persistent_storage": sub.has_persistent_storage, - "storage_size_gb": sub.storage_size_gb, - "is_unlimited": sub.is_unlimited, - "entitled_resolutions": sub.entitled_resolutions, - }); - if let Ok(json) = serde_json::to_string(&cache) { - let _ = std::fs::write(path, json); - } - } -} - -pub fn load_subscription_cache() -> Option { - let path = subscription_cache_path()?; - let content = std::fs::read_to_string(path).ok()?; - let cache: serde_json::Value = serde_json::from_str(&content).ok()?; - - Some(SubscriptionInfo { - membership_tier: cache.get("membership_tier")?.as_str()?.to_string(), - remaining_hours: cache.get("remaining_hours")?.as_f64()? as f32, - total_hours: cache.get("total_hours")?.as_f64()? as f32, - has_persistent_storage: cache.get("has_persistent_storage")?.as_bool()?, - storage_size_gb: cache - .get("storage_size_gb") - .and_then(|v| v.as_u64()) - .map(|v| v as u32), - is_unlimited: cache - .get("is_unlimited") - .and_then(|v| v.as_bool()) - .unwrap_or(false), - entitled_resolutions: cache - .get("entitled_resolutions") - .and_then(|v| serde_json::from_value(v.clone()).ok()) - .unwrap_or_default(), - }) -} - -// ============================================================ -// Session Cache -// ============================================================ - -fn session_cache_path() -> Option { - get_app_data_dir().map(|p| p.join("session_cache.json")) -} - -fn session_error_path() -> Option { - get_app_data_dir().map(|p| p.join("session_error.txt")) -} - -pub fn save_session_cache(session: &SessionInfo) { - if let Some(path) = session_cache_path() { - if let Some(parent) = path.parent() { - let _ = std::fs::create_dir_all(parent); - } - // Serialize session info - let cache = serde_json::json!({ - "session_id": session.session_id, - "server_ip": session.server_ip, - "zone": session.zone, - "state": format!("{:?}", session.state), - "gpu_type": session.gpu_type, - "signaling_url": session.signaling_url, - "is_ready": session.is_ready(), - "is_queued": session.is_queued(), - "queue_position": session.queue_position(), - "media_connection_info": session.media_connection_info.as_ref().map(|mci| { - serde_json::json!({ - "ip": mci.ip, - "port": mci.port, - }) - }), - "ads_required": session.ads_required, - }); - if let Ok(json) = serde_json::to_string(&cache) { - let _ = std::fs::write(path, json); - } - } -} - -pub fn load_session_cache() -> Option { - let path = session_cache_path()?; - let content = std::fs::read_to_string(path).ok()?; - let cache: serde_json::Value = serde_json::from_str(&content).ok()?; - - let state_str = cache.get("state")?.as_str()?; - let state = if state_str.contains("Ready") { - SessionState::Ready - } else if state_str.contains("Streaming") { - SessionState::Streaming - } else if state_str.contains("InQueue") { - // Parse queue position and eta from state string - let position = cache - .get("queue_position") - .and_then(|v| v.as_u64()) - .unwrap_or(0) as u32; - SessionState::InQueue { - position, - eta_secs: 0, - } - } else if state_str.contains("Error") { - SessionState::Error(state_str.to_string()) - } else if state_str.contains("Launching") { - SessionState::Launching - } else { - SessionState::Requesting - }; - - // Parse media_connection_info if present - let media_connection_info = cache - .get("media_connection_info") - .and_then(|v| v.as_object()) - .and_then(|obj| { - let ip = obj.get("ip")?.as_str()?.to_string(); - let port = obj.get("port")?.as_u64()? as u16; - Some(MediaConnectionInfo { ip, port }) - }); - - Some(SessionInfo { - session_id: cache.get("session_id")?.as_str()?.to_string(), - server_ip: cache.get("server_ip")?.as_str()?.to_string(), - zone: cache.get("zone")?.as_str()?.to_string(), - state, - gpu_type: cache - .get("gpu_type") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()), - signaling_url: cache - .get("signaling_url") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()), - ice_servers: Vec::new(), - media_connection_info, - ads_required: cache - .get("ads_required") - .and_then(|v| v.as_bool()) - .unwrap_or(false), - ads_info: None, // Ads info not persisted to cache - }) -} - -pub fn clear_session_cache() { - if let Some(path) = session_cache_path() { - let _ = std::fs::remove_file(path); - } -} - -pub fn save_session_error(error: &str) { - if let Some(path) = session_error_path() { - if let Some(parent) = path.parent() { - let _ = std::fs::create_dir_all(parent); - } - let _ = std::fs::write(path, error); - } -} - -pub fn load_session_error() -> Option { - let path = session_error_path()?; - std::fs::read_to_string(path).ok() -} - -pub fn clear_session_error() { - if let Some(path) = session_error_path() { - let _ = std::fs::remove_file(path); - } -} - -// ============================================================ -// Active Sessions Cache (for conflict detection) -// ============================================================ - -pub fn save_active_sessions_cache(sessions: &[ActiveSessionInfo]) { - if let Some(path) = get_app_data_dir().map(|p| p.join("active_sessions.json")) { - if let Some(parent) = path.parent() { - let _ = std::fs::create_dir_all(parent); - } - if let Ok(json) = serde_json::to_string(sessions) { - let _ = std::fs::write(path, json); - } - } -} - -pub fn load_active_sessions_cache() -> Option> { - let path = get_app_data_dir()?.join("active_sessions.json"); - let content = std::fs::read_to_string(path).ok()?; - serde_json::from_str(&content).ok() -} - -pub fn clear_active_sessions_cache() { - if let Some(path) = get_app_data_dir().map(|p| p.join("active_sessions.json")) { - let _ = std::fs::remove_file(path); - } -} - -// ============================================================ -// Pending Game Cache -// ============================================================ - -pub fn save_pending_game_cache(game: &GameInfo) { - if let Some(path) = get_app_data_dir().map(|p| p.join("pending_game.json")) { - if let Some(parent) = path.parent() { - let _ = std::fs::create_dir_all(parent); - } - if let Ok(json) = serde_json::to_string(game) { - let _ = std::fs::write(path, json); - } - } -} - -pub fn load_pending_game_cache() -> Option { - let path = get_app_data_dir()?.join("pending_game.json"); - let content = std::fs::read_to_string(path).ok()?; - serde_json::from_str(&content).ok() -} - -pub fn clear_pending_game_cache() { - if let Some(path) = get_app_data_dir().map(|p| p.join("pending_game.json")) { - let _ = std::fs::remove_file(path); - } -} - -// ============================================================ -// Launch Proceed Flag -// ============================================================ - -pub fn save_launch_proceed_flag() { - if let Some(path) = get_app_data_dir().map(|p| p.join("launch_proceed.flag")) { - if let Some(parent) = path.parent() { - let _ = std::fs::create_dir_all(parent); - } - let _ = std::fs::write(path, "1"); - } -} - -pub fn check_launch_proceed_flag() -> bool { - if let Some(path) = get_app_data_dir().map(|p| p.join("launch_proceed.flag")) { - if path.exists() { - let _ = std::fs::remove_file(path); - return true; - } - } - false -} - -// ============================================================ -// Ping Results Cache -// ============================================================ - -use super::types::ServerStatus; - -pub fn save_ping_results(results: &[(String, Option, ServerStatus)]) { - if let Some(path) = get_app_data_dir().map(|p| p.join("ping_results.json")) { - let cache: Vec = results - .iter() - .map(|(id, ping, status)| { - serde_json::json!({ - "id": id, - "ping_ms": ping, - "status": format!("{:?}", status), - }) - }) - .collect(); - - if let Ok(json) = serde_json::to_string(&cache) { - let _ = std::fs::write(path, json); - } - } -} - -pub fn load_ping_results() -> Option> { - let path = get_app_data_dir()?.join("ping_results.json"); - let content = std::fs::read_to_string(&path).ok()?; - let results: Vec = serde_json::from_str(&content).ok()?; - // Clear the ping file after loading - let _ = std::fs::remove_file(&path); - Some(results) -} - -// ============================================================ -// Queue Server Ping Results Cache -// ============================================================ - -pub fn save_queue_ping_results(results: &[(String, Option)]) { - if let Some(path) = get_app_data_dir().map(|p| p.join("queue_ping_results.json")) { - let cache: Vec = results - .iter() - .map(|(id, ping)| { - serde_json::json!({ - "server_id": id, - "ping_ms": ping, - }) - }) - .collect(); - - if let Ok(json) = serde_json::to_string(&cache) { - let _ = std::fs::write(path, json); - } - } -} - -pub fn load_queue_ping_results() -> Option)>> { - let path = get_app_data_dir()?.join("queue_ping_results.json"); - let content = std::fs::read_to_string(&path).ok()?; - let results: Vec = serde_json::from_str(&content).ok()?; - // Clear the ping file after loading - let _ = std::fs::remove_file(&path); - - let parsed: Vec<(String, Option)> = results - .iter() - .filter_map(|v| { - let server_id = v.get("server_id")?.as_str()?.to_string(); - let ping_ms = v.get("ping_ms").and_then(|p| p.as_u64()).map(|p| p as u32); - Some((server_id, ping_ms)) - }) - .collect(); - - Some(parsed) -} - -// ============================================================ -// Popup Game Details Cache -// ============================================================ - -pub fn save_popup_game_details(game: &GameInfo) { - if let Some(path) = get_app_data_dir().map(|p| p.join("popup_game.json")) { - if let Some(parent) = path.parent() { - let _ = std::fs::create_dir_all(parent); - } - if let Ok(json) = serde_json::to_string(game) { - let _ = std::fs::write(path, json); - } - } -} - -pub fn load_popup_game_details() -> Option { - let path = get_app_data_dir()?.join("popup_game.json"); - let content = std::fs::read_to_string(&path).ok()?; - let game: GameInfo = serde_json::from_str(&content).ok()?; - - // Clear the file after loading to prevent stale data - let _ = std::fs::remove_file(&path); - - Some(game) -} - -pub fn clear_popup_game_details() { - if let Some(path) = get_app_data_dir().map(|p| p.join("popup_game.json")) { - let _ = std::fs::remove_file(path); - } -} - -// ============================================================ -// Queue Times Cache (from PrintedWaste API) -// ============================================================ - -use crate::api::QueueServerInfo; - -fn queue_cache_path() -> Option { - get_app_data_dir().map(|p| p.join("queue_cache.json")) -} - -pub fn save_queue_cache(servers: &[QueueServerInfo]) { - if let Some(path) = queue_cache_path() { - if let Some(parent) = path.parent() { - let _ = std::fs::create_dir_all(parent); - } - let cache: Vec = servers - .iter() - .map(|s| { - serde_json::json!({ - "server_id": s.server_id, - "display_name": s.display_name, - "region": s.region, - "ping_ms": s.ping_ms, - "queue_position": s.queue_position, - "eta_seconds": s.eta_seconds, - "is_4080_server": s.is_4080_server, - "is_5080_server": s.is_5080_server, - "last_updated": s.last_updated, - }) - }) - .collect(); - - if let Ok(json) = serde_json::to_string(&cache) { - let _ = std::fs::write(path, json); - } - } -} - -pub fn load_queue_cache() -> Option> { - let path = queue_cache_path()?; - let content = std::fs::read_to_string(&path).ok()?; - let cache: Vec = serde_json::from_str(&content).ok()?; - - Some( - cache - .into_iter() - .filter_map(|v| { - Some(QueueServerInfo { - server_id: v.get("server_id")?.as_str()?.to_string(), - display_name: v.get("display_name")?.as_str()?.to_string(), - region: v.get("region")?.as_str()?.to_string(), - ping_ms: v.get("ping_ms").and_then(|v| v.as_u64()).map(|v| v as u32), - queue_position: v.get("queue_position")?.as_i64()? as i32, - eta_seconds: v.get("eta_seconds").and_then(|v| v.as_i64()), - is_4080_server: v - .get("is_4080_server") - .and_then(|v| v.as_bool()) - .unwrap_or(false), - is_5080_server: v - .get("is_5080_server") - .and_then(|v| v.as_bool()) - .unwrap_or(false), - last_updated: v.get("last_updated").and_then(|v| v.as_i64()).unwrap_or(0), - }) - }) - .collect(), - ) -} - -pub fn clear_queue_cache() { - if let Some(path) = queue_cache_path() { - let _ = std::fs::remove_file(path); - } -} - -// ============================================================ -// Welcome Shown Flag (first-time user experience) -// ============================================================ - -fn welcome_shown_path() -> Option { - get_app_data_dir().map(|p| p.join("welcome_shown")) -} - -/// Check if the welcome popup has been shown before -pub fn has_shown_welcome() -> bool { - welcome_shown_path().map(|p| p.exists()).unwrap_or(false) -} - -/// Mark the welcome popup as shown -pub fn mark_welcome_shown() { - if let Some(path) = welcome_shown_path() { - // Just create an empty file as a marker - if let Err(e) = std::fs::write(&path, "1") { - warn!("Failed to save welcome shown flag: {}", e); - } - } -} diff --git a/opennow-streamer/src/app/config.rs b/opennow-streamer/src/app/config.rs deleted file mode 100644 index 477743c..0000000 --- a/opennow-streamer/src/app/config.rs +++ /dev/null @@ -1,567 +0,0 @@ -//! Application Configuration -//! -//! Persistent settings for the OpenNow Streamer. - -use anyhow::Result; -use serde::{Deserialize, Serialize}; -use std::path::PathBuf; - -/// Application settings -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(default)] -pub struct Settings { - // === Video Settings === - /// Stream quality preset - pub quality: StreamQuality, - - /// Custom resolution (e.g., "1920x1080") - pub resolution: String, - - /// Target FPS (30, 60, 120, 240, 360) - pub fps: u32, - - /// Preferred video codec - pub codec: VideoCodec, - - /// Maximum bitrate in Mbps (200 = unlimited) - pub max_bitrate_mbps: u32, - - /// Preferred video decoder backend - pub decoder_backend: VideoDecoderBackend, - - /// Color quality setting (combines bit depth and chroma format) - pub color_quality: ColorQuality, - - /// HDR mode enabled - pub hdr_enabled: bool, - - // === Audio Settings === - /// Audio codec - pub audio_codec: AudioCodec, - - /// Enable surround sound - pub surround: bool, - - // === Performance === - /// Enable VSync - pub vsync: bool, - - /// Low latency mode (reduces buffer) - pub low_latency_mode: bool, - - /// NVIDIA Reflex (auto-enabled for 120+ FPS) - pub nvidia_reflex: bool, - - // === Input === - /// Mouse sensitivity multiplier - pub mouse_sensitivity: f32, - - /// Use raw input (Windows only) - pub raw_input: bool, - - /// Enable clipboard paste (Ctrl+V sends clipboard text to remote session) - /// Max 65536 bytes (64KB) per paste - pub clipboard_paste_enabled: bool, - - // === Display === - /// Start in fullscreen - pub fullscreen: bool, - - /// Borderless fullscreen - pub borderless: bool, - - /// Window width (0 = use default) - pub window_width: u32, - - /// Window height (0 = use default) - pub window_height: u32, - - /// Show stats panel - pub show_stats: bool, - - /// Stats panel position - pub stats_position: StatsPosition, - - // === Network === - /// Preferred server region - pub preferred_region: Option, - - /// Selected server ID (zone ID) - pub selected_server: Option, - - /// Auto server selection (picks best ping) - pub auto_server_selection: bool, - - /// Proxy URL - pub proxy: Option, - - /// Disable telemetry - pub disable_telemetry: bool, -} - -impl Default for Settings { - fn default() -> Self { - Self { - // Video - quality: StreamQuality::Auto, - resolution: "1920x1080".to_string(), - fps: 60, - codec: VideoCodec::H264, - max_bitrate_mbps: 150, - decoder_backend: VideoDecoderBackend::Auto, // Auto-select best decoder - color_quality: ColorQuality::Bit10Yuv420, - hdr_enabled: false, - - // Audio - audio_codec: AudioCodec::Opus, - surround: false, - - // Performance - vsync: false, - low_latency_mode: true, - nvidia_reflex: true, - - // Input - mouse_sensitivity: 1.0, - raw_input: true, - clipboard_paste_enabled: true, // Enable by default like official client - - // Display - fullscreen: false, - borderless: true, - window_width: 0, // 0 = use default - window_height: 0, // 0 = use default - show_stats: true, - stats_position: StatsPosition::BottomLeft, - - // Network - preferred_region: None, - selected_server: None, - auto_server_selection: true, // Default to auto - proxy: None, - disable_telemetry: true, - } - } -} - -impl Settings { - /// Get settings file path - fn file_path() -> Option { - dirs::config_dir().map(|p| p.join("opennow-streamer").join("settings.json")) - } - - /// Load settings from disk - pub fn load() -> Result { - let path = Self::file_path().ok_or_else(|| anyhow::anyhow!("No config directory"))?; - - if !path.exists() { - return Ok(Self::default()); - } - - let content = std::fs::read_to_string(&path)?; - let settings: Settings = serde_json::from_str(&content)?; - Ok(settings) - } - - /// Save settings to disk - pub fn save(&self) -> Result<()> { - let path = Self::file_path().ok_or_else(|| anyhow::anyhow!("No config directory"))?; - - // Ensure directory exists - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent)?; - } - - let content = serde_json::to_string_pretty(self)?; - std::fs::write(&path, content)?; - - Ok(()) - } - - /// Get resolution as (width, height) - pub fn resolution_tuple(&self) -> (u32, u32) { - let parts: Vec<&str> = self.resolution.split('x').collect(); - if parts.len() == 2 { - let width = parts[0].parse().unwrap_or(1920); - let height = parts[1].parse().unwrap_or(1080); - (width, height) - } else { - (1920, 1080) - } - } - - /// Get max bitrate in kbps - pub fn max_bitrate_kbps(&self) -> u32 { - self.max_bitrate_mbps * 1000 - } -} - -/// Stream quality presets -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] -#[serde(rename_all = "lowercase")] -pub enum StreamQuality { - /// Auto-detect based on connection - #[default] - Auto, - /// 720p 30fps - Low, - /// 1080p 60fps - Medium, - /// 1440p 60fps - High, - /// 4K 60fps - Ultra, - /// 1080p 120fps - High120, - /// 1440p 120fps - Ultra120, - /// 1080p 240fps (competitive) - Competitive, - /// 1080p 360fps (extreme) - Extreme, - /// Custom settings - Custom, -} - -impl StreamQuality { - /// Get resolution and FPS for this quality preset - pub fn settings(&self) -> (&str, u32) { - match self { - StreamQuality::Auto => ("1920x1080", 60), - StreamQuality::Low => ("1280x720", 30), - StreamQuality::Medium => ("1920x1080", 60), - StreamQuality::High => ("2560x1440", 60), - StreamQuality::Ultra => ("3840x2160", 60), - StreamQuality::High120 => ("1920x1080", 120), - StreamQuality::Ultra120 => ("2560x1440", 120), - StreamQuality::Competitive => ("1920x1080", 240), - StreamQuality::Extreme => ("1920x1080", 360), - StreamQuality::Custom => ("1920x1080", 60), - } - } - - /// Get display name for UI - pub fn display_name(&self) -> &'static str { - match self { - StreamQuality::Auto => "Auto", - StreamQuality::Low => "720p 30fps", - StreamQuality::Medium => "1080p 60fps", - StreamQuality::High => "1440p 60fps", - StreamQuality::Ultra => "4K 60fps", - StreamQuality::High120 => "1080p 120fps", - StreamQuality::Ultra120 => "1440p 120fps", - StreamQuality::Competitive => "1080p 240fps", - StreamQuality::Extreme => "1080p 360fps", - StreamQuality::Custom => "Custom", - } - } - - /// Get all available presets - pub fn all() -> &'static [StreamQuality] { - &[ - StreamQuality::Auto, - StreamQuality::Low, - StreamQuality::Medium, - StreamQuality::High, - StreamQuality::Ultra, - StreamQuality::High120, - StreamQuality::Ultra120, - StreamQuality::Competitive, - StreamQuality::Extreme, - StreamQuality::Custom, - ] - } -} - -/// Available resolutions -pub const RESOLUTIONS: &[(&str, &str)] = &[ - ("1280x720", "720p"), - ("1920x1080", "1080p"), - ("2560x1440", "1440p"), - ("3840x2160", "4K"), - ("2560x1080", "Ultrawide 1080p"), - ("3440x1440", "Ultrawide 1440p"), - ("5120x1440", "Super Ultrawide"), -]; - -/// Available FPS options -pub const FPS_OPTIONS: &[u32] = &[30, 60, 90, 120, 144, 165, 240, 360]; - -/// Video codec options -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] -#[serde(rename_all = "lowercase")] -pub enum VideoCodec { - /// H.264/AVC - widest compatibility - #[default] - H264, - /// H.265/HEVC - better compression - H265, - /// AV1 - best compression, modern GPUs only - AV1, -} - -/// Video decoder backend preference -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] -#[serde(rename_all = "lowercase")] -pub enum VideoDecoderBackend { - /// Auto-detect best decoder - #[default] - Auto, - /// NVIDIA CUDA/CUVID - Cuvid, - /// Intel QuickSync - Qsv, - /// AMD VA-API - Vaapi, - /// DirectX 11/12 (Windows) via FFmpeg - Dxva, - /// Native D3D11 Video decoder (Windows) - bypasses FFmpeg - /// This is the NVIDIA-style approach with proper RTArray support - NativeDxva, - /// VideoToolbox (macOS) - VideoToolbox, - /// Vulkan Video (Linux) - cross-GPU hardware decode via Vulkan extensions - /// Based on GeForce NOW's VkVideoDecoder implementation - VulkanVideo, - /// Software decoding (CPU) - Software, -} - -impl VideoDecoderBackend { - /// Short display name for dropdown - pub fn as_str(&self) -> &'static str { - match self { - VideoDecoderBackend::Auto => "Auto", - VideoDecoderBackend::Cuvid => "NVDEC", - VideoDecoderBackend::Qsv => "QuickSync", - VideoDecoderBackend::Vaapi => "VA-API", - VideoDecoderBackend::Dxva => "D3D11 (GStreamer)", - VideoDecoderBackend::NativeDxva => "Native D3D11 (HEVC only)", - VideoDecoderBackend::VideoToolbox => "VideoToolbox", - VideoDecoderBackend::VulkanVideo => "GStreamer HW", - VideoDecoderBackend::Software => "Software", - } - } - - /// Detailed description for tooltip - pub fn description(&self) -> &'static str { - match self { - VideoDecoderBackend::Auto => { - "Automatically selects the best available decoder for your system.\n\n\ - Windows: GStreamer D3D11 (d3d11h264dec/d3d11h265dec)\n\ - Linux: GStreamer with VA-API or V4L2\n\ - macOS: FFmpeg with VideoToolbox\n\ - Performance: Optimal for your hardware" - } - VideoDecoderBackend::Cuvid => { - "NVIDIA hardware decoding using NVDEC.\n\n\ - Backend: GStreamer + nvd3d11h264dec/nvd3d11h265dec\n\ - Performance: Excellent on NVIDIA GPUs\n\ - Compatibility: NVIDIA GPUs only (GTX 600+)" - } - VideoDecoderBackend::Qsv => { - "Intel hardware decoding using Quick Sync Video.\n\n\ - Backend: GStreamer + qsvh264dec/qsvh265dec\n\ - Performance: Good on Intel CPUs with integrated graphics\n\ - Compatibility: Intel 2nd gen Core+ with HD Graphics" - } - VideoDecoderBackend::Vaapi => { - "Linux hardware decoding via Video Acceleration API.\n\n\ - Backend: GStreamer + vah264dec/vah265dec (or legacy vaapih264dec)\n\ - Performance: Good on AMD/Intel GPUs (Linux)\n\ - Compatibility: AMD, Intel GPUs on Linux\n\ - Note: Use this if GStreamer HW doesn't work" - } - VideoDecoderBackend::Dxva => { - "Windows DirectX Video Acceleration via GStreamer.\n\n\ - Backend: GStreamer + d3d11h264dec/d3d11h265dec\n\ - Performance: Good hardware acceleration, stable\n\ - Compatibility: Windows with any modern GPU\n\ - Note: Recommended for Windows (supports H.264 and H.265)" - } - VideoDecoderBackend::NativeDxva => { - "Direct D3D11 Video decoder (EXPERIMENTAL).\n\n\ - Backend: Native Windows D3D11 Video API\n\ - Performance: Zero-copy to GPU, lowest latency\n\ - Compatibility: Windows 8+ with D3D11 GPU\n\ - WARNING: Only supports H.265/HEVC! H.264 will fail.\n\ - Use 'D3D11 (GStreamer)' for H.264 streams." - } - VideoDecoderBackend::VideoToolbox => { - "macOS hardware decoding using Apple's VideoToolbox.\n\n\ - Backend: FFmpeg + VideoToolbox\n\ - Performance: Excellent on Apple Silicon/Intel Macs\n\ - Compatibility: macOS only" - } - VideoDecoderBackend::VulkanVideo => { - "GStreamer hardware decoding (Linux).\n\n\ - Backend: GStreamer auto-selects best decoder:\n\ - - V4L2 (Raspberry Pi / embedded)\n\ - - VA plugin (Intel/AMD desktop)\n\ - - VAAPI plugin (legacy fallback)\n\ - Performance: Hardware accelerated\n\ - Compatibility: Linux with GStreamer installed" - } - VideoDecoderBackend::Software => { - "CPU-based software decoding.\n\n\ - Backend: GStreamer + avdec_h264/avdec_h265\n\ - Performance: Slow, high CPU usage\n\ - Compatibility: Any system (fallback)\n\ - Note: Use only if hardware decode fails" - } - } - } - - /// Get the underlying technology/backend name - pub fn backend_name(&self) -> &'static str { - match self { - VideoDecoderBackend::Auto => "Auto", - VideoDecoderBackend::Cuvid => "GStreamer NVDEC", - VideoDecoderBackend::Qsv => "GStreamer QSV", - VideoDecoderBackend::Vaapi => "GStreamer VA-API", - VideoDecoderBackend::Dxva => "GStreamer D3D11", - VideoDecoderBackend::NativeDxva => "Native D3D11", - VideoDecoderBackend::VideoToolbox => "FFmpeg VT", - VideoDecoderBackend::VulkanVideo => "GStreamer HW", - VideoDecoderBackend::Software => "GStreamer CPU", - } - } - - pub fn all() -> &'static [VideoDecoderBackend] { - &[ - VideoDecoderBackend::Auto, - VideoDecoderBackend::Cuvid, - VideoDecoderBackend::Qsv, - VideoDecoderBackend::Vaapi, - VideoDecoderBackend::Dxva, - VideoDecoderBackend::NativeDxva, - VideoDecoderBackend::VideoToolbox, - VideoDecoderBackend::VulkanVideo, - VideoDecoderBackend::Software, - ] - } -} - -impl VideoCodec { - pub fn as_str(&self) -> &'static str { - match self { - VideoCodec::H264 => "H264", - VideoCodec::H265 => "H265", - VideoCodec::AV1 => "AV1", - } - } - - /// Get display name with description - pub fn display_name(&self) -> &'static str { - match self { - VideoCodec::H264 => "H.264 (Wide compatibility)", - VideoCodec::H265 => "H.265/HEVC (Better quality)", - VideoCodec::AV1 => "AV1 (Best compression, modern GPUs)", - } - } - - /// Get all available codecs - pub fn all() -> &'static [VideoCodec] { - &[VideoCodec::H264, VideoCodec::H265, VideoCodec::AV1] - } -} - -/// Audio codec options -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] -#[serde(rename_all = "lowercase")] -pub enum AudioCodec { - /// Opus - low latency - #[default] - Opus, - /// Opus Stereo - OpusStereo, -} - -/// Color quality options (bit depth + chroma subsampling) -/// Matches NVIDIA GFN client options -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] -#[serde(rename_all = "snake_case")] -pub enum ColorQuality { - /// 8-bit YUV 4:2:0 - Most compatible, lowest bandwidth - Bit8Yuv420, - /// 8-bit YUV 4:4:4 - Better color accuracy, higher bandwidth - Bit8Yuv444, - /// 10-bit YUV 4:2:0 - HDR capable, good balance (default) - #[default] - Bit10Yuv420, - /// 10-bit YUV 4:4:4 - Best quality, highest bandwidth (requires HEVC) - Bit10Yuv444, -} - -impl ColorQuality { - /// Get bit depth value (0 = 8-bit SDR, 10 = 10-bit HDR capable) - pub fn bit_depth(&self) -> i32 { - match self { - ColorQuality::Bit8Yuv420 | ColorQuality::Bit8Yuv444 => 0, // 0 means 8-bit SDR - ColorQuality::Bit10Yuv420 | ColorQuality::Bit10Yuv444 => 10, - } - } - - /// Get chroma format value (0 = 4:2:0, 2 = 4:4:4) - /// Note: 4:2:2 is not commonly used in streaming - pub fn chroma_format(&self) -> i32 { - match self { - ColorQuality::Bit8Yuv420 | ColorQuality::Bit10Yuv420 => 0, // YUV 4:2:0 - ColorQuality::Bit8Yuv444 | ColorQuality::Bit10Yuv444 => 2, // YUV 4:4:4 - } - } - - /// Check if this mode requires HEVC codec - pub fn requires_hevc(&self) -> bool { - matches!( - self, - ColorQuality::Bit10Yuv420 | ColorQuality::Bit10Yuv444 | ColorQuality::Bit8Yuv444 - ) - } - - /// Check if this is a 10-bit mode - pub fn is_10bit(&self) -> bool { - matches!(self, ColorQuality::Bit10Yuv420 | ColorQuality::Bit10Yuv444) - } - - /// Get display name for UI - pub fn display_name(&self) -> &'static str { - match self { - ColorQuality::Bit8Yuv420 => "8-bit, YUV 4:2:0", - ColorQuality::Bit8Yuv444 => "8-bit, YUV 4:4:4", - ColorQuality::Bit10Yuv420 => "10-bit, YUV 4:2:0", - ColorQuality::Bit10Yuv444 => "10-bit, YUV 4:4:4", - } - } - - /// Get description for UI - pub fn description(&self) -> &'static str { - match self { - ColorQuality::Bit8Yuv420 => "Most compatible, lower bandwidth", - ColorQuality::Bit8Yuv444 => "Better color, needs HEVC", - ColorQuality::Bit10Yuv420 => "HDR ready, recommended", - ColorQuality::Bit10Yuv444 => "Best quality, needs HEVC", - } - } - - /// Get all available options - pub fn all() -> &'static [ColorQuality] { - &[ - ColorQuality::Bit8Yuv420, - ColorQuality::Bit8Yuv444, - ColorQuality::Bit10Yuv420, - ColorQuality::Bit10Yuv444, - ] - } -} - -/// Stats panel position -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] -#[serde(rename_all = "kebab-case")] -pub enum StatsPosition { - TopLeft, - TopRight, - #[default] - BottomLeft, - BottomRight, -} diff --git a/opennow-streamer/src/app/mod.rs b/opennow-streamer/src/app/mod.rs deleted file mode 100644 index 3ae6760..0000000 --- a/opennow-streamer/src/app/mod.rs +++ /dev/null @@ -1,2227 +0,0 @@ -//! Application State Management -//! -//! Central state machine for the OpenNow Streamer. - -pub mod cache; -pub mod config; -pub mod session; -pub mod types; - -pub use config::{AudioCodec, ColorQuality, Settings, StatsPosition, StreamQuality, VideoCodec}; -pub use session::{ActiveSessionInfo, SessionInfo, SessionState}; -pub use types::{ - parse_resolution, AppState, GameInfo, GameSection, GameVariant, GamesTab, QueueRegionFilter, - QueueSortMode, ServerInfo, ServerStatus, SettingChange, SharedFrame, SubscriptionInfo, - UiAction, -}; - -use log::{error, info, warn}; -use parking_lot::RwLock; -use std::sync::Arc; -use tokio::runtime::Handle; -use tokio::sync::mpsc; - -use crate::api::{self, DynamicServerRegion, GfnApiClient}; -use crate::auth::{self, AuthTokens, LoginProvider, PkceChallenge, UserInfo}; - -use crate::input::InputHandler; - -use crate::media::StreamStats; -use crate::webrtc::StreamingSession; - -/// Cache for dynamic regions fetched from serverInfo API -static DYNAMIC_REGIONS_CACHE: RwLock>> = RwLock::new(None); - -/// Main application structure -pub struct App { - /// Current application state - pub state: AppState, - - /// Tokio runtime handle for async operations - pub runtime: Handle, - - /// User settings - pub settings: Settings, - - /// Authentication tokens - pub auth_tokens: Option, - - /// User info - pub user_info: Option, - - /// Current session info - pub session: Option, - - /// Streaming session (WebRTC) - pub streaming_session: Option>>, - - /// Input handler for the current platform - pub input_handler: Option>, - - /// Whether cursor is captured - pub cursor_captured: bool, - - /// Current video frame (for rendering) - pub current_frame: Option, - - /// Shared frame holder for zero-latency frame delivery - pub shared_frame: Option>, - - /// Stream statistics - pub stats: StreamStats, - - /// Whether to show stats overlay - pub show_stats: bool, - - /// Status message for UI - pub status_message: String, - - /// Error message (if any) - pub error_message: Option, - - /// Games list (flat, for All Games tab) - pub games: Vec, - - /// Game sections (Home tab - Trending, Free to Play, etc.) - pub game_sections: Vec, - - /// Search query - pub search_query: String, - - /// Selected game - pub selected_game: Option, - - /// Channel for receiving stats updates - stats_rx: Option>, - - // === Login State === - /// Available login providers - pub login_providers: Vec, - - /// Selected provider index - pub selected_provider_index: usize, - - /// Whether settings panel is visible - pub show_settings: bool, - - /// Loading state for async operations - pub is_loading: bool, - - /// Login URL for manual copy/paste if browser doesn't open - pub login_url: Option, - - /// VPC ID for current provider - pub vpc_id: Option, - - /// API client - api_client: GfnApiClient, - - /// Subscription info (hours, storage, etc.) - pub subscription: Option, - - /// User's library games - pub library_games: Vec, - - /// Current tab in Games view - pub current_tab: GamesTab, - - /// Selected game for detail popup (None = popup closed) - pub selected_game_popup: Option, - - /// Available servers/regions - pub servers: Vec, - - /// Selected server index - pub selected_server_index: usize, - - /// Auto server selection (picks best ping) - pub auto_server_selection: bool, - - /// Whether ping test is running - pub ping_testing: bool, - - /// Whether queue server ping test is running - pub queue_ping_testing: bool, - - /// Whether settings modal is visible - pub show_settings_modal: bool, - - /// Active sessions detected - pub active_sessions: Vec, - - /// Whether showing session conflict dialog - pub show_session_conflict: bool, - - /// Whether showing AV1 unsupported warning dialog - pub show_av1_warning: bool, - - /// Whether showing Alliance experimental warning dialog - pub show_alliance_warning: bool, - - /// Whether showing first-time welcome popup - pub show_welcome_popup: bool, - - /// Pending game launch (waiting for session conflict resolution) - pub pending_game_launch: Option, - - /// Last time we polled the session (for rate limiting) - last_poll_time: std::time::Instant, - - /// Render FPS tracking - render_frame_count: u64, - last_render_fps_time: std::time::Instant, - last_render_frame_count: u64, - - /// Number of times we've polled after session became ready (to ensure candidates) - session_ready_poll_count: u32, - - /// Anti-AFK mode enabled (Ctrl+Shift+F10 to toggle) - pub anti_afk_enabled: bool, - - /// Last time anti-AFK sent a key press - anti_afk_last_send: std::time::Instant, - - /// Whether a token refresh is currently in progress - token_refresh_in_progress: bool, - - /// Queue times data from PrintedWaste API - pub queue_servers: Vec, - - /// Whether queue data is loading - pub queue_loading: bool, - - /// Last time queue data was fetched - queue_last_fetch: std::time::Instant, - - /// Queue sort mode - pub queue_sort_mode: QueueSortMode, - - /// Queue region filter - pub queue_region_filter: QueueRegionFilter, - - /// Whether to show server selection modal (for free tier) - pub show_server_selection: bool, - - /// Selected server for queue (when user picks manually) - pub selected_queue_server: Option, - - /// Pending game for server selection (stored when showing modal) - pub pending_server_selection_game: Option, - - /// Whether session requires ads (free tier) - pub ads_required: bool, - - /// Ads remaining seconds (for countdown display) - pub ads_remaining_secs: u32, - - /// Ads total duration in seconds - pub ads_total_secs: u32, -} - -/// Poll interval for session status (2 seconds) -const POLL_INTERVAL: std::time::Duration = std::time::Duration::from_secs(2); - -// Mutex re-export for streaming session -use parking_lot::Mutex; - -// VideoFrame re-import for current_frame field -use crate::media::VideoFrame; - -impl App { - /// Create new application instance - pub fn new(runtime: Handle) -> Self { - // Load settings - let settings = Settings::load().unwrap_or_default(); - let auto_server = settings.auto_server_selection; // Save before move - - // Try to load saved tokens - let auth_tokens = cache::load_tokens(); - let has_token = auth_tokens - .as_ref() - .map(|t| !t.is_expired()) - .unwrap_or(false); - - // Load cached login provider (for Alliance persistence) - if let Some(provider) = cache::load_login_provider() { - auth::set_login_provider(provider); - } - - let initial_state = if has_token { - AppState::Games - } else { - AppState::Login - }; - - // Start fetching login providers - let rt = runtime.clone(); - rt.spawn(async { - if let Err(e) = auth::fetch_login_providers().await { - warn!("Failed to fetch login providers: {}", e); - } - }); - - // Start checking active sessions if we have a token - // Clear stale cache first to ensure we always use fresh data from API - cache::clear_active_sessions_cache(); - - if has_token { - let rt = runtime.clone(); - let token = auth_tokens.as_ref().unwrap().jwt().to_string(); - rt.spawn(async move { - let mut api_client = GfnApiClient::new(); - api_client.set_access_token(token); - match api_client.get_active_sessions().await { - Ok(sessions) => { - info!( - "Checked active sessions at startup: found {}", - sessions.len() - ); - // Only save to cache if we have sessions - this is fresh data from API - if !sessions.is_empty() { - cache::save_active_sessions_cache(&sessions); - } - } - Err(e) => { - warn!("Failed to check active sessions at startup: {}", e); - } - } - }); - - // Also fetch subscription info to ensure dynamic resolutions are available - let rt = runtime.clone(); - let token = auth_tokens.as_ref().unwrap().jwt().to_string(); - let user_id = auth_tokens.as_ref().unwrap().user_id().to_string(); - rt.spawn(async move { - match crate::api::fetch_subscription(&token, &user_id).await { - Ok(sub) => { - info!("Fetched subscription startup: tier={}", sub.membership_tier); - cache::save_subscription_cache(&sub); - } - Err(e) => { - warn!("Failed to fetch subscription at startup: {}", e); - } - } - }); - } - - Self { - state: initial_state, - runtime, - settings, - auth_tokens, - user_info: None, - session: None, - streaming_session: None, - input_handler: None, - cursor_captured: false, - current_frame: None, - shared_frame: None, - stats: StreamStats::default(), - show_stats: true, - status_message: "Welcome to OpenNOW".to_string(), - error_message: None, - games: Vec::new(), - game_sections: Vec::new(), - search_query: String::new(), - selected_game: None, - stats_rx: None, - login_providers: vec![LoginProvider::nvidia_default()], - selected_provider_index: 0, - show_settings: false, - is_loading: false, - login_url: None, - vpc_id: None, - api_client: GfnApiClient::new(), - subscription: None, - library_games: Vec::new(), - current_tab: GamesTab::Home, - selected_game_popup: None, - servers: Vec::new(), - selected_server_index: 0, - auto_server_selection: auto_server, // Load from settings - ping_testing: false, - queue_ping_testing: false, - show_settings_modal: false, - active_sessions: Vec::new(), - show_session_conflict: false, - show_av1_warning: false, - show_alliance_warning: false, - show_welcome_popup: !cache::has_shown_welcome(), - - pending_game_launch: None, - last_poll_time: std::time::Instant::now(), - render_frame_count: 0, - last_render_fps_time: std::time::Instant::now(), - last_render_frame_count: 0, - session_ready_poll_count: 0, - anti_afk_enabled: false, - anti_afk_last_send: std::time::Instant::now(), - token_refresh_in_progress: false, - queue_servers: Vec::new(), - queue_loading: false, - queue_last_fetch: std::time::Instant::now() - std::time::Duration::from_secs(60), // Force initial fetch - queue_sort_mode: QueueSortMode::default(), - queue_region_filter: QueueRegionFilter::default(), - show_server_selection: false, - selected_queue_server: None, - pending_server_selection_game: None, - ads_required: false, - ads_remaining_secs: 0, - ads_total_secs: 0, - } - } - - /// Toggle anti-AFK mode - pub fn toggle_anti_afk(&mut self) { - self.anti_afk_enabled = !self.anti_afk_enabled; - if self.anti_afk_enabled { - self.anti_afk_last_send = std::time::Instant::now(); - info!("Anti-AFK mode ENABLED - sending F13 every 4 minutes"); - } else { - info!("Anti-AFK mode DISABLED"); - } - } - - /// Send anti-AFK key press (F13) if enabled and interval elapsed - pub fn update_anti_afk(&mut self) { - if !self.anti_afk_enabled || self.state != AppState::Streaming { - return; - } - - // Check if 4 minutes have passed - if self.anti_afk_last_send.elapsed() >= std::time::Duration::from_secs(240) { - if let Some(ref input_handler) = self.input_handler { - // F13 virtual key code is 0x7C (124) - const VK_F13: u16 = 0x7C; - - // Send key down then key up - input_handler.handle_key(VK_F13, true, 0); // Key down - input_handler.handle_key(VK_F13, false, 0); // Key up - - self.anti_afk_last_send = std::time::Instant::now(); - log::debug!("Anti-AFK: sent F13 key press"); - } - } - } - - /// Handle a UI action - pub fn handle_action(&mut self, action: UiAction) { - match action { - UiAction::StartLogin => { - self.start_oauth_login(); - } - UiAction::SelectProvider(index) => { - self.select_provider(index); - } - UiAction::Logout => { - self.logout(); - } - UiAction::LaunchGame(index) => { - // Get game from appropriate list based on current tab - let game = match self.current_tab { - GamesTab::Home => self.games.get(index).cloned(), // Use flat list for Home too - GamesTab::AllGames => self.games.get(index).cloned(), - GamesTab::MyLibrary => self.library_games.get(index).cloned(), - GamesTab::QueueTimes => None, // Queue Times tab doesn't launch games - }; - if let Some(game) = game { - self.launch_game(&game); - } - } - UiAction::LaunchGameDirect(game) => { - self.launch_game(&game); - } - UiAction::StopStreaming => { - self.stop_streaming(); - } - UiAction::ToggleStats => { - self.toggle_stats(); - } - UiAction::UpdateSearch(query) => { - self.search_query = query; - } - UiAction::ToggleSettings => { - self.show_settings = !self.show_settings; - } - UiAction::UpdateSetting(change) => { - match change { - SettingChange::Resolution(res) => self.settings.resolution = res, - SettingChange::Fps(fps) => self.settings.fps = fps, - SettingChange::Codec(codec) => { - self.settings.codec = codec; - } - SettingChange::MaxBitrate(bitrate) => self.settings.max_bitrate_mbps = bitrate, - SettingChange::Fullscreen(fs) => self.settings.fullscreen = fs, - SettingChange::VSync(vsync) => self.settings.vsync = vsync, - SettingChange::LowLatency(ll) => self.settings.low_latency_mode = ll, - SettingChange::DecoderBackend(backend) => { - self.settings.decoder_backend = backend - } - SettingChange::ColorQuality(quality) => { - self.settings.color_quality = quality; - // Auto-switch codec based on color quality requirements - if quality.requires_hevc() && self.settings.codec == VideoCodec::H264 { - // 10-bit or 4:4:4 requires HEVC or AV1 - self.settings.codec = VideoCodec::H265; - } - } - SettingChange::Hdr(enabled) => { - self.settings.hdr_enabled = enabled; - // HDR requires 10-bit and HEVC/AV1 - if enabled { - // Switch to 10-bit if currently 8-bit - if !self.settings.color_quality.is_10bit() { - self.settings.color_quality = ColorQuality::Bit10Yuv420; - } - // Switch to HEVC if currently H.264 - if self.settings.codec == VideoCodec::H264 { - self.settings.codec = VideoCodec::H265; - } - } - } - SettingChange::ClipboardPasteEnabled(enabled) => { - self.settings.clipboard_paste_enabled = enabled; - } - } - self.save_settings(); - } - UiAction::RefreshGames => { - self.fetch_games(); - } - UiAction::SwitchTab(tab) => { - self.current_tab = tab; - // Fetch library if switching to My Library and it's empty - if tab == GamesTab::MyLibrary && self.library_games.is_empty() { - self.fetch_library(); - } - // Fetch sections if switching to Home and sections are empty - if tab == GamesTab::Home && self.game_sections.is_empty() { - self.fetch_sections(); - } - // Fetch queue data if switching to Queue Times - if tab == GamesTab::QueueTimes { - self.fetch_queue_times(); - } - } - UiAction::OpenGamePopup(game) => { - self.selected_game_popup = Some(game.clone()); - - // Spawn async task to fetch full details (Play Type, Membership, etc.) only if missing - // User reports library games already have this info, so avoid redundant 400-prone fetches - let mut needs_fetch = game.play_type.is_none(); - - // If we have a description, we definitely don't need to fetch - if game.description.is_some() { - needs_fetch = false; - } - - let token = self.auth_tokens.as_ref().map(|t| t.jwt().to_string()); - let query_id = game.id.clone(); - let runtime = self.runtime.clone(); - - if needs_fetch { - if let Some(token) = token { - runtime.spawn(async move { - let mut api_client = GfnApiClient::new(); - api_client.set_access_token(token); - - // Fetch details - match api_client.fetch_app_details(&query_id).await { - Ok(Some(details)) => { - info!("Fetched details for popup: {}", details.title); - cache::save_popup_game_details(&details); - } - Ok(None) => warn!("No details found for popup game: {}", query_id), - Err(e) => warn!("Failed to fetch popup details: {}", e), - } - }); - } - } else { - info!("Using existing details for popup: {}", game.title); - } - } - UiAction::CloseGamePopup => { - self.selected_game_popup = None; - } - UiAction::SelectVariant(index) => { - // Update the selected variant for the game popup - if let Some(ref mut game) = self.selected_game_popup { - if index < game.variants.len() { - game.selected_variant_index = index; - // Update the game's store and id to match the selected variant - if let Some(variant) = game.variants.get(index) { - game.store = variant.store.clone(); - game.id = variant.id.clone(); - game.app_id = variant.id.parse::().ok(); - info!( - "Selected platform variant: {} ({})", - variant.store, variant.id - ); - } - } - } - } - UiAction::SelectServer(index) => { - if index < self.servers.len() { - self.selected_server_index = index; - self.auto_server_selection = false; // Disable auto when manually selecting - // Save selected server and auto mode to settings - self.settings.selected_server = Some(self.servers[index].id.clone()); - self.settings.auto_server_selection = false; - self.save_settings(); - info!("Selected server: {}", self.servers[index].name); - } - } - UiAction::SetAutoServerSelection(enabled) => { - self.auto_server_selection = enabled; - self.settings.auto_server_selection = enabled; - self.save_settings(); - if enabled { - // Auto-select best server based on ping - self.select_best_server(); - } - } - UiAction::StartPingTest => { - self.start_ping_test(); - } - UiAction::ToggleSettingsModal => { - self.show_settings_modal = !self.show_settings_modal; - // Load servers when opening settings if not loaded - if self.show_settings_modal && self.servers.is_empty() { - self.load_servers(); - } - } - UiAction::ResumeSession(session_info) => { - self.resume_session(session_info); - } - UiAction::TerminateAndLaunch(session_id, game) => { - self.terminate_and_launch(session_id, game); - } - UiAction::CloseSessionConflict => { - self.show_session_conflict = false; - self.pending_game_launch = None; - } - UiAction::CloseAV1Warning => { - self.show_av1_warning = false; - } - UiAction::CloseAllianceWarning => { - self.show_alliance_warning = false; - } - UiAction::CloseWelcomePopup => { - self.show_welcome_popup = false; - cache::mark_welcome_shown(); - } - UiAction::ResetSettings => { - info!("Resetting all settings to defaults"); - self.settings = Settings::default(); - if let Err(e) = self.settings.save() { - warn!("Failed to save default settings: {}", e); - } - } - UiAction::SetQueueSortMode(mode) => { - self.queue_sort_mode = mode; - } - UiAction::SetQueueRegionFilter(filter) => { - self.queue_region_filter = filter; - } - UiAction::ShowServerSelection(game) => { - self.show_server_selection = true; - self.pending_server_selection_game = Some(game); - // Refresh queue data when showing modal - self.fetch_queue_times(); - } - UiAction::CloseServerSelection => { - self.show_server_selection = false; - self.selected_queue_server = None; - self.pending_server_selection_game = None; - } - UiAction::SelectQueueServer(server_id) => { - self.selected_queue_server = server_id; - } - UiAction::LaunchWithServer(game, server_id) => { - // Close modal and launch game - self.show_server_selection = false; - self.selected_queue_server = None; - self.pending_server_selection_game = None; - // Note: The backend API currently does not support explicit server selection. - // The selected server_id is recorded for logging/telemetry only; the server - // used for the session will still be chosen automatically by the backend. - if let Some(ref id) = server_id { - info!( - "Launching game '{}' (requested server: {}), but explicit server selection is not yet supported by the backend; a server will be auto-selected.", - game.title, - id - ); - } else { - info!( - "Launching game '{}' with backend auto-selected server (no explicit server requested).", - game.title - ); - } - self.launch_game(&game); - } - UiAction::RefreshQueueTimes => { - // Force refresh by resetting last fetch time - self.queue_last_fetch = - std::time::Instant::now() - std::time::Duration::from_secs(60); - self.fetch_queue_times(); - } - UiAction::UpdateWindowSize(width, height) => { - // Only save if size is valid and different from current - if width >= 640 && height >= 480 { - if self.settings.window_width != width || self.settings.window_height != height - { - self.settings.window_width = width; - self.settings.window_height = height; - self.save_settings(); - } - } - } - } - } - - /// Get filtered games based on search query - pub fn filtered_games(&self) -> Vec<(usize, &GameInfo)> { - let query = self.search_query.to_lowercase(); - self.games - .iter() - .enumerate() - .filter(|(_, game)| query.is_empty() || game.title.to_lowercase().contains(&query)) - .collect() - } - - /// Select a login provider - pub fn select_provider(&mut self, index: usize) { - // Update cached providers from global state - self.login_providers = auth::get_cached_providers(); - if self.login_providers.is_empty() { - self.login_providers = vec![LoginProvider::nvidia_default()]; - } - - if index < self.login_providers.len() { - self.selected_provider_index = index; - let provider = self.login_providers[index].clone(); - auth::set_login_provider(provider.clone()); - info!( - "Selected provider: {}", - provider.login_provider_display_name - ); - } - } - - /// Start OAuth login flow - pub fn start_oauth_login(&mut self) { - // Find available port - let port = match auth::find_available_port() { - Some(p) => p, - None => { - self.error_message = Some("No available ports for OAuth callback".to_string()); - return; - } - }; - - self.is_loading = true; - self.status_message = "Opening browser for login...".to_string(); - - // Clear old caches when switching accounts - self.subscription = None; - self.games.clear(); - self.game_sections.clear(); - self.library_games.clear(); - cache::clear_games_cache(); - - let pkce = PkceChallenge::new(); - let auth_url = auth::build_auth_url(&pkce, port); - let verifier = pkce.verifier.clone(); - - // Store the URL for manual copy/paste fallback - self.login_url = Some(auth_url.clone()); - - // Try to open browser (don't fail if it doesn't work - user can copy URL) - match open::that(&auth_url) { - Ok(_) => info!("Opened browser for OAuth login"), - Err(e) => { - warn!("Failed to open browser: {} - user can copy URL manually", e); - } - } - - // Spawn task to wait for callback - let runtime = self.runtime.clone(); - runtime.spawn(async move { - match auth::start_callback_server(port).await { - Ok(code) => { - info!("Received OAuth code"); - match auth::exchange_code(&code, &verifier, port).await { - Ok(tokens) => { - info!("Token exchange successful!"); - // Tokens will be picked up in update() - // For now, we store them in a temp file - cache::save_tokens(&tokens); - } - Err(e) => { - error!("Token exchange failed: {}", e); - } - } - } - Err(e) => { - error!("OAuth callback failed: {}", e); - } - } - }); - } - - /// Update application state (called each frame) - pub fn update(&mut self) { - // Track render FPS - we'll increment frame count only when we get a new video frame - // This ensures render FPS matches decode FPS, not the UI repaint rate - let now = std::time::Instant::now(); - let elapsed = now.duration_since(self.last_render_fps_time).as_secs_f64(); - if elapsed >= 1.0 { - let frames_this_period = self.render_frame_count - self.last_render_frame_count; - self.stats.render_fps = (frames_this_period as f64 / elapsed) as f32; - self.stats.frames_rendered = self.render_frame_count; - self.last_render_frame_count = self.render_frame_count; - self.last_render_fps_time = now; - } - - // Update anti-AFK (sends F13 every 4 minutes when enabled) - self.update_anti_afk(); - - // Proactive token refresh: refresh before expiration to avoid session interruption - if !self.token_refresh_in_progress { - if let Some(ref tokens) = self.auth_tokens { - if tokens.should_refresh() && tokens.can_refresh() { - info!("Token nearing expiry, proactively refreshing..."); - self.token_refresh_in_progress = true; - - let refresh_token = tokens.refresh_token.clone().unwrap(); - let runtime = self.runtime.clone(); - runtime.spawn(async move { - match auth::refresh_token(&refresh_token).await { - Ok(new_tokens) => { - info!("Proactive token refresh successful!"); - cache::save_tokens(&new_tokens); - } - Err(e) => { - warn!("Proactive token refresh failed: {}", e); - } - } - }); - } - } - } - - // Check for refreshed tokens from async refresh task - if self.token_refresh_in_progress { - if let Some(new_tokens) = cache::load_tokens() { - if let Some(ref old_tokens) = self.auth_tokens { - // Check if tokens were actually refreshed (new expires_at) - if new_tokens.expires_at > old_tokens.expires_at { - info!("Loaded refreshed tokens"); - self.auth_tokens = Some(new_tokens.clone()); - self.api_client - .set_access_token(new_tokens.jwt().to_string()); - self.token_refresh_in_progress = false; - } - } - } - } - - // Check for new video frames from shared frame holder - if let Some(ref shared) = self.shared_frame { - if let Some(frame) = shared.read() { - // Only log the first frame (when current_frame is None) - if self.current_frame.is_none() { - log::info!( - "First video frame received: {}x{}", - frame.width, - frame.height - ); - } - - // Update HDR status in stats from frame's transfer function - use crate::media::{ColorSpace, TransferFunction}; - let is_hdr = frame.transfer_function == TransferFunction::PQ - || frame.transfer_function == TransferFunction::HLG; - if self.stats.is_hdr != is_hdr { - self.stats.is_hdr = is_hdr; - self.stats.color_space = match frame.color_space { - ColorSpace::BT2020 => "BT.2020".to_string(), - ColorSpace::BT709 => "BT.709".to_string(), - ColorSpace::BT601 => "BT.601".to_string(), - }; - } - - // Update resolution in stats from actual decoded frame dimensions - // This catches resolution changes from SSRC switches (GFN adaptive quality) - let new_res = format!("{}x{}", frame.width, frame.height); - if self.stats.resolution != new_res { - if !self.stats.resolution.is_empty() { - log::info!( - "Resolution changed: {} -> {}", - self.stats.resolution, - new_res - ); - } - self.stats.resolution = new_res; - } - - self.current_frame = Some(frame); - // Increment render frame count only when we get a new video frame - // This ensures render FPS matches decode FPS - self.render_frame_count += 1; - } - } - - // Check for stats updates - if let Some(ref mut rx) = self.stats_rx { - while let Ok(mut stats) = rx.try_recv() { - // Preserve render_fps from our local tracking - stats.render_fps = self.stats.render_fps; - stats.frames_rendered = self.stats.frames_rendered; - // Preserve resolution from actual decoded frames (more accurate than SDP) - if !self.stats.resolution.is_empty() { - stats.resolution = self.stats.resolution.clone(); - } - self.stats = stats; - } - } - - // Update cached providers - let cached = auth::get_cached_providers(); - if !cached.is_empty() && cached.len() != self.login_providers.len() { - self.login_providers = cached; - } - - // Check if tokens were saved by OAuth callback - if self.state == AppState::Login && self.is_loading { - if let Some(tokens) = cache::load_tokens() { - if !tokens.is_expired() { - info!("OAuth login successful!"); - self.auth_tokens = Some(tokens.clone()); - self.api_client.set_access_token(tokens.jwt().to_string()); - self.is_loading = false; - self.login_url = None; // Clear login URL after successful login - self.state = AppState::Games; - self.status_message = "Login successful!".to_string(); - self.fetch_games(); - self.fetch_sections(); // Fetch sections for Home tab - self.fetch_subscription(); // Also fetch subscription info - self.load_servers(); // Load servers (fetches dynamic regions) - - // Check for active sessions after login - self.check_active_sessions(); - - // Show Alliance experimental warning if using an Alliance partner - if auth::get_selected_provider().is_alliance_partner() { - self.show_alliance_warning = true; - } - } - } - } - - // Check if games were fetched and saved to cache - if self.state == AppState::Games && self.is_loading && self.games.is_empty() { - if let Some(games) = cache::load_games_cache() { - if !games.is_empty() { - // Check if cache has images - if not, it's old cache that needs refresh - let has_images = games.iter().any(|g| g.image_url.is_some()); - if has_images { - info!("Loaded {} games from cache (with images)", games.len()); - self.games = games; - self.is_loading = false; - self.status_message = format!("Loaded {} games", self.games.len()); - } else { - info!( - "Cache has {} games but no images - forcing refresh", - games.len() - ); - cache::clear_games_cache(); - self.fetch_games(); - } - } - } - } - - // Check if library was fetched and saved to cache - if self.state == AppState::Games - && self.current_tab == GamesTab::MyLibrary - && self.library_games.is_empty() - { - if let Some(games) = cache::load_library_cache() { - if !games.is_empty() { - info!("Loaded {} games from library cache", games.len()); - self.library_games = games; - self.is_loading = false; - self.status_message = - format!("Your Library: {} games", self.library_games.len()); - } - } - } - - // Check if sections were fetched and saved to cache (Home tab) - if self.state == AppState::Games - && self.current_tab == GamesTab::Home - && self.game_sections.is_empty() - { - if let Some(sections) = cache::load_sections_cache() { - if !sections.is_empty() { - info!("Loaded {} sections from cache", sections.len()); - self.game_sections = sections; - self.is_loading = false; - self.status_message = format!("Loaded {} sections", self.game_sections.len()); - } - } - } - - // Check if subscription was fetched and saved to cache - if self.state == AppState::Games && self.subscription.is_none() { - if let Some(sub) = cache::load_subscription_cache() { - info!("Loaded subscription from cache: {}", sub.membership_tier); - self.subscription = Some(sub); - } - } - - // Check if queue data was fetched and saved to cache (Queue Times tab) - if self.state == AppState::Games - && self.current_tab == GamesTab::QueueTimes - && self.queue_loading - { - if let Some(servers) = cache::load_queue_cache() { - if !servers.is_empty() { - info!("Loaded {} queue servers from cache", servers.len()); - self.queue_servers = servers; - self.queue_loading = false; - // Start ping test for queue servers - self.start_queue_ping_test(); - } - } - } - - // Check for dynamic regions from serverInfo API - self.check_dynamic_regions(); - - // Check for ping test results - if self.ping_testing { - self.load_ping_results(); - } - - // Check for queue ping test results - if self.queue_ping_testing { - self.load_queue_ping_results(); - } - - // Check for active sessions from async check - if let Some(sessions) = cache::load_active_sessions_cache() { - self.active_sessions = sessions.clone(); - if let Some(pending) = cache::load_pending_game_cache() { - self.pending_game_launch = Some(pending); - self.show_session_conflict = true; - cache::clear_active_sessions_cache(); - } else if !self.active_sessions.is_empty() { - // Auto-resume logic: no pending game, but active sessions exist -> resume the first one - if let Some(session) = self.active_sessions.first() { - info!("Auto-resuming active session found: {}", session.session_id); - let session_clone = session.clone(); - self.resume_session(session_clone); - cache::clear_active_sessions_cache(); - } - } - } - - // Check for launch proceed flag (after session termination) - if cache::check_launch_proceed_flag() { - if let Some(game) = cache::load_pending_game_cache() { - cache::clear_pending_game_cache(); - self.start_new_session(&game); - } - } - - // Poll session status when in session state - if self.state == AppState::Session && self.is_loading { - self.poll_session_status(); - } - } - - /// Logout and return to login screen - pub fn logout(&mut self) { - self.auth_tokens = None; - self.user_info = None; - self.subscription = None; - auth::clear_login_provider(); - cache::clear_login_provider(); // Clear persisted provider too - cache::clear_tokens(); - cache::clear_games_cache(); // Clear cached games - self.state = AppState::Login; - self.games.clear(); - self.game_sections.clear(); - self.library_games.clear(); - self.status_message = "Logged out".to_string(); - } - - /// Fetch games library - pub fn fetch_games(&mut self) { - if self.auth_tokens.is_none() { - return; - } - - self.is_loading = true; - self.status_message = "Loading games...".to_string(); - - let token = self.auth_tokens.as_ref().unwrap().jwt().to_string(); - let mut api_client = GfnApiClient::new(); - api_client.set_access_token(token.clone()); - - let runtime = self.runtime.clone(); - runtime.spawn(async move { - // Fetch games from GraphQL MAIN panel (has images) - // This is the same approach as the official GFN client - match api_client.fetch_main_games(None).await { - Ok(games) => { - info!( - "Fetched {} games from GraphQL MAIN panel (with images)", - games.len() - ); - cache::save_games_cache(&games); - } - Err(e) => { - error!("Failed to fetch main games from GraphQL: {}", e); - - // Fallback: try public games list (no images, but has all games) - info!("Falling back to public games list..."); - match api_client.fetch_public_games().await { - Ok(games) => { - info!("Fetched {} games from public list (fallback)", games.len()); - cache::save_games_cache(&games); - } - Err(e2) => { - error!("Failed to fetch public games: {}", e2); - } - } - } - } - }); - } - - /// Fetch user's library games - pub fn fetch_library(&mut self) { - if self.auth_tokens.is_none() { - return; - } - - self.is_loading = true; - self.status_message = "Loading your library...".to_string(); - - let token = self.auth_tokens.as_ref().unwrap().jwt().to_string(); - let mut api_client = GfnApiClient::new(); - api_client.set_access_token(token.clone()); - - let runtime = self.runtime.clone(); - runtime.spawn(async move { - match api_client.fetch_library(None).await { - Ok(games) => { - info!("Fetched {} games from LIBRARY panel", games.len()); - cache::save_library_cache(&games); - } - Err(e) => { - error!("Failed to fetch library: {}", e); - } - } - }); - } - - /// Fetch game sections for Home tab (Trending, Free to Play, etc.) - pub fn fetch_sections(&mut self) { - if self.auth_tokens.is_none() { - return; - } - - self.is_loading = true; - self.status_message = "Loading sections...".to_string(); - - let token = self.auth_tokens.as_ref().unwrap().jwt().to_string(); - let mut api_client = GfnApiClient::new(); - api_client.set_access_token(token.clone()); - - let runtime = self.runtime.clone(); - runtime.spawn(async move { - match api_client.fetch_sectioned_games(None).await { - Ok(sections) => { - info!("Fetched {} sections from GraphQL", sections.len()); - cache::save_sections_cache(§ions); - } - Err(e) => { - error!("Failed to fetch sections: {}", e); - } - } - }); - } - - /// Fetch subscription info (hours, addons, etc.) - pub fn fetch_subscription(&mut self) { - if self.auth_tokens.is_none() { - return; - } - - // Clear current subscription so update loop will reload from cache after fetch completes - self.subscription = None; - - let token = self.auth_tokens.as_ref().unwrap().jwt().to_string(); - let user_id = self.auth_tokens.as_ref().unwrap().user_id().to_string(); - - let runtime = self.runtime.clone(); - runtime.spawn(async move { - match crate::api::fetch_subscription(&token, &user_id).await { - Ok(sub) => { - info!("Fetched subscription: tier={}, hours={:.1}/{:.1}, storage={}, unlimited={}", - sub.membership_tier, - sub.remaining_hours, - sub.total_hours, - sub.has_persistent_storage, - sub.is_unlimited - ); - cache::save_subscription_cache(&sub); - } - Err(e) => { - warn!("Failed to fetch subscription: {}", e); - } - } - }); - } - - /// Fetch queue times from PrintedWaste API - pub fn fetch_queue_times(&mut self) { - // Rate limit: only fetch if more than 30 seconds since last fetch - const QUEUE_CACHE_TTL: std::time::Duration = std::time::Duration::from_secs(30); - if self.queue_last_fetch.elapsed() < QUEUE_CACHE_TTL && !self.queue_servers.is_empty() { - return; - } - - self.queue_loading = true; - self.queue_last_fetch = std::time::Instant::now(); - - let runtime = self.runtime.clone(); - runtime.spawn(async move { - let client = reqwest::Client::new(); - match crate::api::fetch_queue_servers(&client).await { - Ok(servers) => { - info!( - "Fetched queue times for {} servers from PrintedWaste", - servers.len() - ); - cache::save_queue_cache(&servers); - } - Err(e) => { - warn!("Failed to fetch queue times: {}", e); - } - } - }); - } - - /// Load available servers/regions (tries dynamic fetch first, falls back to hardcoded) - pub fn load_servers(&mut self) { - info!("Loading servers..."); - - let runtime = self.runtime.clone(); - let token = self.auth_tokens.as_ref().map(|t| t.jwt().to_string()); - - // Spawn async task to fetch dynamic regions - runtime.spawn(async move { - let client = reqwest::Client::new(); - let regions = api::fetch_dynamic_regions(&client, token.as_deref()).await; - - // Store the results for the main thread to pick up - DYNAMIC_REGIONS_CACHE.write().replace(regions); - }); - - // For now, start with hardcoded servers (will update when dynamic fetch completes) - self.load_hardcoded_servers(); - } - - /// Load hardcoded servers as fallback - fn load_hardcoded_servers(&mut self) { - let server_zones: Vec<(&str, &str, &str)> = vec![ - // Europe - ("eu-netherlands-north", "Netherlands North", "Europe"), - ("eu-netherlands-south", "Netherlands South", "Europe"), - ("eu-united-kingdom-1", "United Kingdom", "Europe"), - ("eu-germany-frankfurt-1", "Frankfurt", "Europe"), - ("eu-france-paris-1", "Paris", "Europe"), - ("eu-finland-helsinki-1", "Helsinki", "Europe"), - ("eu-norway-oslo-1", "Oslo", "Europe"), - ("eu-sweden-stockholm-1", "Stockholm", "Europe"), - ("eu-poland-warsaw-1", "Warsaw", "Europe"), - ("eu-italy-rome-1", "Rome", "Europe"), - ("eu-spain-madrid-1", "Madrid", "Europe"), - // North America - ("us-california-north", "California North", "North America"), - ("us-california-south", "California South", "North America"), - ("us-texas-dallas-1", "Dallas", "North America"), - ("us-virginia-north", "Virginia North", "North America"), - ("us-illinois-chicago-1", "Chicago", "North America"), - ("us-washington-seattle-1", "Seattle", "North America"), - ("us-arizona-phoenix-1", "Phoenix", "North America"), - // Canada - ("ca-quebec", "Quebec", "Canada"), - // Asia-Pacific - ("ap-japan-tokyo-1", "Tokyo", "Asia-Pacific"), - ("ap-japan-osaka-1", "Osaka", "Asia-Pacific"), - ("ap-south-korea-seoul-1", "Seoul", "Asia-Pacific"), - ("ap-australia-sydney-1", "Sydney", "Asia-Pacific"), - ("ap-singapore-1", "Singapore", "Asia-Pacific"), - ]; - - self.servers = server_zones - .iter() - .map(|(id, name, region)| ServerInfo { - id: id.to_string(), - name: name.to_string(), - region: region.to_string(), - url: None, - ping_ms: None, - status: ServerStatus::Unknown, - }) - .collect(); - - // Restore selected server from settings - if let Some(ref selected_id) = self.settings.selected_server { - if let Some(idx) = self.servers.iter().position(|s| s.id == *selected_id) { - self.selected_server_index = idx; - } - } - - info!("Loaded {} hardcoded servers", self.servers.len()); - } - - /// Update servers from dynamic region cache (call this periodically from update loop) - pub fn check_dynamic_regions(&mut self) { - let dynamic_regions = DYNAMIC_REGIONS_CACHE.write().take(); - - if let Some(regions) = dynamic_regions { - if !regions.is_empty() { - info!("[serverInfo] Applying {} dynamic regions", regions.len()); - - // Convert dynamic regions to ServerInfo - // Group by region based on URL pattern - self.servers = regions - .iter() - .map(|r| { - // Extract server ID from URL hostname - // e.g., "https://eu-netherlands-south.cloudmatchbeta.nvidiagrid.net" -> "eu-netherlands-south" - let hostname = r - .url - .trim_start_matches("https://") - .trim_start_matches("http://") - .split('.') - .next() - .unwrap_or(&r.name); - - // Determine region from name or hostname - let region = if hostname.starts_with("eu-") - || r.name.contains("Europe") - || r.name.contains("UK") - || r.name.contains("France") - || r.name.contains("Germany") - { - "Europe" - } else if hostname.starts_with("us-") - || r.name.contains("US") - || r.name.contains("California") - || r.name.contains("Texas") - { - "North America" - } else if hostname.starts_with("ca-") - || r.name.contains("Canada") - || r.name.contains("Quebec") - { - "Canada" - } else if hostname.starts_with("ap-") - || r.name.contains("Japan") - || r.name.contains("Korea") - || r.name.contains("Singapore") - { - "Asia-Pacific" - } else { - "Other" - }; - - ServerInfo { - id: hostname.to_string(), - name: r.name.clone(), - region: region.to_string(), - url: Some(r.url.clone()), - ping_ms: None, - status: ServerStatus::Unknown, - } - }) - .collect(); - - // Restore selected server - if let Some(ref selected_id) = self.settings.selected_server { - if let Some(idx) = self.servers.iter().position(|s| s.id == *selected_id) { - self.selected_server_index = idx; - } - } - - info!("[serverInfo] Now have {} servers", self.servers.len()); - - // Auto-start ping test after loading dynamic servers - self.start_ping_test(); - } - } - } - - /// Check for active sessions explicitly - pub fn check_active_sessions(&mut self) { - if self.auth_tokens.is_none() { - return; - } - - let token = self.auth_tokens.as_ref().unwrap().jwt().to_string(); - let mut api_client = GfnApiClient::new(); - api_client.set_access_token(token); - - let runtime = self.runtime.clone(); - runtime.spawn(async move { - match api_client.get_active_sessions().await { - Ok(sessions) => { - info!("Checked active sessions: found {}", sessions.len()); - if !sessions.is_empty() { - cache::save_active_sessions_cache(&sessions); - } - } - Err(e) => { - warn!("Failed to check active sessions: {}", e); - } - } - }); - } - - /// Start ping test for all servers - pub fn start_ping_test(&mut self) { - if self.ping_testing { - return; // Already running - } - - self.ping_testing = true; - info!("Starting ping test for {} servers", self.servers.len()); - - // Mark all servers as testing - for server in &mut self.servers { - server.status = ServerStatus::Testing; - server.ping_ms = None; - } - - // Collect server info with URLs for pinging - let server_data: Vec<(String, Option)> = self - .servers - .iter() - .map(|s| (s.id.clone(), s.url.clone())) - .collect(); - let runtime = self.runtime.clone(); - - runtime.spawn(async move { - use futures_util::future::join_all; - - // Create ping futures for all servers (run in parallel) - let ping_futures: Vec<_> = server_data - .into_iter() - .map(|(server_id, url_opt)| async move { - // Extract hostname from URL or construct from server_id - let hostname = if let Some(url) = url_opt { - url.trim_start_matches("https://") - .trim_start_matches("http://") - .split('/') - .next() - .unwrap_or(&format!("{}.cloudmatchbeta.nvidiagrid.net", server_id)) - .to_string() - } else { - format!("{}.cloudmatchbeta.nvidiagrid.net", server_id) - }; - - // TCP ping with timeout (faster and more reliable than ICMP on Windows) - let ping_result = Self::tcp_ping(&hostname).await; - - let (ping_ms, status) = match ping_result { - Some(ms) => (Some(ms), ServerStatus::Online), - None => (None, ServerStatus::Offline), - }; - - (server_id, ping_ms, status) - }) - .collect(); - - // Run all pings concurrently - let results: Vec<(String, Option, ServerStatus)> = join_all(ping_futures).await; - - // Save results to cache - cache::save_ping_results(&results); - }); - } - - /// Fast TCP ping - measures connection time to server (TLS handshake) - async fn tcp_ping(hostname: &str) -> Option { - use std::time::Instant; - use tokio::net::TcpStream; - use tokio::time::{timeout, Duration}; - - let addr = format!("{}:443", hostname); - let start = Instant::now(); - // Short timeout for fast failure - let result = timeout(Duration::from_millis(1500), TcpStream::connect(&addr)).await; - - match result { - Ok(Ok(_stream)) => { - let elapsed = start.elapsed().as_millis() as u32; - Some(elapsed) - } - _ => None, - } - } - - /// Load ping results from cache - fn load_ping_results(&mut self) { - if let Some(results) = cache::load_ping_results() { - for result in results { - if let Some(id) = result.get("id").and_then(|v| v.as_str()) { - if let Some(server) = self.servers.iter_mut().find(|s| s.id == id) { - server.ping_ms = result - .get("ping_ms") - .and_then(|v| v.as_u64()) - .map(|v| v as u32); - server.status = match result.get("status").and_then(|v| v.as_str()) { - Some("Online") => ServerStatus::Online, - Some("Offline") => ServerStatus::Offline, - _ => ServerStatus::Unknown, - }; - } - } - } - - self.ping_testing = false; - - // Sort servers by ping (online first, then by ping) - self.servers.sort_by(|a, b| match (&a.status, &b.status) { - (ServerStatus::Online, ServerStatus::Online) => { - a.ping_ms.unwrap_or(9999).cmp(&b.ping_ms.unwrap_or(9999)) - } - (ServerStatus::Online, _) => std::cmp::Ordering::Less, - (_, ServerStatus::Online) => std::cmp::Ordering::Greater, - _ => std::cmp::Ordering::Equal, - }); - - // Update selected index after sort - if self.auto_server_selection { - // Auto-select best server - self.select_best_server(); - } else if let Some(ref selected_id) = self.settings.selected_server { - if let Some(idx) = self.servers.iter().position(|s| s.id == *selected_id) { - self.selected_server_index = idx; - } - } - } - } - - /// Select the best server based on ping (lowest ping online server) - fn select_best_server(&mut self) { - // Find the server with the lowest ping that is online - let best_server = self - .servers - .iter() - .enumerate() - .filter(|(_, s)| s.status == ServerStatus::Online && s.ping_ms.is_some()) - .min_by_key(|(_, s)| s.ping_ms.unwrap_or(9999)); - - if let Some((idx, server)) = best_server { - self.selected_server_index = idx; - info!( - "Auto-selected best server: {} ({}ms)", - server.name, - server.ping_ms.unwrap_or(0) - ); - } - } - - /// Start ping test for queue servers (uses VPC IDs like NP-AMS-07) - pub fn start_queue_ping_test(&mut self) { - if self.queue_ping_testing || self.queue_servers.is_empty() { - return; - } - - self.queue_ping_testing = true; - info!( - "Starting queue ping test for {} servers", - self.queue_servers.len() - ); - - // Collect server IDs for pinging - let server_ids: Vec = self - .queue_servers - .iter() - .map(|s| s.server_id.clone()) - .collect(); - let runtime = self.runtime.clone(); - - runtime.spawn(async move { - use futures_util::future::join_all; - - // Create ping futures for all queue servers (run in parallel) - let ping_futures: Vec<_> = server_ids - .into_iter() - .map(|server_id| async move { - // Construct hostname from VPC ID (e.g., NP-AMS-07 -> np-ams-07.cloudmatchbeta.nvidiagrid.net) - let hostname = - format!("{}.cloudmatchbeta.nvidiagrid.net", server_id.to_lowercase()); - - let ping_result = Self::tcp_ping(&hostname).await; - (server_id, ping_result) - }) - .collect(); - - // Run all pings concurrently - let results: Vec<(String, Option)> = join_all(ping_futures).await; - - // Save results to cache - cache::save_queue_ping_results(&results); - }); - } - - /// Load queue ping results from cache - fn load_queue_ping_results(&mut self) { - if let Some(results) = cache::load_queue_ping_results() { - for (server_id, ping_ms) in results { - if let Some(server) = self - .queue_servers - .iter_mut() - .find(|s| s.server_id == server_id) - { - server.ping_ms = ping_ms; - } - } - - self.queue_ping_testing = false; - info!("Queue ping test completed"); - } - } - - /// Launch a game session - pub fn launch_game(&mut self, game: &GameInfo) { - info!("Launching game: {} (ID: {})", game.title, game.id); - - // Get token first - let token = match &self.auth_tokens { - Some(t) => t.jwt().to_string(), - None => { - self.error_message = Some("Not logged in".to_string()); - return; - } - }; - - let game_clone = game.clone(); - - let mut api_client = GfnApiClient::new(); - api_client.set_access_token(token); - - let runtime = self.runtime.clone(); - runtime.spawn(async move { - match api_client.get_active_sessions().await { - Ok(sessions) => { - if !sessions.is_empty() { - info!("Found {} active session(s)", sessions.len()); - cache::save_active_sessions_cache(&sessions); - cache::save_pending_game_cache(&game_clone); - } else { - info!("No active sessions, proceeding with launch"); - cache::clear_active_sessions_cache(); - cache::save_pending_game_cache(&game_clone); - cache::save_launch_proceed_flag(); - } - } - Err(e) => { - warn!( - "Failed to check active sessions: {}, proceeding with launch", - e - ); - cache::clear_active_sessions_cache(); - cache::save_pending_game_cache(&game_clone); - cache::save_launch_proceed_flag(); - } - } - }); - } - - /// Start creating a new session (after checking for conflicts) - fn start_new_session(&mut self, game: &GameInfo) { - info!("Starting new session for: {}", game.title); - - cache::clear_session_cache(); - cache::clear_session_error(); - - self.selected_game = Some(game.clone()); - self.state = AppState::Session; - self.status_message = format!("Starting {}...", game.title); - self.error_message = None; - self.is_loading = true; - self.last_poll_time = std::time::Instant::now() - POLL_INTERVAL; - - let token = match &self.auth_tokens { - Some(t) => t.jwt().to_string(), - None => { - self.error_message = Some("Not logged in".to_string()); - self.is_loading = false; - return; - } - }; - - let app_id = game.id.clone(); - let game_title = game.title.clone(); - let settings = self.settings.clone(); - - let zone = self - .servers - .get(self.selected_server_index) - .map(|s| s.id.clone()) - .unwrap_or_else(|| "eu-netherlands-south".to_string()); - - let is_install_to_play = game.is_install_to_play; - - let mut api_client = GfnApiClient::new(); - api_client.set_access_token(token); - - let runtime = self.runtime.clone(); - runtime.spawn(async move { - // Fetch latest app details to check for playType="INSTALL_TO_PLAY" - // This is critical for demos which require account_linked=false - let mut account_linked = !is_install_to_play; - - match api_client.fetch_app_details(&app_id).await { - Ok(Some(details)) => { - info!( - "Fetched fresh app details: is_install_to_play={}", - details.is_install_to_play - ); - account_linked = !details.is_install_to_play; - } - Ok(None) => warn!( - "App details not found, using cached info: is_install_to_play={}", - is_install_to_play - ), - Err(e) => warn!("Failed to fetch app details ({}): {}", app_id, e), - } - - info!( - "Starting session for '{}' with account_linked: {}", - game_title, account_linked - ); - - match api_client - .create_session(&app_id, &game_title, &settings, &zone, account_linked) - .await - { - Ok(session) => { - info!( - "Session created: {} (state: {:?})", - session.session_id, session.state - ); - cache::save_session_cache(&session); - } - Err(e) => { - error!("Failed to create session: {}", e); - cache::save_session_error(&format!("Failed to create session: {}", e)); - } - } - }); - } - - /// Resume an existing session - fn resume_session(&mut self, session_info: ActiveSessionInfo) { - info!( - "Resuming session: {} (status: {}, server_ip: {:?}, signaling_url: {:?})", - session_info.session_id, - session_info.status, - session_info.server_ip, - session_info.signaling_url - ); - - self.show_session_conflict = false; - self.pending_game_launch = None; - self.state = AppState::Session; - self.status_message = "Resuming session...".to_string(); - self.error_message = None; - self.is_loading = true; - self.last_poll_time = std::time::Instant::now() - POLL_INTERVAL; - - // Session status codes: - // 2 = Ready (needs RESUME PUT to re-attach client) - // 3 = Streaming (also needs RESUME PUT to re-attach client) - // Both status 2 and 3 need the claim/resume PUT request - - let token = match &self.auth_tokens { - Some(t) => t.jwt().to_string(), - None => { - self.error_message = Some("Not logged in".to_string()); - self.is_loading = false; - return; - } - }; - - let server_ip = match session_info.server_ip { - Some(ip) => ip, - None => { - self.error_message = Some("Session has no server IP".to_string()); - self.is_loading = false; - return; - } - }; - - let app_id = session_info.app_id.to_string(); - let settings = self.settings.clone(); - - let mut api_client = GfnApiClient::new(); - api_client.set_access_token(token); - - let runtime = self.runtime.clone(); - runtime.spawn(async move { - match api_client - .claim_session(&session_info.session_id, &server_ip, &app_id, &settings) - .await - { - Ok(session) => { - info!( - "Session claimed: {} (state: {:?})", - session.session_id, session.state - ); - cache::save_session_cache(&session); - } - Err(e) => { - error!("Failed to claim session: {}", e); - cache::save_session_error(&format!("Failed to resume session: {}", e)); - } - } - }); - } - - /// Terminate existing session and start new game - fn terminate_and_launch(&mut self, session_id: String, game: GameInfo) { - info!( - "Terminating session {} and launching {}", - session_id, game.title - ); - - self.show_session_conflict = false; - self.pending_game_launch = None; - self.status_message = "Ending previous session...".to_string(); - - let token = match &self.auth_tokens { - Some(t) => t.jwt().to_string(), - None => { - self.error_message = Some("Not logged in".to_string()); - return; - } - }; - - let mut api_client = GfnApiClient::new(); - api_client.set_access_token(token); - - let runtime = self.runtime.clone(); - let game_for_launch = game.clone(); - runtime.spawn(async move { - match api_client.stop_session(&session_id, "", None).await { - Ok(_) => { - info!("Session terminated, waiting before launching new session"); - tokio::time::sleep(tokio::time::Duration::from_millis(1000)).await; - cache::save_launch_proceed_flag(); - cache::save_pending_game_cache(&game_for_launch); - } - Err(e) => { - warn!("Session termination failed ({}), proceeding anyway", e); - cache::save_launch_proceed_flag(); - cache::save_pending_game_cache(&game_for_launch); - } - } - }); - } - - /// Poll session state and update UI - fn poll_session_status(&mut self) { - // First check cache for state updates (from in-flight or completed requests) - // First check cache for state updates (from in-flight or completed requests) - if let Some(session) = cache::load_session_cache() { - if session.state == SessionState::Ready { - // User requested: "make it pull few times before connecting to it so you can get the candidates" - // We delay streaming start until we've polled a few times in Ready state - if self.session_ready_poll_count < 3 { - self.status_message = format!( - "Session ready, finalizing connection ({}/3)...", - self.session_ready_poll_count + 1 - ); - // Don't return, allow fall-through to polling logic - } else { - info!( - "Session ready! GPU: {:?}, Server: {}", - session.gpu_type, session.server_ip - ); - - // Update status message - if let Some(gpu) = &session.gpu_type { - self.status_message = format!("Connecting to GPU: {}", gpu); - } else { - self.status_message = - format!("Connecting to server: {}", session.server_ip); - } - - cache::clear_session_cache(); - self.start_streaming(session); - return; - } - } else if let SessionState::InQueue { position, eta_secs } = session.state { - self.status_message = format!("Queue position: {} (ETA: {}s)", position, eta_secs); - } else if let SessionState::WatchingAds { - remaining_secs, - total_secs, - } = session.state - { - self.ads_required = true; - self.ads_remaining_secs = remaining_secs; - self.ads_total_secs = total_secs; - self.status_message = - format!("Waiting for ads... (~{}s remaining)", remaining_secs); - } else if let SessionState::Error(ref msg) = session.state { - self.error_message = Some(msg.clone()); - self.is_loading = false; - cache::clear_session_cache(); - return; - } else if session.state == SessionState::Connecting { - self.status_message = "Connecting to server...".to_string(); - } else if session.state == SessionState::CleaningUp { - self.status_message = "Cleaning up previous session...".to_string(); - } else if session.state == SessionState::WaitingForStorage { - self.status_message = "Waiting for storage to be ready...".to_string(); - } else { - self.status_message = "Setting up session...".to_string(); - } - } - - // Rate limit polling - only poll every POLL_INTERVAL (2 seconds) - let now = std::time::Instant::now(); - if now.duration_since(self.last_poll_time) < POLL_INTERVAL { - return; - } - - if let Some(session) = cache::load_session_cache() { - let mut should_poll = matches!( - session.state, - SessionState::Requesting - | SessionState::Launching - | SessionState::Connecting - | SessionState::CleaningUp - | SessionState::WaitingForStorage - | SessionState::InQueue { .. } - | SessionState::WatchingAds { .. } - ); - - // Also poll if Ready but count < 3 - if session.state == SessionState::Ready && self.session_ready_poll_count < 3 { - should_poll = true; - // Increment poll count here, as we are about to poll - self.session_ready_poll_count += 1; - } - - if should_poll { - // Update timestamp to rate limit next poll - self.last_poll_time = now; - - let token = match &self.auth_tokens { - Some(t) => t.jwt().to_string(), - None => return, - }; - - let session_id = session.session_id.clone(); - let zone = session.zone.clone(); - let server_ip = if session.server_ip.is_empty() { - None - } else { - Some(session.server_ip.clone()) - }; - - let mut api_client = GfnApiClient::new(); - api_client.set_access_token(token); - - let runtime = self.runtime.clone(); - runtime.spawn(async move { - match api_client - .poll_session(&session_id, &zone, server_ip.as_deref()) - .await - { - Ok(updated_session) => { - info!("Session poll: state={:?}", updated_session.state); - cache::save_session_cache(&updated_session); - } - Err(e) => { - error!("Session poll failed: {}", e); - } - } - }); - } - } - - // Check for session errors - if let Some(error) = cache::load_session_error() { - self.error_message = Some(error); - self.is_loading = false; - cache::clear_session_error(); - } - - // Check for popup game details updates - if let Some(detailed_game) = cache::load_popup_game_details() { - // Only update if we still have the popup open for the same game - if let Some(current_popup) = &self.selected_game_popup { - if current_popup.id == detailed_game.id { - info!( - "Updating popup with detailed info for: {}", - detailed_game.title - ); - self.selected_game_popup = Some(detailed_game); - } - } - } - } - - /// Start streaming once session is ready - pub fn start_streaming(&mut self, session: SessionInfo) { - info!("Starting streaming to {}", session.server_ip); - info!("Session Info Debug: {:?}", session); - - self.session = Some(session.clone()); - self.state = AppState::Streaming; - self.cursor_captured = true; - self.is_loading = false; - - // Reset session ready poll count for this new session - self.session_ready_poll_count = 0; - - // Initialize session timing for proper input timestamps - // This must be called BEFORE any input events are sent - crate::input::init_session_timing(); - - // Set local cursor dimensions for instant visual feedback - // Parse resolution from settings (e.g., "1920x1080" -> width, height) - let (width, height) = parse_resolution(&self.settings.resolution); - #[cfg(any(target_os = "windows", target_os = "macos"))] - crate::input::set_local_cursor_dimensions(width, height); - - // Reset coalescing state to ensure clean input state for new session - #[cfg(any(target_os = "windows", target_os = "macos"))] - crate::input::reset_coalescing(); - - info!( - "Input system initialized: session timing + local cursor {}x{}", - width, height - ); - - // Create shared frame holder for zero-latency frame delivery - // No buffering - decoder writes latest frame, renderer reads it immediately - let shared_frame = Arc::new(SharedFrame::new()); - self.shared_frame = Some(shared_frame.clone()); - - // Stats channel (small buffer is fine for stats) - let (stats_tx, stats_rx) = mpsc::channel(8); - info!( - "Using zero-latency shared frame delivery for {}fps", - self.settings.fps - ); - - self.stats_rx = Some(stats_rx); - - // Create input handler with clean state - let input_handler = Arc::new(InputHandler::new()); - self.input_handler = Some(input_handler.clone()); - - self.status_message = "Connecting...".to_string(); - - // Clone settings for the async task - let settings = self.settings.clone(); - - // Spawn the streaming task - let runtime = self.runtime.clone(); - runtime.spawn(async move { - use crate::webrtc::StreamingResult; - - match crate::webrtc::run_streaming( - session.clone(), - settings.clone(), - shared_frame.clone(), - stats_tx.clone(), - input_handler.clone(), - ) - .await - { - StreamingResult::Normal => { - info!("Streaming ended normally"); - } - StreamingResult::Error(e) => { - error!("Streaming error: {}", e); - } - StreamingResult::SsrcChangeDetected { stall_duration_ms } => { - // SSRC change detected - attempt auto-reconnect - warn!( - "SSRC change detected after {}ms stall. Attempting auto-reconnect...", - stall_duration_ms - ); - - // Brief delay to let resources clean up - tokio::time::sleep(std::time::Duration::from_millis(500)).await; - - // Retry the connection with the same session info - // The server-side session is still active, we just need to re-establish WebRTC - info!("Auto-reconnecting to session {}...", session.session_id); - - // Create new shared frame for reconnection - let new_shared_frame = std::sync::Arc::new(crate::app::SharedFrame::new()); - - // Create new stats channel - let (new_stats_tx, _new_stats_rx) = tokio::sync::mpsc::channel(8); - - // Create new input handler - let new_input_handler = std::sync::Arc::new(crate::input::InputHandler::new()); - - // Attempt reconnection - match crate::webrtc::run_streaming( - session, - settings, - new_shared_frame, - new_stats_tx, - new_input_handler, - ) - .await - { - StreamingResult::Normal => { - info!("Reconnected stream ended normally"); - } - StreamingResult::Error(e) => { - error!("Reconnected stream error: {}", e); - } - StreamingResult::SsrcChangeDetected { stall_duration_ms } => { - // Second SSRC change - give up and let user know - error!( - "Second SSRC change detected after {}ms. Auto-reconnect failed. Please restart the session manually.", - stall_duration_ms - ); - } - } - } - } - }); - } - - /// Terminate current session via API and stop streaming - pub fn terminate_current_session(&mut self) { - if let Some(session) = &self.session { - info!( - "Ctrl+Shift+Q: Terminating active session: {}", - session.session_id - ); - - let token = match &self.auth_tokens { - Some(t) => t.jwt().to_string(), - None => { - self.stop_streaming(); - return; - } - }; - - let session_id = session.session_id.clone(); - let zone = session.zone.clone(); - let server_ip = if session.server_ip.is_empty() { - None - } else { - Some(session.server_ip.clone()) - }; - - let mut api_client = GfnApiClient::new(); - api_client.set_access_token(token); - - let runtime = self.runtime.clone(); - runtime.spawn(async move { - match api_client - .stop_session(&session_id, &zone, server_ip.as_deref()) - .await - { - Ok(_) => info!("Session {} terminated successfully", session_id), - Err(e) => warn!("Failed to stop session {}: {}", session_id, e), - } - }); - } - - self.stop_streaming(); - } - - /// Stop streaming and return to games - pub fn stop_streaming(&mut self) { - info!("Stopping streaming"); - - // Clear session cache first to prevent stale session data - cache::clear_session_cache(); - - // Reset input timing for next session - crate::input::reset_session_timing(); - - // Reset input coalescing and local cursor state - #[cfg(any(target_os = "windows", target_os = "macos"))] - crate::input::reset_coalescing(); - - // Clear raw input sender to prevent stale events from being processed - #[cfg(any(target_os = "windows", target_os = "macos"))] - crate::input::clear_raw_input_sender(); - - self.cursor_captured = false; - self.state = AppState::Games; - self.streaming_session = None; - self.session = None; // Clear session info - self.input_handler = None; - self.current_frame = None; - self.shared_frame = None; - self.stats_rx = None; - self.selected_game = None; - self.is_loading = false; - self.error_message = None; - - // Reset session ready poll count for next session - self.session_ready_poll_count = 0; - - // Reset ads state - self.ads_required = false; - self.ads_remaining_secs = 0; - self.ads_total_secs = 0; - - self.status_message = "Stream ended".to_string(); - } - - /// Toggle stats overlay - pub fn toggle_stats(&mut self) { - self.show_stats = !self.show_stats; - } - - /// Save settings - pub fn save_settings(&self) { - if let Err(e) = self.settings.save() { - error!("Failed to save settings: {}", e); - } - } - - /// Get current user display name - pub fn user_display_name(&self) -> &str { - self.user_info - .as_ref() - .map(|u| u.display_name.as_str()) - .unwrap_or("User") - } - - /// Get current membership tier - pub fn membership_tier(&self) -> &str { - self.user_info - .as_ref() - .map(|u| u.membership_tier.as_str()) - .unwrap_or("FREE") - } -} diff --git a/opennow-streamer/src/app/session.rs b/opennow-streamer/src/app/session.rs deleted file mode 100644 index 76591be..0000000 --- a/opennow-streamer/src/app/session.rs +++ /dev/null @@ -1,600 +0,0 @@ -//! Session Management -//! -//! GFN session state and lifecycle. - -use serde::{Deserialize, Serialize}; - -/// Session information -#[derive(Debug, Clone)] -pub struct SessionInfo { - /// Session ID from CloudMatch - pub session_id: String, - - /// Streaming server IP - pub server_ip: String, - - /// Server region/zone - pub zone: String, - - /// Current session state - pub state: SessionState, - - /// GPU type allocated - pub gpu_type: Option, - - /// Signaling WebSocket URL (full URL like wss://server/nvst/) - pub signaling_url: Option, - - /// ICE servers from session API (for Alliance Partners with TURN servers) - pub ice_servers: Vec, - - /// Media connection info (real UDP port for streaming) - pub media_connection_info: Option, - - /// Whether ads are required for this session (free tier) - pub ads_required: bool, - - /// Ad configuration if ads are required - pub ads_info: Option, -} - -/// ICE server configuration -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct IceServerConfig { - pub urls: Vec, - pub username: Option, - pub credential: Option, -} - -/// Media connection info (real port for Alliance Partners) -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct MediaConnectionInfo { - pub ip: String, - pub port: u16, -} - -/// Session ads information for free tier users -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SessionAdsInfo { - /// Video ad URL to play - #[serde(default)] - pub video_url: Option, - - /// Ad duration in seconds - #[serde(default)] - pub duration_secs: u32, - - /// Tracking/completion callback URL - #[serde(default)] - pub completion_url: Option, - - /// Raw ad configuration from server - #[serde(default)] - pub raw_config: Option, -} - -/// Session state -#[derive(Debug, Clone, PartialEq)] -pub enum SessionState { - /// Requesting session from CloudMatch - Requesting, - - /// Connecting to server (seatSetupStep = 0) - Connecting, - - /// Session created, seat being set up (configuring) - Launching, - - /// In queue waiting for a seat (seatSetupStep = 1) - InQueue { position: u32, eta_secs: u32 }, - - /// Watching ads (free tier users during queue) - WatchingAds { - /// Time remaining in seconds - remaining_secs: u32, - /// Total ad duration in seconds - total_secs: u32, - }, - - /// Cleaning up previous session (seatSetupStep = 5) - CleaningUp, - - /// Waiting for storage to be ready (seatSetupStep = 6) - WaitingForStorage, - - /// Session ready for streaming - Ready, - - /// Actively streaming - Streaming, - - /// Session error - Error(String), - - /// Session terminated - Terminated, -} - -impl SessionInfo { - /// Create a new session in requesting state - pub fn new_requesting(zone: &str) -> Self { - Self { - session_id: String::new(), - server_ip: String::new(), - zone: zone.to_string(), - state: SessionState::Requesting, - gpu_type: None, - signaling_url: None, - ice_servers: Vec::new(), - media_connection_info: None, - ads_required: false, - ads_info: None, - } - } - - /// Check if session is ready to stream - pub fn is_ready(&self) -> bool { - matches!(self.state, SessionState::Ready) - } - - /// Check if session is in queue - pub fn is_queued(&self) -> bool { - matches!(self.state, SessionState::InQueue { .. }) - } - - /// Get queue position if in queue - pub fn queue_position(&self) -> Option { - match self.state { - SessionState::InQueue { position, .. } => Some(position), - _ => None, - } - } -} - -// ============================================ -// CloudMatch API Request Types (Browser Format) -// ============================================ - -/// CloudMatch API request structure (browser format) -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct CloudMatchRequest { - pub session_request_data: SessionRequestData, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct SessionRequestData { - /// App ID as STRING (browser format) - pub app_id: String, - pub internal_title: Option, - pub available_supported_controllers: Vec, - pub network_test_session_id: Option, - pub parent_session_id: Option, - pub client_identification: String, - pub device_hash_id: String, - pub client_version: String, - pub sdk_version: String, - /// Streamer version as NUMBER (browser format) - pub streamer_version: i32, - pub client_platform_name: String, - pub client_request_monitor_settings: Vec, - pub use_ops: bool, - pub audio_mode: i32, - pub meta_data: Vec, - pub sdr_hdr_mode: i32, - pub client_display_hdr_capabilities: Option, - pub surround_audio_info: i32, - pub remote_controllers_bitmap: i32, - pub client_timezone_offset: i64, - pub enhanced_stream_mode: i32, - pub app_launch_mode: i32, - pub secure_rtsp_supported: bool, - pub partner_custom_data: Option, - pub account_linked: bool, - pub enable_persisting_in_game_settings: bool, - pub user_age: i32, - pub requested_streaming_features: Option, -} - -#[derive(Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct MonitorSettings { - pub width_in_pixels: u32, - pub height_in_pixels: u32, - pub frames_per_second: u32, - pub sdr_hdr_mode: i32, - pub display_data: DisplayData, - pub dpi: i32, -} - -#[derive(Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct DisplayData { - pub desired_content_max_luminance: i32, - pub desired_content_min_luminance: i32, - pub desired_content_max_frame_average_luminance: i32, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct HdrCapabilities { - pub version: i32, - pub hdr_edr_supported_flags_in_uint32: i32, - pub static_metadata_descriptor_id: i32, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct MetaDataEntry { - pub key: String, - pub value: String, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct StreamingFeatures { - pub reflex: bool, - pub bit_depth: i32, - pub cloud_gsync: bool, - pub enabled_l4s: bool, - pub mouse_movement_flags: i32, - pub true_hdr: bool, - pub supported_hid_devices: i32, - pub profile: i32, - pub fallback_to_logical_resolution: bool, - pub hid_devices: Option, - pub chroma_format: i32, - pub prefilter_mode: i32, - pub prefilter_sharpness: i32, - pub prefilter_noise_reduction: i32, - pub hud_streaming_mode: i32, - /// SDR color space (2 = BT.709 / YCBCR_LIMITED_BT709) - pub sdr_color_space: i32, - /// HDR color space (4 = BT.2020 / YCBCR_LIMITED_BT2020, 0 = disabled) - pub hdr_color_space: i32, -} - -// ============================================ -// CloudMatch API Response Types -// ============================================ - -/// CloudMatch API response -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct CloudMatchResponse { - pub session: CloudMatchSession, - pub request_status: RequestStatus, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct CloudMatchSession { - pub session_id: String, - #[serde(default)] - pub seat_setup_info: Option, - #[serde(default)] - pub session_control_info: Option, - #[serde(default)] - pub connection_info: Option>, - #[serde(default)] - pub gpu_type: Option, - #[serde(default)] - pub status: i32, - #[serde(default)] - pub error_code: i32, - #[serde(default)] - pub ice_server_configuration: Option, - /// Whether ads are required for this session (free tier) - #[serde(default)] - pub session_ads_required: bool, - /// Ad configuration data from server - #[serde(default)] - pub session_ads: Option, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SeatSetupInfo { - #[serde(default)] - pub queue_position: i32, - #[serde(default)] - pub seat_setup_eta: i32, - #[serde(default)] - pub seat_setup_step: i32, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SessionControlInfo { - #[serde(default)] - pub ip: Option, - #[serde(default)] - pub port: u16, - #[serde(default)] - pub resource_path: Option, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ConnectionInfoData { - #[serde(default)] - pub ip: Option, - #[serde(default)] - pub port: u16, - #[serde(default)] - pub resource_path: Option, - /// Usage type: - /// - 2: Primary media path (UDP) - /// - 14: Signaling (WSS) - /// - 17: Alternative media path - #[serde(default)] - pub usage: i32, - /// Protocol: 1 = TCP/WSS, 2 = UDP - #[serde(default)] - pub protocol: i32, -} - -/// ICE server configuration from session API -#[derive(Debug, Deserialize, Clone)] -#[serde(rename_all = "camelCase")] -pub struct IceServerConfiguration { - #[serde(default)] - pub ice_servers: Vec, -} - -/// Individual ICE server from session API -#[derive(Debug, Deserialize, Clone)] -#[serde(rename_all = "camelCase")] -pub struct SessionIceServer { - pub urls: String, - #[serde(default)] - pub username: Option, - #[serde(default)] - pub credential: Option, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct RequestStatus { - pub status_code: i32, - #[serde(default)] - pub status_description: Option, - #[serde(default)] - pub unified_error_code: i32, - #[serde(default)] - pub server_id: Option, -} - -impl CloudMatchSession { - /// Extract streaming server IP from connection info - pub fn streaming_server_ip(&self) -> Option { - // Look for connection with usage=14 (signaling) first - self.connection_info - .as_ref() - .and_then(|conns| conns.iter().find(|c| c.usage == 14)) - .and_then(|conn| { - // Try direct IP first - conn.ip.clone().or_else(|| { - // If IP is null, extract from resourcePath (Alliance format) - // e.g., "rtsps://161-248-11-132.bpc.geforcenow.nvidiagrid.net:48322" - conn.resource_path - .as_ref() - .and_then(|path| Self::extract_host_from_url(path)) - }) - }) - .or_else(|| { - self.session_control_info - .as_ref() - .and_then(|sci| sci.ip.clone()) - }) - } - - /// Extract host from URL (handles rtsps://, wss://, etc.) - fn extract_host_from_url(url: &str) -> Option { - // Remove protocol prefix - let after_proto = url - .strip_prefix("rtsps://") - .or_else(|| url.strip_prefix("rtsp://")) - .or_else(|| url.strip_prefix("wss://")) - .or_else(|| url.strip_prefix("https://"))?; - - // Get host (before port or path) - let host = after_proto - .split(':') - .next() - .or_else(|| after_proto.split('/').next())?; - - if host.is_empty() || host.starts_with('.') { - None - } else { - Some(host.to_string()) - } - } - - /// Extract signaling URL from connection info - pub fn signaling_url(&self) -> Option { - self.connection_info - .as_ref() - .and_then(|conns| conns.iter().find(|c| c.usage == 14)) - .and_then(|conn| conn.resource_path.clone()) - } - - /// Extract media connection info (usage=2, usage=17, or fallback to usage=14 for Alliance) - pub fn media_connection_info(&self) -> Option { - self.connection_info.as_ref().and_then(|conns| { - // Try standard media paths first (usage=2 or usage=17) - let media_conn = conns - .iter() - .find(|c| c.usage == 2) - .or_else(|| conns.iter().find(|c| c.usage == 17)); - - // If found, try to get IP/port - if let Some(conn) = media_conn { - let ip = conn.ip.clone().or_else(|| { - conn.resource_path - .as_ref() - .and_then(|p| Self::extract_host_from_url(p)) - }); - let port = if conn.port > 0 { - conn.port - } else { - conn.resource_path - .as_ref() - .and_then(|p| Self::extract_port_from_url(p)) - .unwrap_or(0) - }; - - if let Some(ip) = ip { - if port > 0 { - return Some(MediaConnectionInfo { ip, port }); - } - } - } - - // For Alliance: fall back to usage=14 with highest port (usually the UDP streaming port) - // Alliance sessions have usage=14 for both signaling and media - let alliance_conn = conns - .iter() - .filter(|c| c.usage == 14) - .max_by_key(|c| c.port); - - alliance_conn.and_then(|conn| { - let ip = conn.ip.clone().or_else(|| { - conn.resource_path - .as_ref() - .and_then(|p| Self::extract_host_from_url(p)) - }); - let port = if conn.port > 0 { - conn.port - } else { - conn.resource_path - .as_ref() - .and_then(|p| Self::extract_port_from_url(p)) - .unwrap_or(0) - }; - - ip.filter(|_| port > 0) - .map(|ip| MediaConnectionInfo { ip, port }) - }) - }) - } - - /// Extract port from URL - fn extract_port_from_url(url: &str) -> Option { - // Find host:port pattern after :// - let after_proto = url - .strip_prefix("rtsps://") - .or_else(|| url.strip_prefix("rtsp://")) - .or_else(|| url.strip_prefix("wss://")) - .or_else(|| url.strip_prefix("https://"))?; - - // Extract port after colon - let parts: Vec<&str> = after_proto.split(':').collect(); - if parts.len() >= 2 { - // Port is after the colon, before any path - let port_str = parts[1].split('/').next()?; - port_str.parse().ok() - } else { - None - } - } - - /// Convert ICE server configuration - pub fn ice_servers(&self) -> Vec { - self.ice_server_configuration - .as_ref() - .map(|config| { - config - .ice_servers - .iter() - .map(|server| IceServerConfig { - urls: vec![server.urls.clone()], - username: server.username.clone(), - credential: server.credential.clone(), - }) - .collect() - }) - .unwrap_or_default() - } -} - -// ============================================ -// Session Management Types (GET /v2/session) -// ============================================ - -/// Response from GET /v2/session endpoint (list active sessions) -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct GetSessionsResponse { - #[serde(default)] - pub sessions: Vec, - pub request_status: RequestStatus, -} - -/// Session data from GET /v2/session API -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SessionFromApi { - pub session_id: String, - #[serde(default)] - pub session_request_data: Option, - #[serde(default)] - pub gpu_type: Option, - #[serde(default)] - pub status: i32, - #[serde(default)] - pub session_control_info: Option, - #[serde(default)] - pub connection_info: Option>, - #[serde(default)] - pub monitor_settings: Option>, -} - -/// Lenient MonitorSettings for API response -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct MonitorSettingsFromApi { - #[serde(default)] - pub width_in_pixels: Option, - #[serde(default)] - pub height_in_pixels: Option, - #[serde(default)] - pub frames_per_second: Option, -} - -/// Session request data from API (contains app_id) -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SessionRequestDataFromApi { - /// App ID can be string or number - #[serde(default)] - pub app_id: Option, -} - -impl SessionRequestDataFromApi { - pub fn get_app_id(&self) -> i64 { - match &self.app_id { - Some(serde_json::Value::Number(n)) => n.as_i64().unwrap_or(0), - Some(serde_json::Value::String(s)) => s.parse::().unwrap_or(0), - _ => 0, - } - } -} - -/// Simplified active session info for UI -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ActiveSessionInfo { - pub session_id: String, - pub app_id: i64, - pub gpu_type: Option, - pub status: i32, - pub server_ip: Option, - pub signaling_url: Option, - pub resolution: Option, - pub fps: Option, -} diff --git a/opennow-streamer/src/app/types.rs b/opennow-streamer/src/app/types.rs deleted file mode 100644 index df089b9..0000000 --- a/opennow-streamer/src/app/types.rs +++ /dev/null @@ -1,311 +0,0 @@ -//! Application Types -//! -//! Common types used across the application. - -use parking_lot::Mutex; -use std::sync::atomic::{AtomicU64, Ordering}; - -use super::config::{ColorQuality, VideoCodec, VideoDecoderBackend}; -use crate::media::VideoFrame; - -/// Shared frame holder for zero-latency frame delivery -/// Decoder writes latest frame, renderer reads it - no buffering -pub struct SharedFrame { - frame: Mutex>, - frame_count: AtomicU64, - last_read_count: AtomicU64, -} - -impl SharedFrame { - pub fn new() -> Self { - Self { - frame: Mutex::new(None), - frame_count: AtomicU64::new(0), - last_read_count: AtomicU64::new(0), - } - } - - /// Write a new frame (called by decoder) - pub fn write(&self, frame: VideoFrame) { - *self.frame.lock() = Some(frame); - self.frame_count.fetch_add(1, Ordering::Release); - } - - /// Check if there's a new frame since last read - pub fn has_new_frame(&self) -> bool { - let current = self.frame_count.load(Ordering::Acquire); - let last = self.last_read_count.load(Ordering::Acquire); - current > last - } - - /// Read the latest frame (called by renderer) - /// Returns None if no frame available or no new frame since last read - /// Uses take() instead of clone() to avoid copying ~3MB per frame - pub fn read(&self) -> Option { - let current = self.frame_count.load(Ordering::Acquire); - let last = self.last_read_count.load(Ordering::Acquire); - - if current > last { - self.last_read_count.store(current, Ordering::Release); - self.frame.lock().take() // Move instead of clone - zero copy - } else { - None - } - } - - /// Get frame count for stats - pub fn frame_count(&self) -> u64 { - self.frame_count.load(Ordering::Relaxed) - } -} - -impl Default for SharedFrame { - fn default() -> Self { - Self::new() - } -} - -/// Parse resolution string (e.g., "1920x1080") into (width, height) -/// Returns (1920, 1080) as default if parsing fails -pub fn parse_resolution(res: &str) -> (u32, u32) { - let parts: Vec<&str> = res.split('x').collect(); - if parts.len() == 2 { - let width = parts[0].parse().unwrap_or(1920); - let height = parts[1].parse().unwrap_or(1080); - (width, height) - } else { - (1920, 1080) // Default to 1080p - } -} - -/// Game variant (platform/store option) -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub struct GameVariant { - pub id: String, - pub store: String, - #[serde(default)] - pub supported_controls: Vec, -} - -/// Game information -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub struct GameInfo { - pub id: String, - pub title: String, - pub publisher: Option, - pub image_url: Option, - pub store: String, - pub app_id: Option, - #[serde(default)] - pub is_install_to_play: bool, - #[serde(default)] - pub play_type: Option, - #[serde(default)] - pub membership_tier_label: Option, - #[serde(default)] - pub playability_text: Option, - #[serde(default)] - pub uuid: Option, - #[serde(default)] - pub description: Option, - /// Available platform variants (e.g., Steam, Epic, Xbox) - #[serde(default)] - pub variants: Vec, - /// Index of the currently selected variant - #[serde(default)] - pub selected_variant_index: usize, -} - -/// Section of games with a title (e.g., "Trending", "Free to Play") -#[derive(Debug, Clone, Default)] -pub struct GameSection { - pub id: Option, - pub title: String, - pub games: Vec, -} - -/// Subscription information -#[derive(Debug, Clone, Default)] -pub struct SubscriptionInfo { - pub membership_tier: String, - pub remaining_hours: f32, - pub total_hours: f32, - pub has_persistent_storage: bool, - pub storage_size_gb: Option, - pub is_unlimited: bool, // true if subType is UNLIMITED (no hour cap) - pub entitled_resolutions: Vec, -} - -#[derive( - Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize, serde::Deserialize, -)] -pub struct EntitledResolution { - pub width: u32, - pub height: u32, - pub fps: u32, -} - -/// Current tab in Games view -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum GamesTab { - Home, // Sectioned home view (like official GFN client) - AllGames, // Flat grid view - MyLibrary, // User's library - QueueTimes, // Queue times for games (hidden, for free tier users) -} - -impl Default for GamesTab { - fn default() -> Self { - GamesTab::Home // Default to sectioned home view - } -} - -/// Sort mode for queue times display -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub enum QueueSortMode { - #[default] - BestValue, // Balanced score of ping + queue time (recommended) - QueueTime, // Shortest queue first - Ping, // Lowest ping first - Alphabetical, // A-Z by server name -} - -impl QueueSortMode { - pub fn label(&self) -> &'static str { - match self { - QueueSortMode::BestValue => "Best Value", - QueueSortMode::QueueTime => "Shortest Queue", - QueueSortMode::Ping => "Lowest Ping", - QueueSortMode::Alphabetical => "A-Z", - } - } -} - -/// Filter mode for queue times display -#[derive(Debug, Clone, PartialEq, Eq, Default)] -pub enum QueueRegionFilter { - #[default] - All, - Region(String), // Filter by specific region -} - -/// Server/Region information -#[derive(Debug, Clone)] -pub struct ServerInfo { - pub id: String, - pub name: String, - pub region: String, - pub url: Option, - pub ping_ms: Option, - pub status: ServerStatus, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ServerStatus { - Online, - Testing, - Offline, - Unknown, -} - -/// UI actions that can be triggered from the renderer -#[derive(Debug, Clone)] -pub enum UiAction { - /// Start OAuth login flow - StartLogin, - /// Select a login provider - SelectProvider(usize), - /// Logout - Logout, - /// Launch a game by index - LaunchGame(usize), - /// Launch a specific game - LaunchGameDirect(GameInfo), - /// Stop streaming - StopStreaming, - /// Toggle stats overlay - ToggleStats, - /// Update search query - UpdateSearch(String), - /// Toggle settings panel - ToggleSettings, - /// Update a setting - UpdateSetting(SettingChange), - /// Refresh games list - RefreshGames, - /// Switch to a tab - SwitchTab(GamesTab), - /// Open game detail popup - OpenGamePopup(GameInfo), - /// Close game detail popup - CloseGamePopup, - /// Select a platform variant for the current game popup - SelectVariant(usize), - /// Select a server/region - SelectServer(usize), - /// Enable auto server selection (best ping) - SetAutoServerSelection(bool), - /// Start ping test for all servers - StartPingTest, - /// Toggle settings modal - ToggleSettingsModal, - /// Resume an active session - ResumeSession(super::session::ActiveSessionInfo), - /// Terminate existing session and start new game - TerminateAndLaunch(String, GameInfo), - /// Close session conflict dialog - CloseSessionConflict, - /// Close AV1 warning dialog - CloseAV1Warning, - /// Close Alliance experimental warning dialog - CloseAllianceWarning, - /// Close welcome popup - CloseWelcomePopup, - /// Reset all settings to defaults - ResetSettings, - /// Set queue sort mode - SetQueueSortMode(QueueSortMode), - /// Set queue region filter - SetQueueRegionFilter(QueueRegionFilter), - /// Show server selection modal (for free tier users) - ShowServerSelection(GameInfo), - /// Close server selection modal - CloseServerSelection, - /// Select a queue server for launching - SelectQueueServer(Option), - /// Launch game with selected queue server - LaunchWithServer(GameInfo, Option), - /// Refresh queue times - RefreshQueueTimes, - /// Update window size (width, height) - saved to settings - UpdateWindowSize(u32, u32), -} - -/// Setting changes -#[derive(Debug, Clone)] -pub enum SettingChange { - Resolution(String), - Fps(u32), - Codec(VideoCodec), - MaxBitrate(u32), - Fullscreen(bool), - VSync(bool), - LowLatency(bool), - DecoderBackend(VideoDecoderBackend), - ColorQuality(ColorQuality), - Hdr(bool), - ClipboardPasteEnabled(bool), -} - -/// Application state enum -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum AppState { - /// Login screen - Login, - /// Browsing games library - Games, - /// Session being set up (queue, launching) - Session, - /// Active streaming - Streaming, -} diff --git a/opennow-streamer/src/auth/mod.rs b/opennow-streamer/src/auth/mod.rs deleted file mode 100644 index 0304454..0000000 --- a/opennow-streamer/src/auth/mod.rs +++ /dev/null @@ -1,722 +0,0 @@ -//! Authentication Module -//! -//! OAuth flow and token management for NVIDIA accounts. -//! Supports multi-region login via Alliance Partners. - -use anyhow::{Result, Context}; -use log::{info, debug}; -use serde::{Deserialize, Serialize}; -use sha2::{Sha256, Digest}; -use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; -use std::sync::Arc; -use parking_lot::RwLock; - -/// Service URLs API endpoint -const SERVICE_URLS_ENDPOINT: &str = "https://pcs.geforcenow.com/v1/serviceUrls"; - -/// OAuth client configuration -const CLIENT_ID: &str = "ZU7sPN-miLujMD95LfOQ453IB0AtjM8sMyvgJ9wCXEQ"; -const SCOPES: &str = "openid consent email tk_client age"; - -/// Default NVIDIA IDP ID -const DEFAULT_IDP_ID: &str = "PDiAhv2kJTFeQ7WOPqiQ2tRZ7lGhR2X11dXvM4TZSxg"; - -/// GFN CEF User-Agent -const GFN_USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 NVIDIACEFClient/HEAD/debb5919f6 GFN-PC/2.0.80.173"; - -/// CEF Origin for CORS -const CEF_ORIGIN: &str = "https://nvfile"; - -/// Available redirect ports -const REDIRECT_PORTS: [u16; 5] = [2259, 6460, 7119, 8870, 9096]; - -// ============================================ -// Login Provider (Alliance Partner) Support -// ============================================ - -/// Login provider from service URLs API -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct LoginProvider { - /// Unique IDP ID for OAuth - pub idp_id: String, - /// Provider code (e.g., "NVIDIA", "KDD", "TWM") - pub login_provider_code: String, - /// Display name (e.g., "NVIDIA", "au", "Taiwan Mobile") - pub login_provider_display_name: String, - /// Internal provider name - pub login_provider: String, - /// Streaming service base URL - pub streaming_service_url: String, - /// Priority for sorting - #[serde(default)] - pub login_provider_priority: i32, -} - -impl LoginProvider { - /// Create default NVIDIA provider - pub fn nvidia_default() -> Self { - Self { - idp_id: DEFAULT_IDP_ID.to_string(), - login_provider_code: "NVIDIA".to_string(), - login_provider_display_name: "NVIDIA".to_string(), - login_provider: "NVIDIA".to_string(), - streaming_service_url: "https://prod.cloudmatchbeta.nvidiagrid.net/".to_string(), - login_provider_priority: 0, - } - } - - /// Check if this is an Alliance Partner (non-NVIDIA) - pub fn is_alliance_partner(&self) -> bool { - self.login_provider_code != "NVIDIA" - } -} - -/// Service URLs API response -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct ServiceUrlsResponse { - request_status: RequestStatus, - gfn_service_info: Option, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct RequestStatus { - status_code: i32, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct GfnServiceInfo { - gfn_service_endpoints: Vec, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct ServiceEndpoint { - idp_id: String, - login_provider_code: String, - login_provider_display_name: String, - login_provider: String, - streaming_service_url: String, - #[serde(default)] - login_provider_priority: i32, -} - -lazy_static::lazy_static! { - static ref SELECTED_PROVIDER: Arc>> = Arc::new(RwLock::new(None)); - static ref CACHED_PROVIDERS: Arc>> = Arc::new(RwLock::new(Vec::new())); -} - -/// Fetch available login providers from GFN service URLs API -pub async fn fetch_login_providers() -> Result> { - info!("Fetching login providers from {}", SERVICE_URLS_ENDPOINT); - - let client = reqwest::Client::builder() - .user_agent(GFN_USER_AGENT) - .build()?; - - let response = client - .get(SERVICE_URLS_ENDPOINT) - .header("Accept", "application/json") - .send() - .await - .context("Failed to fetch service URLs")?; - - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - return Err(anyhow::anyhow!("Service URLs request failed: {} - {}", status, body)); - } - - let service_response: ServiceUrlsResponse = response.json().await - .context("Failed to parse service URLs response")?; - - if service_response.request_status.status_code != 1 { - return Err(anyhow::anyhow!("Service URLs API error: status_code={}", - service_response.request_status.status_code)); - } - - let service_info = service_response.gfn_service_info - .ok_or_else(|| anyhow::anyhow!("No service info in response"))?; - - let mut providers: Vec = service_info.gfn_service_endpoints - .into_iter() - .map(|ep| { - // Rename "Brothers Pictures" to "bro.game" - let display_name = if ep.login_provider_code == "BPC" { - "bro.game".to_string() - } else { - ep.login_provider_display_name - }; - - LoginProvider { - idp_id: ep.idp_id, - login_provider_code: ep.login_provider_code, - login_provider_display_name: display_name, - login_provider: ep.login_provider, - streaming_service_url: ep.streaming_service_url, - login_provider_priority: ep.login_provider_priority, - } - }) - .collect(); - - // Sort by priority - providers.sort_by_key(|p| p.login_provider_priority); - - info!("Found {} login providers", providers.len()); - for provider in &providers { - debug!(" - {} ({})", provider.login_provider_display_name, provider.login_provider_code); - } - - // Cache providers - { - let mut cache = CACHED_PROVIDERS.write(); - *cache = providers.clone(); - } - - Ok(providers) -} - -/// Get cached login providers -pub fn get_cached_providers() -> Vec { - CACHED_PROVIDERS.read().clone() -} - -/// Set the selected login provider -pub fn set_login_provider(provider: LoginProvider) { - info!("Setting login provider to: {} ({})", - provider.login_provider_display_name, provider.idp_id); - - // Save to cache for persistence across restarts - crate::app::cache::save_login_provider(&provider); - - let mut selected = SELECTED_PROVIDER.write(); - *selected = Some(provider); -} - -/// Get the selected login provider (or default NVIDIA) -pub fn get_selected_provider() -> LoginProvider { - SELECTED_PROVIDER.read() - .clone() - .unwrap_or_else(LoginProvider::nvidia_default) -} - -/// Get the streaming base URL for the selected provider -pub fn get_streaming_base_url() -> String { - let provider = get_selected_provider(); - let url = provider.streaming_service_url; - if url.ends_with('/') { url } else { format!("{}/", url) } -} - -/// Clear the selected provider (reset to NVIDIA default) -pub fn clear_login_provider() { - let mut selected = SELECTED_PROVIDER.write(); - *selected = None; -} - -// ============================================ -// Authentication Tokens -// ============================================ - -/// Authentication tokens -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AuthTokens { - pub access_token: String, - pub refresh_token: Option, - pub id_token: Option, - pub expires_at: i64, -} - -impl AuthTokens { - /// Check if token is expired - pub fn is_expired(&self) -> bool { - let now = chrono::Utc::now().timestamp(); - now >= self.expires_at - } - - /// Check if token should be refreshed (expires within 10 minutes) - pub fn should_refresh(&self) -> bool { - let now = chrono::Utc::now().timestamp(); - // Refresh if less than 10 minutes (600 seconds) remaining - self.expires_at - now < 600 - } - - /// Check if we have a refresh token available - pub fn can_refresh(&self) -> bool { - self.refresh_token.is_some() - } - - /// Get the JWT token for API calls (id_token if available, else access_token) - pub fn jwt(&self) -> &str { - self.id_token.as_deref().unwrap_or(&self.access_token) - } - - /// Extract user_id from the JWT token - pub fn user_id(&self) -> String { - // Try to extract user_id from JWT payload - let token = self.jwt(); - let parts: Vec<&str> = token.split('.').collect(); - if parts.len() == 3 { - let payload_b64 = parts[1]; - // Add padding if needed - let padded = match payload_b64.len() % 4 { - 2 => format!("{}==", payload_b64), - 3 => format!("{}=", payload_b64), - _ => payload_b64.to_string(), - }; - if let Ok(payload_bytes) = URL_SAFE_NO_PAD.decode(&padded) - .or_else(|_| base64::engine::general_purpose::STANDARD.decode(&padded)) - { - if let Ok(payload_str) = String::from_utf8(payload_bytes) { - #[derive(Deserialize)] - struct JwtSub { sub: String } - if let Ok(payload) = serde_json::from_str::(&payload_str) { - return payload.sub; - } - } - } - } - // Fallback - "unknown".to_string() - } -} - -/// User info from JWT or userinfo endpoint -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct UserInfo { - pub user_id: String, - pub display_name: String, - pub email: Option, - pub avatar_url: Option, - pub membership_tier: String, -} - -// ============================================ -// PKCE Challenge -// ============================================ - -/// PKCE code verifier and challenge -pub struct PkceChallenge { - pub verifier: String, - pub challenge: String, -} - -impl PkceChallenge { - /// Generate a new PKCE challenge - pub fn new() -> Self { - // Generate random 64-character verifier - let verifier: String = (0..64) - .map(|_| { - let idx = rand::random::() % 62; - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" - .chars() - .nth(idx) - .unwrap() - }) - .collect(); - - // Generate SHA256 challenge - let mut hasher = Sha256::new(); - hasher.update(verifier.as_bytes()); - let challenge = URL_SAFE_NO_PAD.encode(hasher.finalize()); - - Self { verifier, challenge } - } -} - -impl Default for PkceChallenge { - fn default() -> Self { - Self::new() - } -} - -// ============================================ -// OAuth Flow -// ============================================ - -/// Find an available port for OAuth callback -pub fn find_available_port() -> Option { - for port in REDIRECT_PORTS { - if std::net::TcpListener::bind(format!("127.0.0.1:{}", port)).is_ok() { - return Some(port); - } - } - None -} - -/// Build the OAuth authorization URL with provider-specific IDP ID -pub fn build_auth_url(pkce: &PkceChallenge, port: u16) -> String { - let provider = get_selected_provider(); - let redirect_uri = format!("http://localhost:{}", port); - let nonce = generate_nonce(); - let device_id = get_device_id(); - - format!( - "https://login.nvidia.com/authorize?\ - response_type=code&\ - device_id={}&\ - scope={}&\ - client_id={}&\ - redirect_uri={}&\ - ui_locales=en_US&\ - nonce={}&\ - prompt=select_account&\ - code_challenge={}&\ - code_challenge_method=S256&\ - idp_id={}", - device_id, - urlencoding::encode(SCOPES), - CLIENT_ID, - urlencoding::encode(&redirect_uri), - nonce, - pkce.challenge, - provider.idp_id - ) -} - -/// Generate a UUID-like nonce -fn generate_nonce() -> String { - use std::time::{SystemTime, UNIX_EPOCH}; - - let timestamp = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos(); - - let mut hasher = Sha256::new(); - hasher.update(timestamp.to_le_bytes()); - hasher.update(std::process::id().to_le_bytes()); - hasher.update(b"nonce"); - let hash = hasher.finalize(); - - format!( - "{:08x}-{:04x}-{:04x}-{:04x}-{:012x}", - u32::from_le_bytes([hash[0], hash[1], hash[2], hash[3]]), - u16::from_le_bytes([hash[4], hash[5]]), - u16::from_le_bytes([hash[6], hash[7]]), - u16::from_le_bytes([hash[8], hash[9]]), - u64::from_le_bytes([hash[10], hash[11], hash[12], hash[13], hash[14], hash[15], 0, 0]) & 0xffffffffffff - ) -} - -/// Get or generate device ID -fn get_device_id() -> String { - // Try to read from official GFN client config - if let Some(app_data) = std::env::var_os("LOCALAPPDATA") { - let gfn_config = std::path::PathBuf::from(app_data) - .join("NVIDIA Corporation") - .join("GeForceNOW") - .join("sharedstorage.json"); - - if gfn_config.exists() { - if let Ok(content) = std::fs::read_to_string(&gfn_config) { - if let Ok(json) = serde_json::from_str::(&content) { - if let Some(device_id) = json.get("gfnTelemetry") - .and_then(|t| t.get("deviceId")) - .and_then(|d| d.as_str()) { - return device_id.to_string(); - } - } - } - } - } - - // Generate stable device ID - let mut hasher = Sha256::new(); - if let Ok(hostname) = std::env::var("COMPUTERNAME") { - hasher.update(hostname.as_bytes()); - } - if let Ok(username) = std::env::var("USERNAME") { - hasher.update(username.as_bytes()); - } - hasher.update(b"opennow-streamer"); - hex::encode(hasher.finalize()) -} - -/// Exchange authorization code for tokens -pub async fn exchange_code(code: &str, verifier: &str, port: u16) -> Result { - let redirect_uri = format!("http://localhost:{}", port); - - info!("Exchanging authorization code for tokens..."); - - let client = reqwest::Client::builder() - .user_agent(GFN_USER_AGENT) - .build()?; - - // Official client does NOT include client_id in token request - let params = [ - ("grant_type", "authorization_code"), - ("code", code), - ("redirect_uri", redirect_uri.as_str()), - ("code_verifier", verifier), - ]; - - let response = client - .post("https://login.nvidia.com/token") - .header("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8") - .header("Origin", CEF_ORIGIN) - .header("Referer", format!("{}/", CEF_ORIGIN)) - .header("Accept", "application/json, text/plain, */*") - .form(¶ms) - .send() - .await - .context("Token exchange request failed")?; - - if !response.status().is_success() { - let status = response.status(); - let error_text = response.text().await.unwrap_or_default(); - return Err(anyhow::anyhow!("Token exchange failed: {} - {}", status, error_text)); - } - - #[derive(Deserialize)] - struct TokenResponse { - access_token: String, - refresh_token: Option, - id_token: Option, - expires_in: Option, - } - - let token_response: TokenResponse = response.json().await - .context("Failed to parse token response")?; - - let expires_at = chrono::Utc::now().timestamp() - + token_response.expires_in.unwrap_or(86400); - - info!("Token exchange successful!"); - - Ok(AuthTokens { - access_token: token_response.access_token, - refresh_token: token_response.refresh_token, - id_token: token_response.id_token, - expires_at, - }) -} - -/// Refresh an expired token -pub async fn refresh_token(refresh_token: &str) -> Result { - let client = reqwest::Client::builder() - .user_agent(GFN_USER_AGENT) - .build()?; - - let params = [ - ("grant_type", "refresh_token"), - ("refresh_token", refresh_token), - ("client_id", CLIENT_ID), - ]; - - let response = client - .post("https://login.nvidia.com/token") - .header("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8") - .header("Origin", CEF_ORIGIN) - .header("Accept", "application/json, text/plain, */*") - .form(¶ms) - .send() - .await - .context("Token refresh request failed")?; - - if !response.status().is_success() { - let error_text = response.text().await.unwrap_or_default(); - return Err(anyhow::anyhow!("Token refresh failed: {}", error_text)); - } - - #[derive(Deserialize)] - struct TokenResponse { - access_token: String, - refresh_token: Option, - id_token: Option, - expires_in: Option, - } - - let token_response: TokenResponse = response.json().await - .context("Failed to parse refresh response")?; - - let expires_at = chrono::Utc::now().timestamp() - + token_response.expires_in.unwrap_or(86400); - - Ok(AuthTokens { - access_token: token_response.access_token, - refresh_token: token_response.refresh_token, - id_token: token_response.id_token, - expires_at, - }) -} - -/// Decode JWT and extract user info -pub fn decode_jwt_user_info(token: &str) -> Result { - let parts: Vec<&str> = token.split('.').collect(); - if parts.len() != 3 { - return Err(anyhow::anyhow!("Invalid JWT format")); - } - - let payload_b64 = parts[1]; - let padded = match payload_b64.len() % 4 { - 2 => format!("{}==", payload_b64), - 3 => format!("{}=", payload_b64), - _ => payload_b64.to_string(), - }; - - let payload_bytes = URL_SAFE_NO_PAD.decode(&padded) - .or_else(|_| base64::engine::general_purpose::STANDARD.decode(&padded)) - .context("Failed to decode JWT payload")?; - - let payload_str = String::from_utf8(payload_bytes) - .context("Invalid UTF-8 in JWT")?; - - #[derive(Deserialize)] - struct JwtPayload { - sub: String, - email: Option, - preferred_username: Option, - gfn_tier: Option, - picture: Option, - } - - let payload: JwtPayload = serde_json::from_str(&payload_str) - .context("Failed to parse JWT payload")?; - - let display_name = payload.preferred_username - .or_else(|| payload.email.as_ref().map(|e| e.split('@').next().unwrap_or("User").to_string())) - .unwrap_or_else(|| "User".to_string()); - - let membership_tier = payload.gfn_tier.unwrap_or_else(|| "FREE".to_string()); - - Ok(UserInfo { - user_id: payload.sub, - display_name, - email: payload.email, - avatar_url: payload.picture, - membership_tier, - }) -} - -/// Fetch user info from /userinfo endpoint -pub async fn fetch_userinfo(access_token: &str) -> Result { - let client = reqwest::Client::builder() - .user_agent(GFN_USER_AGENT) - .build()?; - - let response = client - .get("https://login.nvidia.com/userinfo") - .header("Authorization", format!("Bearer {}", access_token)) - .header("Origin", CEF_ORIGIN) - .header("Accept", "application/json") - .send() - .await - .context("Userinfo request failed")?; - - if !response.status().is_success() { - return Err(anyhow::anyhow!("Userinfo failed: {}", response.status())); - } - - #[derive(Deserialize)] - struct UserinfoResponse { - sub: String, - preferred_username: Option, - email: Option, - picture: Option, - } - - let userinfo: UserinfoResponse = response.json().await - .context("Failed to parse userinfo")?; - - let display_name = userinfo.preferred_username - .or_else(|| userinfo.email.as_ref().map(|e| e.split('@').next().unwrap_or("User").to_string())) - .unwrap_or_else(|| "User".to_string()); - - Ok(UserInfo { - user_id: userinfo.sub, - display_name, - email: userinfo.email, - avatar_url: userinfo.picture, - membership_tier: "FREE".to_string(), // /userinfo doesn't return tier - }) -} - -/// Get user info from tokens (prefer id_token JWT, fallback to /userinfo) -pub async fn get_user_info(tokens: &AuthTokens) -> Result { - // Try id_token first - if let Some(ref id_token) = tokens.id_token { - if let Ok(user) = decode_jwt_user_info(id_token) { - return Ok(user); - } - } - - // Fallback to /userinfo - fetch_userinfo(&tokens.access_token).await -} - -/// Start OAuth callback server and wait for code -pub async fn start_callback_server(port: u16) -> Result { - use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; - use tokio::net::TcpListener; - - let listener = TcpListener::bind(format!("127.0.0.1:{}", port)).await - .context("Failed to bind callback server")?; - - info!("OAuth callback server listening on http://127.0.0.1:{}", port); - - let (mut socket, _) = listener.accept().await - .context("Failed to accept connection")?; - - let mut reader = BufReader::new(&mut socket); - let mut request_line = String::new(); - reader.read_line(&mut request_line).await?; - - // Parse the code from: GET /callback?code=abc123 HTTP/1.1 - let code = request_line - .split_whitespace() - .nth(1) - .and_then(|path| { - path.split('?') - .nth(1) - .and_then(|query| { - query.split('&') - .find(|param| param.starts_with("code=")) - .map(|param| param.trim_start_matches("code=").to_string()) - }) - }) - .context("No authorization code in callback")?; - - // Send success response - let response = r#"HTTP/1.1 200 OK -Content-Type: text/html - - - - - Login Successful - - - -
-

Login Successful!

-

You can close this window and return to OpenNow Streamer.

-
- - -"#; - - socket.write_all(response.as_bytes()).await?; - - Ok(code) -} diff --git a/opennow-streamer/src/gui/image_cache.rs b/opennow-streamer/src/gui/image_cache.rs deleted file mode 100644 index 35553e7..0000000 --- a/opennow-streamer/src/gui/image_cache.rs +++ /dev/null @@ -1,188 +0,0 @@ -//! Image Cache for Game Art -//! -//! Loads and caches game box art images for display in the UI. - -use std::collections::HashMap; -use std::sync::Arc; -use parking_lot::RwLock; -use log::{debug, warn}; - -/// Image loading state -#[derive(Clone)] -pub enum ImageState { - /// Not yet requested - NotLoaded, - /// Currently loading - Loading, - /// Successfully loaded (RGBA pixels, width, height) - Loaded(Arc>, u32, u32), - /// Failed to load - Failed, -} - -/// Global image cache -pub struct ImageCache { - /// Map from URL to image state - images: RwLock>, - /// HTTP client for fetching images - client: reqwest::Client, -} - -impl ImageCache { - pub fn new() -> Self { - let client = reqwest::Client::builder() - .user_agent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/128.0.0.0") - .build() - .expect("Failed to create HTTP client"); - - Self { - images: RwLock::new(HashMap::new()), - client, - } - } - - /// Get image state for a URL - pub fn get(&self, url: &str) -> ImageState { - let images = self.images.read(); - images.get(url).cloned().unwrap_or(ImageState::NotLoaded) - } - - /// Request loading an image (non-blocking) - pub fn request_load(&self, url: String, runtime: tokio::runtime::Handle) { - // Check if already loading or loaded - { - let images = self.images.read(); - if images.contains_key(&url) { - return; // Already in progress or loaded - } - } - - // Mark as loading - { - let mut images = self.images.write(); - images.insert(url.clone(), ImageState::Loading); - } - - // Spawn async task to load image - let client = self.client.clone(); - let url_clone = url.clone(); - let _images = Arc::new(self.images.read().clone()); - - // We need to use a static or leaked reference for the cache update - // For simplicity, we'll use a channel pattern - runtime.spawn(async move { - match Self::load_image_async(&client, &url_clone).await { - Ok((pixels, width, height)) => { - debug!("Loaded image: {} ({}x{})", url_clone, width, height); - LOADED_IMAGES.write().insert(url_clone, ImageState::Loaded(Arc::new(pixels), width, height)); - } - Err(e) => { - warn!("Failed to load image {}: {}", url_clone, e); - LOADED_IMAGES.write().insert(url_clone, ImageState::Failed); - } - } - }); - } - - /// Load an image asynchronously - /// Images are resized to max 300x400 for efficient UI rendering on low-end devices - async fn load_image_async(client: &reqwest::Client, url: &str) -> anyhow::Result<(Vec, u32, u32)> { - use anyhow::Context; - - let response = client.get(url) - .header("Accept", "image/webp,image/png,image/jpeg,*/*") - .header("Referer", "https://play.geforcenow.com/") - .send() - .await - .context("Failed to fetch image")?; - - if !response.status().is_success() { - return Err(anyhow::anyhow!("Image fetch failed: {}", response.status())); - } - - let bytes = response.bytes().await - .context("Failed to read image bytes")?; - - // Decode image - let img = image::load_from_memory(&bytes) - .context("Failed to decode image")?; - - // Resize to thumbnail size for efficient UI rendering - // Game cards are typically displayed at ~200x280 pixels - // Cap at 300x400 to balance quality and performance on low-end devices - const MAX_WIDTH: u32 = 300; - const MAX_HEIGHT: u32 = 400; - - let (orig_w, orig_h) = (img.width(), img.height()); - let img = if orig_w > MAX_WIDTH || orig_h > MAX_HEIGHT { - let scale = (MAX_WIDTH as f32 / orig_w as f32) - .min(MAX_HEIGHT as f32 / orig_h as f32); - let new_w = (orig_w as f32 * scale) as u32; - let new_h = (orig_h as f32 * scale) as u32; - debug!("Resizing image {}x{} -> {}x{}", orig_w, orig_h, new_w, new_h); - img.resize(new_w, new_h, image::imageops::FilterType::Triangle) - } else { - img - }; - - let rgba = img.to_rgba8(); - let width = rgba.width(); - let height = rgba.height(); - let pixels = rgba.into_raw(); - - Ok((pixels, width, height)) - } - - /// Check for newly loaded images and update cache - pub fn update(&self) { - let loaded = LOADED_IMAGES.read().clone(); - if !loaded.is_empty() { - let mut images = self.images.write(); - for (url, state) in loaded.iter() { - images.insert(url.clone(), state.clone()); - } - } - } -} - -impl Default for ImageCache { - fn default() -> Self { - Self::new() - } -} - -// Global storage for loaded images (workaround for async updates) -lazy_static::lazy_static! { - static ref LOADED_IMAGES: RwLock> = RwLock::new(HashMap::new()); -} - -lazy_static::lazy_static! { - pub static ref IMAGE_CACHE: ImageCache = ImageCache::new(); -} - -/// Convenience function to get an image (returns None if not loaded yet) -pub fn get_image(url: &str) -> Option<(Arc>, u32, u32)> { - // First check the global loaded images - { - let loaded = LOADED_IMAGES.read(); - if let Some(ImageState::Loaded(pixels, w, h)) = loaded.get(url) { - return Some((pixels.clone(), *w, *h)); - } - } - - // Then check the main cache - match IMAGE_CACHE.get(url) { - ImageState::Loaded(pixels, w, h) => Some((pixels, w, h)), - _ => None, - } -} - -/// Request loading an image -pub fn request_image(url: &str, runtime: &tokio::runtime::Handle) { - IMAGE_CACHE.request_load(url.to_string(), runtime.clone()); -} - -/// Update the image cache (call from main loop) -pub fn update_cache() { - IMAGE_CACHE.update(); -} diff --git a/opennow-streamer/src/gui/mod.rs b/opennow-streamer/src/gui/mod.rs deleted file mode 100644 index 56cbb5f..0000000 --- a/opennow-streamer/src/gui/mod.rs +++ /dev/null @@ -1,13 +0,0 @@ -//! GUI Module -//! -//! Window management, rendering, and stats overlay. - -mod renderer; -mod stats_panel; -mod shaders; -pub mod screens; -pub mod image_cache; - -pub use renderer::Renderer; -pub use stats_panel::StatsPanel; -pub use image_cache::{get_image, request_image, update_cache}; diff --git a/opennow-streamer/src/gui/renderer.rs b/opennow-streamer/src/gui/renderer.rs deleted file mode 100644 index 3a7a506..0000000 --- a/opennow-streamer/src/gui/renderer.rs +++ /dev/null @@ -1,6461 +0,0 @@ -//! GPU Renderer -//! -//! wgpu-based rendering for video frames and UI overlays. - -// Local profiling macro for Tracy integration -// When tracy feature is enabled, creates tracing spans that Tracy visualizes -macro_rules! profile_scope { - ($name:expr) => { - #[cfg(feature = "tracy")] - let _span = tracing::info_span!($name).entered(); - #[cfg(not(feature = "tracy"))] - let _ = $name; // Suppress unused warning - }; -} - -use anyhow::{Context, Result}; -use log::{debug, error, info, warn}; -use std::sync::Arc; -use winit::dpi::PhysicalSize; -use winit::event::WindowEvent; -use winit::event_loop::ActiveEventLoop; -use winit::window::{CursorGrabMode, Fullscreen, Window, WindowAttributes}; - -#[cfg(target_os = "macos")] -use winit::raw_window_handle::{HasWindowHandle, RawWindowHandle}; -// use wgpu::util::DeviceExt; - -use super::image_cache; -use super::screens::{ - render_ads_required_screen, render_alliance_warning_dialog, render_av1_warning_dialog, - render_login_screen, render_session_conflict_dialog, render_session_screen, - render_settings_modal, render_welcome_popup, -}; -use super::shaders::{EXTERNAL_TEXTURE_SHADER, NV12_HDR_TONEMAP_SHADER, NV12_SHADER, VIDEO_SHADER}; -use super::StatsPanel; -use crate::app::session::ActiveSessionInfo; -use crate::app::{App, AppState, GameInfo, GamesTab, UiAction}; -#[cfg(target_os = "windows")] -use crate::media::D3D11TextureWrapper; -#[cfg(target_os = "linux")] -use crate::media::VAAPISurfaceWrapper; -#[cfg(target_os = "macos")] -use crate::media::{CVMetalTexture, MetalVideoRenderer, ZeroCopyTextureManager}; -use crate::media::{ColorSpace, PixelFormat, StreamStats, TransferFunction, VideoFrame}; -use std::collections::HashMap; -use std::time::{Duration, Instant}; -#[cfg(target_os = "windows")] -// unused: use windows::core::Interface; -#[cfg(target_os = "windows")] -#[cfg(target_os = "macos")] -use wgpu_hal::dx12; -#[cfg(target_os = "windows")] -use windows::Win32::Foundation::HANDLE; -#[cfg(target_os = "windows")] -use windows::Win32::Graphics::Direct3D12::{ID3D12Device, ID3D12Resource}; - -// Color conversion is now hardcoded in the shader using official GFN client BT.709 values -// This eliminates potential initialization bugs with uniform buffers - -/// Resolution change notification for animated popup -struct ResolutionNotification { - old_resolution: String, - new_resolution: String, - direction: ResolutionDirection, - start_time: Instant, -} - -#[derive(Clone, Copy, PartialEq)] -enum ResolutionDirection { - Up, - Down, - Same, -} - -impl ResolutionNotification { - const DURATION_SECS: f32 = 5.0; - const FADE_IN_SECS: f32 = 0.3; - const FADE_OUT_SECS: f32 = 0.7; - - fn new(old_res: &str, new_res: &str) -> Self { - // Parse resolutions to determine direction - let old_pixels = Self::parse_resolution(old_res); - let new_pixels = Self::parse_resolution(new_res); - - let direction = if new_pixels > old_pixels { - ResolutionDirection::Up - } else if new_pixels < old_pixels { - ResolutionDirection::Down - } else { - ResolutionDirection::Same - }; - - Self { - old_resolution: old_res.to_string(), - new_resolution: new_res.to_string(), - direction, - start_time: Instant::now(), - } - } - - fn parse_resolution(res: &str) -> u64 { - // Parse "1920x1080" or "1920x1080 @ 60fps" format - let parts: Vec<&str> = res.split(['x', ' ', '@']).collect(); - if parts.len() >= 2 { - let w: u64 = parts[0].trim().parse().unwrap_or(0); - let h: u64 = parts[1].trim().parse().unwrap_or(0); - w * h - } else { - 0 - } - } - - fn is_expired(&self) -> bool { - self.start_time.elapsed().as_secs_f32() > Self::DURATION_SECS - } - - fn alpha(&self) -> f32 { - let elapsed = self.start_time.elapsed().as_secs_f32(); - - if elapsed < Self::FADE_IN_SECS { - // Fade in - elapsed / Self::FADE_IN_SECS - } else if elapsed > Self::DURATION_SECS - Self::FADE_OUT_SECS { - // Fade out - let fade_progress = (Self::DURATION_SECS - elapsed) / Self::FADE_OUT_SECS; - fade_progress.max(0.0) - } else { - // Full opacity - 1.0 - } - } -} - -/// Racing wheel connection notification for animated popup -/// Shows when a racing wheel is detected during a streaming session -struct WheelNotification { - wheel_count: usize, - start_time: Instant, -} - -impl WheelNotification { - const DURATION_SECS: f32 = 6.0; - const FADE_IN_SECS: f32 = 0.3; - const FADE_OUT_SECS: f32 = 0.8; - - fn new(wheel_count: usize) -> Self { - Self { - wheel_count, - start_time: Instant::now(), - } - } - - fn is_expired(&self) -> bool { - self.start_time.elapsed().as_secs_f32() > Self::DURATION_SECS - } - - fn alpha(&self) -> f32 { - let elapsed = self.start_time.elapsed().as_secs_f32(); - - if elapsed < Self::FADE_IN_SECS { - // Fade in - elapsed / Self::FADE_IN_SECS - } else if elapsed > Self::DURATION_SECS - Self::FADE_OUT_SECS { - // Fade out - let fade_progress = (Self::DURATION_SECS - elapsed) / Self::FADE_OUT_SECS; - fade_progress.max(0.0) - } else { - // Full opacity - 1.0 - } - } -} - -/// Main renderer -pub struct Renderer { - window: Arc, - surface: wgpu::Surface<'static>, - device: wgpu::Device, - queue: wgpu::Queue, - config: wgpu::SurfaceConfiguration, - size: PhysicalSize, - - // egui integration - egui_ctx: egui::Context, - egui_state: egui_winit::State, - egui_renderer: egui_wgpu::Renderer, - - // Video rendering pipeline (GPU YUV->RGB conversion) - video_pipeline: wgpu::RenderPipeline, - video_bind_group_layout: wgpu::BindGroupLayout, - video_sampler: wgpu::Sampler, - // YUV420P planar textures (Y = full res, U/V = half res for 4:2:0) - y_texture: Option, - u_texture: Option, - v_texture: Option, - video_bind_group: Option, - video_size: (u32, u32), - - // NV12 pipeline (for VideoToolbox on macOS - faster than CPU scaler) - nv12_pipeline: wgpu::RenderPipeline, - nv12_bind_group_layout: wgpu::BindGroupLayout, - // NV12 HDR tone mapping pipeline (for HDR content on SDR displays) - nv12_hdr_pipeline: wgpu::RenderPipeline, - // NV12 textures: Y (R8) and UV interleaved (Rg8) - uv_texture: Option, - nv12_bind_group: Option, - // Current pixel format - current_format: PixelFormat, - // Current transfer function (for HDR detection) - current_transfer_function: TransferFunction, - - // Direct access to decoder's frame buffer - pull frames here, not from App - shared_frame: Option>, - - // External Texture pipeline (true zero-copy hardware YUV->RGB) - external_texture_pipeline: Option, - external_texture_bind_group_layout: Option, - external_texture_bind_group: Option, - external_texture: Option, - external_texture_supported: bool, - - // Stats panel - stats_panel: StatsPanel, - - // Fullscreen state - fullscreen: bool, - - // Swapchain error recovery state - // Tracks consecutive Outdated errors to avoid panic-fixing with wrong resolution - consecutive_surface_errors: u32, - - // Supported present modes (for fallback when Immediate isn't available) - supported_present_modes: Vec, - - // Game art texture cache (URL -> TextureHandle) - game_textures: HashMap, - - // === UI Optimization: Stats throttling === - // Cached stats for throttled rendering (updates every 200ms instead of every frame) - cached_stats: Option, - stats_last_update: Instant, - - // === UI Optimization: Game grid caching === - // Cached game grid to avoid re-laying out every frame - games_cache_hash: u64, - - // Track last uploaded frame to avoid redundant GPU uploads - last_uploaded_frame_id: u64, - - // Resolution change notification - resolution_notification: Option, - last_resolution: String, - - // Racing wheel connection notification - wheel_notification: Option, - last_wheel_count: usize, - - // macOS zero-copy video rendering (Metal-based, no CPU copy) - #[cfg(target_os = "macos")] - zero_copy_manager: Option, - #[cfg(target_os = "macos")] - zero_copy_enabled: bool, - // Store current CVMetalTextures to keep them alive during rendering - #[cfg(target_os = "macos")] - current_y_cv_texture: Option, - #[cfg(target_os = "macos")] - current_uv_cv_texture: Option, - #[cfg(target_os = "windows")] - current_imported_handle: Option, - #[cfg(target_os = "windows")] - current_imported_texture: Option, -} - -impl Renderer { - /// Create a new renderer - pub async fn new(event_loop: &ActiveEventLoop) -> Result { - // Load settings to get saved window size - let settings = crate::app::Settings::load().unwrap_or_default(); - - // Create window attributes - // Use saved window size if available, otherwise use defaults - // ARM64 Linux: Start with smaller window to reduce initial GPU memory usage - #[cfg(all(target_os = "linux", target_arch = "aarch64"))] - let default_size = PhysicalSize::new(800u32, 600u32); - #[cfg(not(all(target_os = "linux", target_arch = "aarch64")))] - let default_size = PhysicalSize::new(1280u32, 720u32); - - // Use saved size if valid (non-zero and reasonable), otherwise use default - let initial_size = if settings.window_width >= 640 && settings.window_height >= 480 { - PhysicalSize::new(settings.window_width, settings.window_height) - } else { - default_size - }; - - let window_attrs = WindowAttributes::default() - .with_title("OpenNow") - .with_inner_size(initial_size) - .with_min_inner_size(PhysicalSize::new(640, 480)) - .with_resizable(true); - - // Create window and wrap in Arc for surface creation - let window = Arc::new( - event_loop - .create_window(window_attrs) - .context("Failed to create window")?, - ); - - let size = window.inner_size(); - - info!("Window created: {}x{}", size.width, size.height); - - // On macOS, enable high-performance mode and disable App Nap - #[cfg(target_os = "macos")] - Self::enable_macos_high_performance(); - - // On macOS, set display to 120Hz immediately (before fullscreen) - // This ensures Direct mode uses high refresh rate - #[cfg(target_os = "macos")] - Self::set_macos_display_mode_120hz(); - - // Create wgpu instance - // Force DX12 on Windows for better exclusive fullscreen support and lower latency - // Vulkan on Windows has issues with exclusive fullscreen transitions causing DWM composition - #[cfg(target_os = "windows")] - let backends = wgpu::Backends::DX12; - // ARM Linux (Raspberry Pi, etc): Check WGPU_BACKEND env var, default to Vulkan - #[cfg(all(target_os = "linux", target_arch = "aarch64"))] - let backends = { - match std::env::var("WGPU_BACKEND").ok().as_deref() { - Some("gl") | Some("GL") | Some("gles") | Some("GLES") => { - info!("ARM64 Linux: Using GL backend (from WGPU_BACKEND env var)"); - wgpu::Backends::GL - } - Some("vulkan") | Some("VULKAN") => { - info!("ARM64 Linux: Using Vulkan backend (from WGPU_BACKEND env var)"); - wgpu::Backends::VULKAN - } - _ => { - info!("ARM64 Linux: Using Vulkan backend (default - set WGPU_BACKEND=gl to try OpenGL)"); - wgpu::Backends::VULKAN - } - } - }; - #[cfg(all( - not(target_os = "windows"), - not(all(target_os = "linux", target_arch = "aarch64")) - ))] - let backends = wgpu::Backends::all(); - - info!("Using wgpu backend: {:?}", backends); - - let instance = wgpu::Instance::new(&wgpu::InstanceDescriptor { - backends, - ..Default::default() - }); - - // Create surface from Arc - let surface = instance.create_surface(window.clone()).map_err(|e| { - error!("Surface creation failed: {:?}", e); - #[cfg(all(target_os = "linux", target_arch = "aarch64"))] - { - error!("ARM64 Linux troubleshooting:"); - error!( - " - Ensure Vulkan drivers are installed: sudo apt install mesa-vulkan-drivers" - ); - error!(" - Try: WAYLAND_DISPLAY= ./run.sh (force X11)"); - } - anyhow::anyhow!("Failed to create surface: {:?}", e) - })?; - - // Get adapter - #[cfg(not(all(target_os = "linux", target_arch = "aarch64")))] - let adapter = instance - .request_adapter(&wgpu::RequestAdapterOptions { - power_preference: wgpu::PowerPreference::HighPerformance, - compatible_surface: Some(&surface), - force_fallback_adapter: false, - }) - .await - .context("Failed to find GPU adapter")?; - - // ARM64 Linux: Try hardware GPU first, fall back to llvmpipe if it fails - #[cfg(all(target_os = "linux", target_arch = "aarch64"))] - let adapter = { - let force_sw = std::env::var("OPENNOW_FORCE_SOFTWARE_GPU").is_ok(); - - // Print V3D troubleshooting info - info!("ARM64 Linux: GPU memory tips:"); - info!(" - Check GPU memory: vcgencmd get_mem gpu"); - info!(" - Increase GPU memory: Add 'gpu_mem=512' to /boot/firmware/config.txt"); - info!(" - V3D env vars: MESA_VK_ABORT_ON_DEVICE_LOSS=0 V3D_DEBUG=perf"); - - if force_sw { - info!("ARM64 Linux: Forcing software renderer (OPENNOW_FORCE_SOFTWARE_GPU set)"); - instance - .request_adapter(&wgpu::RequestAdapterOptions { - power_preference: wgpu::PowerPreference::LowPower, - compatible_surface: Some(&surface), - force_fallback_adapter: true, - }) - .await - .context("Failed to find software GPU adapter")? - } else { - // Try hardware GPU first - info!("ARM64 Linux: Trying hardware GPU..."); - match instance - .request_adapter(&wgpu::RequestAdapterOptions { - power_preference: wgpu::PowerPreference::LowPower, - compatible_surface: Some(&surface), - force_fallback_adapter: false, - }) - .await - { - Ok(hw_adapter) => { - let info = hw_adapter.get_info(); - info!(" Hardware GPU found: {}", info.name); - // Check if this is V3D and warn about potential OOM - if info.name.to_lowercase().contains("v3d") { - warn!(" V3D GPU detected - may OOM during device creation"); - warn!(" If OOM occurs, try: OPENNOW_FORCE_SOFTWARE_GPU=1 ./run.sh"); - warn!(" Or increase GPU memory to 512MB in config.txt"); - } - hw_adapter - } - Err(e) => { - warn!(" Hardware GPU failed: {:?}, using software renderer", e); - instance - .request_adapter(&wgpu::RequestAdapterOptions { - power_preference: wgpu::PowerPreference::LowPower, - compatible_surface: Some(&surface), - force_fallback_adapter: true, - }) - .await - .context("Failed to find any GPU adapter")? - } - } - } - }; - - let adapter_info = adapter.get_info(); - info!( - "GPU: {} (Backend: {:?}, Driver: {})", - adapter_info.name, adapter_info.backend, adapter_info.driver_info - ); - - // Print to console directly for visibility (bypasses log filter) - crate::utils::console_print(&format!( - "[GPU] {} using {:?} backend", - adapter_info.name, adapter_info.backend - )); - - // Create device and queue - // Request EXTERNAL_TEXTURE feature for true zero-copy video rendering - let mut required_features = wgpu::Features::empty(); - let adapter_features = adapter.features(); - - // Check if EXTERNAL_TEXTURE is supported (hardware YUV->RGB conversion) - let external_texture_supported = - adapter_features.contains(wgpu::Features::EXTERNAL_TEXTURE); - if external_texture_supported { - required_features |= wgpu::Features::EXTERNAL_TEXTURE; - info!("EXTERNAL_TEXTURE feature supported - enabling true zero-copy video"); - } else { - info!("EXTERNAL_TEXTURE not supported - using NV12 shader path"); - } - - // Detect Raspberry Pi V3D hardware GPU for ultra-minimal settings - let is_v3d_hardware = adapter_info.name.to_lowercase().contains("v3d") - || adapter_info.name.to_lowercase().contains("videocore"); - // Detect if we're on ARM64 Linux (includes llvmpipe on Pi) - let is_arm64_linux = cfg!(all(target_os = "linux", target_arch = "aarch64")); - - // V3D: Don't request any optional features to minimize memory - if is_v3d_hardware { - required_features = wgpu::Features::empty(); - info!("V3D hardware: Disabling all optional features to save memory"); - } - - // Check if we're in legacy macOS mode (for 2015 and older Intel Macs) - #[cfg(all(target_os = "macos", feature = "legacy-macos"))] - let is_legacy_macos = true; - #[cfg(not(all(target_os = "macos", feature = "legacy-macos")))] - let is_legacy_macos = false; - - // Use appropriate limits based on GPU type - let limits = if is_v3d_hardware { - // V3D hardware: Use conservative limits (Pi 4/5 with 512MB+ GPU memory) - info!("V3D hardware GPU detected - using conservative limits for 1080p"); - let mut lim = wgpu::Limits::downlevel_webgl2_defaults(); - // Support 1080p video (1920x1080) and some headroom - lim.max_texture_dimension_1d = 2048; - lim.max_texture_dimension_2d = 2048; // Enough for 1080p - lim.max_texture_dimension_3d = 256; - lim.max_buffer_size = 32 * 1024 * 1024; // 32MB - lim.max_uniform_buffer_binding_size = 64 * 1024; - lim.max_storage_buffer_binding_size = 32 * 1024 * 1024; - lim.max_vertex_buffers = 8; - lim.max_bind_groups = 4; - lim.max_bindings_per_bind_group = 16; - lim.max_samplers_per_shader_stage = 4; - lim.max_sampled_textures_per_shader_stage = 8; - info!(" Max texture: 2048, Max buffer: 32MB, Bind groups: 4"); - lim - } else if is_legacy_macos { - // Legacy macOS (2015 and older Intel Macs with Metal 1.0/1.1) - // Use conservative limits that work on Intel Iris Graphics - info!("Legacy macOS mode: Using conservative limits for Intel Iris Graphics"); - let mut lim = wgpu::Limits::downlevel_defaults(); - // Intel Iris Graphics (2015) supports up to 4096x4096 textures - lim.max_texture_dimension_1d = 4096; - lim.max_texture_dimension_2d = 4096; - lim.max_texture_dimension_3d = 256; - // Conservative buffer sizes for older GPUs - lim.max_buffer_size = 256 * 1024 * 1024; // 256MB - lim.max_uniform_buffer_binding_size = 64 * 1024; - lim.max_storage_buffer_binding_size = 128 * 1024 * 1024; - // Reduce bind groups to be safe on older Metal - lim.max_bind_groups = 4; - lim.max_bindings_per_bind_group = 16; - info!(" Max texture: 4096, Max buffer: 256MB, Bind groups: 4"); - lim - } else if is_arm64_linux { - // llvmpipe or other ARM64: Use downlevel defaults - info!("ARM64 Linux: Using downlevel defaults"); - wgpu::Limits::downlevel_defaults().using_resolution(adapter.limits()) - } else { - // Desktop: Use full adapter limits - wgpu::Limits::downlevel_defaults().using_resolution(adapter.limits()) - }; - - info!( - "Requesting device limits: Max Texture Dimension 2D: {}", - limits.max_texture_dimension_2d - ); - - // ARM64 Linux: Try device creation, fallback to software if V3D OOMs - #[cfg(all(target_os = "linux", target_arch = "aarch64"))] - let (device, queue, adapter, is_v3d_hardware, required_features, limits): ( - wgpu::Device, - wgpu::Queue, - wgpu::Adapter, - bool, - wgpu::Features, - wgpu::Limits, - ) = { - let device_result = adapter - .request_device(&wgpu::DeviceDescriptor { - label: Some("OpenNow Device"), - required_features, - required_limits: limits.clone(), - memory_hints: wgpu::MemoryHints::MemoryUsage, - experimental_features: wgpu::ExperimentalFeatures::disabled(), - trace: wgpu::Trace::Off, - }) - .await; - - match device_result { - Ok((device, queue)) => { - info!("Device created successfully with {}", adapter_info.name); - ( - device, - queue, - adapter, - is_v3d_hardware, - required_features, - limits, - ) - } - Err(e) => { - // V3D device creation failed (likely OOM), fallback to software renderer - warn!("Hardware GPU device creation failed: {:?}", e); - warn!("Falling back to software renderer (llvmpipe)..."); - - crate::utils::console_print( - "[GPU] Hardware GPU failed, using software renderer", - ); - - // Get software (llvmpipe) adapter - let sw_adapter = instance - .request_adapter(&wgpu::RequestAdapterOptions { - power_preference: wgpu::PowerPreference::LowPower, - compatible_surface: Some(&surface), - force_fallback_adapter: true, - }) - .await - .context("Failed to find software GPU adapter after hardware GPU failed")?; - - let sw_info = sw_adapter.get_info(); - info!( - "Fallback GPU: {} (Backend: {:?})", - sw_info.name, sw_info.backend - ); - - // Use downlevel defaults for llvmpipe - let sw_limits = - wgpu::Limits::downlevel_defaults().using_resolution(sw_adapter.limits()); - let sw_features = wgpu::Features::empty(); - - let (device, queue) = sw_adapter - .request_device(&wgpu::DeviceDescriptor { - label: Some("OpenNow Device (Software Fallback)"), - required_features: sw_features, - required_limits: sw_limits.clone(), - memory_hints: wgpu::MemoryHints::MemoryUsage, - experimental_features: wgpu::ExperimentalFeatures::disabled(), - trace: wgpu::Trace::Off, - }) - .await - .context("Failed to create software GPU device")?; - - info!("Software renderer device created successfully"); - (device, queue, sw_adapter, false, sw_features, sw_limits) - } - } - }; - - // Non-ARM64: Standard device creation - #[cfg(not(all(target_os = "linux", target_arch = "aarch64")))] - let (device, queue): (wgpu::Device, wgpu::Queue) = adapter - .request_device(&wgpu::DeviceDescriptor { - label: Some("OpenNow Device"), - required_features, - required_limits: limits, - // Use MemoryUsage hint to avoid aggressive memory allocation which causes OOM on RPi5 - memory_hints: wgpu::MemoryHints::MemoryUsage, - experimental_features: wgpu::ExperimentalFeatures::disabled(), - trace: wgpu::Trace::Off, - }) - .await - .context("Failed to create device")?; - - // Update adapter_info after potential ARM64 fallback to software renderer - #[cfg(all(target_os = "linux", target_arch = "aarch64"))] - let adapter_info = adapter.get_info(); - - // Configure surface - // Use non-sRGB (linear) format for video - H.264/HEVC output is already gamma-corrected - // Using sRGB format would apply double gamma correction, causing washed-out colors - let surface_caps = surface.get_capabilities(&adapter); - let surface_format = surface_caps - .formats - .iter() - .find(|f| !f.is_srgb()) // Prefer linear format for video - .copied() - .unwrap_or(surface_caps.formats[0]); - - // Start with Fifo (VSync) for low CPU usage in menus - // Switches to Immediate when streaming for lowest latency - let present_mode = wgpu::PresentMode::Fifo; - info!("Using Fifo present mode (vsync) - low CPU usage for UI"); - - // Frame latency: 2 for smoother pacing - let frame_latency = 2; - - let config = wgpu::SurfaceConfiguration { - usage: wgpu::TextureUsages::RENDER_ATTACHMENT, - format: surface_format, - width: size.width, - height: size.height, - present_mode, - alpha_mode: if surface_caps - .alpha_modes - .contains(&wgpu::CompositeAlphaMode::PostMultiplied) - { - wgpu::CompositeAlphaMode::PostMultiplied - } else if surface_caps - .alpha_modes - .contains(&wgpu::CompositeAlphaMode::PreMultiplied) - { - wgpu::CompositeAlphaMode::PreMultiplied - } else { - surface_caps.alpha_modes[0] - }, - view_formats: vec![], - desired_maximum_frame_latency: frame_latency, - }; - - surface.configure(&device, &config); - - // Create egui context - let egui_ctx = egui::Context::default(); - - // Create egui-winit state (egui 0.33 API) - let egui_state = egui_winit::State::new( - egui_ctx.clone(), - egui::ViewportId::default(), - &window, - Some(window.scale_factor() as f32), - None, - None, - ); - - // Create egui-wgpu renderer (egui 0.33 API) - let egui_renderer = egui_wgpu::Renderer::new( - &device, - surface_format, - egui_wgpu::RendererOptions::default(), - ); - - // Create video rendering pipeline - let video_shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { - label: Some("Video Shader"), - source: wgpu::ShaderSource::Wgsl(VIDEO_SHADER.into()), - }); - - // Bind group layout for YUV planar textures (GPU color conversion) - let video_bind_group_layout = - device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { - label: Some("Video YUV Bind Group Layout"), - entries: &[ - // Y texture (full resolution) - wgpu::BindGroupLayoutEntry { - binding: 0, - visibility: wgpu::ShaderStages::FRAGMENT, - ty: wgpu::BindingType::Texture { - sample_type: wgpu::TextureSampleType::Float { filterable: true }, - view_dimension: wgpu::TextureViewDimension::D2, - multisampled: false, - }, - count: None, - }, - // U texture (half resolution) - wgpu::BindGroupLayoutEntry { - binding: 1, - visibility: wgpu::ShaderStages::FRAGMENT, - ty: wgpu::BindingType::Texture { - sample_type: wgpu::TextureSampleType::Float { filterable: true }, - view_dimension: wgpu::TextureViewDimension::D2, - multisampled: false, - }, - count: None, - }, - // V texture (half resolution) - wgpu::BindGroupLayoutEntry { - binding: 2, - visibility: wgpu::ShaderStages::FRAGMENT, - ty: wgpu::BindingType::Texture { - sample_type: wgpu::TextureSampleType::Float { filterable: true }, - view_dimension: wgpu::TextureViewDimension::D2, - multisampled: false, - }, - count: None, - }, - // Sampler - wgpu::BindGroupLayoutEntry { - binding: 3, - visibility: wgpu::ShaderStages::FRAGMENT, - ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), - count: None, - }, - ], - }); - - let video_pipeline_layout = - device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { - label: Some("Video Pipeline Layout"), - bind_group_layouts: &[&video_bind_group_layout], - immediate_size: 0, - }); - - let video_pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { - label: Some("Video Pipeline"), - layout: Some(&video_pipeline_layout), - vertex: wgpu::VertexState { - module: &video_shader, - entry_point: Some("vs_main"), - buffers: &[], - compilation_options: Default::default(), - }, - fragment: Some(wgpu::FragmentState { - module: &video_shader, - entry_point: Some("fs_main"), - targets: &[Some(wgpu::ColorTargetState { - format: surface_format, - blend: Some(wgpu::BlendState::REPLACE), - write_mask: wgpu::ColorWrites::ALL, - })], - compilation_options: Default::default(), - }), - primitive: wgpu::PrimitiveState { - topology: wgpu::PrimitiveTopology::TriangleList, - strip_index_format: None, - front_face: wgpu::FrontFace::Ccw, - cull_mode: None, - polygon_mode: wgpu::PolygonMode::Fill, - unclipped_depth: false, - conservative: false, - }, - depth_stencil: None, - multisample: wgpu::MultisampleState::default(), - multiview_mask: None, - cache: None, - }); - - let video_sampler = device.create_sampler(&wgpu::SamplerDescriptor { - label: Some("Video Sampler"), - address_mode_u: wgpu::AddressMode::ClampToEdge, - address_mode_v: wgpu::AddressMode::ClampToEdge, - address_mode_w: wgpu::AddressMode::ClampToEdge, - mag_filter: wgpu::FilterMode::Linear, - min_filter: wgpu::FilterMode::Linear, - mipmap_filter: wgpu::MipmapFilterMode::Nearest, - ..Default::default() - }); - - // Create NV12 pipeline (for VideoToolbox on macOS - GPU deinterleaving) - let nv12_shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { - label: Some("NV12 Shader"), - source: wgpu::ShaderSource::Wgsl(NV12_SHADER.into()), - }); - - let nv12_bind_group_layout = - device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { - label: Some("NV12 Bind Group Layout"), - entries: &[ - // Y texture (full resolution, R8) - wgpu::BindGroupLayoutEntry { - binding: 0, - visibility: wgpu::ShaderStages::FRAGMENT, - ty: wgpu::BindingType::Texture { - sample_type: wgpu::TextureSampleType::Float { filterable: true }, - view_dimension: wgpu::TextureViewDimension::D2, - multisampled: false, - }, - count: None, - }, - // UV texture (half resolution, Rg8 interleaved) - wgpu::BindGroupLayoutEntry { - binding: 1, - visibility: wgpu::ShaderStages::FRAGMENT, - ty: wgpu::BindingType::Texture { - sample_type: wgpu::TextureSampleType::Float { filterable: true }, - view_dimension: wgpu::TextureViewDimension::D2, - multisampled: false, - }, - count: None, - }, - // Sampler - wgpu::BindGroupLayoutEntry { - binding: 2, - visibility: wgpu::ShaderStages::FRAGMENT, - ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), - count: None, - }, - ], - }); - - let nv12_pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { - label: Some("NV12 Pipeline Layout"), - bind_group_layouts: &[&nv12_bind_group_layout], - immediate_size: 0, - }); - - let nv12_pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { - label: Some("NV12 Pipeline"), - layout: Some(&nv12_pipeline_layout), - vertex: wgpu::VertexState { - module: &nv12_shader, - entry_point: Some("vs_main"), - buffers: &[], - compilation_options: Default::default(), - }, - fragment: Some(wgpu::FragmentState { - module: &nv12_shader, - entry_point: Some("fs_main"), - targets: &[Some(wgpu::ColorTargetState { - format: surface_format, - blend: Some(wgpu::BlendState::REPLACE), - write_mask: wgpu::ColorWrites::ALL, - })], - compilation_options: Default::default(), - }), - primitive: wgpu::PrimitiveState { - topology: wgpu::PrimitiveTopology::TriangleList, - strip_index_format: None, - front_face: wgpu::FrontFace::Ccw, - cull_mode: None, - polygon_mode: wgpu::PolygonMode::Fill, - unclipped_depth: false, - conservative: false, - }, - depth_stencil: None, - multisample: wgpu::MultisampleState::default(), - multiview_mask: None, - cache: None, - }); - - // Create NV12 HDR tone mapping pipeline (for HDR content on SDR displays) - let nv12_hdr_shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { - label: Some("NV12 HDR Tonemap Shader"), - source: wgpu::ShaderSource::Wgsl(NV12_HDR_TONEMAP_SHADER.into()), - }); - - // HDR pipeline uses the same bind group layout as NV12 - let nv12_hdr_pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { - label: Some("NV12 HDR Tonemap Pipeline"), - layout: Some(&nv12_pipeline_layout), - vertex: wgpu::VertexState { - module: &nv12_hdr_shader, - entry_point: Some("vs_main"), - buffers: &[], - compilation_options: Default::default(), - }, - fragment: Some(wgpu::FragmentState { - module: &nv12_hdr_shader, - entry_point: Some("fs_main"), - targets: &[Some(wgpu::ColorTargetState { - format: surface_format, - blend: Some(wgpu::BlendState::REPLACE), - write_mask: wgpu::ColorWrites::ALL, - })], - compilation_options: Default::default(), - }), - primitive: wgpu::PrimitiveState { - topology: wgpu::PrimitiveTopology::TriangleList, - strip_index_format: None, - front_face: wgpu::FrontFace::Ccw, - cull_mode: None, - polygon_mode: wgpu::PolygonMode::Fill, - unclipped_depth: false, - conservative: false, - }, - depth_stencil: None, - multisample: wgpu::MultisampleState::default(), - multiview_mask: None, - cache: None, - }); - - info!("NV12 HDR tone mapping pipeline created"); - - // Create External Texture pipeline (true zero-copy hardware YUV->RGB) - let (external_texture_pipeline, external_texture_bind_group_layout) = - if external_texture_supported { - let external_texture_shader = - device.create_shader_module(wgpu::ShaderModuleDescriptor { - label: Some("External Texture Shader"), - source: wgpu::ShaderSource::Wgsl(EXTERNAL_TEXTURE_SHADER.into()), - }); - - let external_texture_bind_group_layout = - device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { - label: Some("External Texture Bind Group Layout"), - entries: &[ - // External texture (hardware YUV->RGB conversion) - wgpu::BindGroupLayoutEntry { - binding: 0, - visibility: wgpu::ShaderStages::FRAGMENT, - ty: wgpu::BindingType::ExternalTexture, - count: None, - }, - // Sampler for external texture - wgpu::BindGroupLayoutEntry { - binding: 1, - visibility: wgpu::ShaderStages::FRAGMENT, - ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), - count: None, - }, - ], - }); - - let external_texture_pipeline_layout = - device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { - label: Some("External Texture Pipeline Layout"), - bind_group_layouts: &[&external_texture_bind_group_layout], - immediate_size: 0, - }); - - let external_texture_pipeline = - device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { - label: Some("External Texture Pipeline"), - layout: Some(&external_texture_pipeline_layout), - vertex: wgpu::VertexState { - module: &external_texture_shader, - entry_point: Some("vs_main"), - buffers: &[], - compilation_options: Default::default(), - }, - fragment: Some(wgpu::FragmentState { - module: &external_texture_shader, - entry_point: Some("fs_main"), - targets: &[Some(wgpu::ColorTargetState { - format: surface_format, - blend: Some(wgpu::BlendState::REPLACE), - write_mask: wgpu::ColorWrites::ALL, - })], - compilation_options: Default::default(), - }), - primitive: wgpu::PrimitiveState { - topology: wgpu::PrimitiveTopology::TriangleList, - strip_index_format: None, - front_face: wgpu::FrontFace::Ccw, - cull_mode: None, - polygon_mode: wgpu::PolygonMode::Fill, - unclipped_depth: false, - conservative: false, - }, - depth_stencil: None, - multisample: wgpu::MultisampleState::default(), - multiview_mask: None, - cache: None, - }); - - info!("External Texture pipeline created - true zero-copy video rendering enabled"); - ( - Some(external_texture_pipeline), - Some(external_texture_bind_group_layout), - ) - } else { - (None, None) - }; - - // Create stats panel - let stats_panel = StatsPanel::new(); - - Ok(Self { - window, - surface, - device, - queue, - config, - size, - egui_ctx, - egui_state, - egui_renderer, - video_pipeline, - video_bind_group_layout, - video_sampler, - y_texture: None, - u_texture: None, - v_texture: None, - video_bind_group: None, - video_size: (0, 0), - nv12_pipeline, - nv12_bind_group_layout, - nv12_hdr_pipeline, - uv_texture: None, - nv12_bind_group: None, - current_format: PixelFormat::YUV420P, - current_transfer_function: TransferFunction::SDR, - shared_frame: None, - external_texture_pipeline, - external_texture_bind_group_layout, - external_texture_bind_group: None, - external_texture: None, - external_texture_supported, - stats_panel, - fullscreen: false, - consecutive_surface_errors: 0, - supported_present_modes: surface_caps.present_modes.clone(), - game_textures: HashMap::new(), - // UI optimization: stats throttling (200ms intervals) - cached_stats: None, - stats_last_update: Instant::now(), - // UI optimization: game grid caching - games_cache_hash: 0, - // Track last uploaded frame to avoid redundant GPU uploads - last_uploaded_frame_id: 0, - // Resolution change notification - resolution_notification: None, - last_resolution: String::new(), - // Racing wheel connection notification - wheel_notification: None, - last_wheel_count: 0, - #[cfg(target_os = "macos")] - zero_copy_manager: ZeroCopyTextureManager::new(), - #[cfg(target_os = "macos")] - zero_copy_enabled: true, // GPU blit via Metal for zero-copy CVPixelBuffer rendering - #[cfg(target_os = "macos")] - current_y_cv_texture: None, - #[cfg(target_os = "macos")] - current_uv_cv_texture: None, - #[cfg(target_os = "windows")] - current_imported_handle: None, - #[cfg(target_os = "windows")] - current_imported_texture: None, - }) - } - - /// Get window reference - pub fn window(&self) -> &Window { - &self.window - } - - /// Handle window event - returns (consumed, repaint) - pub fn handle_event(&mut self, event: &WindowEvent) -> egui_winit::EventResponse { - self.egui_state.on_window_event(&self.window, event) - } - - /// Resize the renderer - /// Filters out spurious resize events that occur during fullscreen transitions - pub fn resize(&mut self, new_size: PhysicalSize) { - if new_size.width == 0 || new_size.height == 0 { - return; - } - - // If we're in fullscreen mode, STRICTLY enforce that the resize matches the monitor - // This prevents the race condition where the old windowed size (e.g., 1296x759) - // is briefly reported during the fullscreen transition, causing DWM composition. - if self.fullscreen { - if let Some(monitor) = self.window.current_monitor() { - let monitor_size = monitor.size(); - - // Calculate deviation from monitor size (must be within 5%) - let width_ratio = new_size.width as f32 / monitor_size.width as f32; - let height_ratio = new_size.height as f32 / monitor_size.height as f32; - - // Reject if not within 95-105% of monitor resolution - if width_ratio < 0.95 - || width_ratio > 1.05 - || height_ratio < 0.95 - || height_ratio > 1.05 - { - debug!( - "Ignoring resize to {}x{} while in fullscreen (monitor: {}x{}, ratio: {:.2}x{:.2})", - new_size.width, new_size.height, - monitor_size.width, monitor_size.height, - width_ratio, height_ratio - ); - return; - } - } - } - - self.size = new_size; - self.configure_surface(); - } - - /// Configure the surface with current size and optimal present mode - /// Called on resize and to recover from swapchain errors - fn configure_surface(&mut self) { - self.config.width = self.size.width; - self.config.height = self.size.height; - self.surface.configure(&self.device, &self.config); - info!( - "Surface configured: {}x{} @ {:?} (frame latency: {})", - self.config.width, - self.config.height, - self.config.present_mode, - self.config.desired_maximum_frame_latency - ); - - // On macOS, set ProMotion frame rate and disable VSync on every configure - // This ensures the Metal layer always requests 120fps from ProMotion - #[cfg(target_os = "macos")] - Self::disable_macos_vsync(&self.window); - } - - /// Set VSync mode - use Fifo (vsync) for UI, Immediate/Mailbox for streaming - /// This lets the GPU handle frame pacing, reducing CPU usage to near zero when idle - pub fn set_vsync(&mut self, enabled: bool) { - let new_mode = if enabled { - wgpu::PresentMode::Fifo // VSync on - GPU waits for display refresh - } else { - // VSync off - prefer Immediate for lowest latency, fall back to Mailbox - if self - .supported_present_modes - .contains(&wgpu::PresentMode::Immediate) - { - wgpu::PresentMode::Immediate - } else if self - .supported_present_modes - .contains(&wgpu::PresentMode::Mailbox) - { - wgpu::PresentMode::Mailbox // Good low-latency alternative - } else { - wgpu::PresentMode::Fifo // Fallback to VSync if nothing else available - } - }; - - if self.config.present_mode != new_mode { - self.config.present_mode = new_mode; - self.surface.configure(&self.device, &self.config); - info!("Present mode changed to {:?}", new_mode); - } - } - - /// Recover from swapchain errors (Outdated/Lost) - /// Returns true if recovery was successful - fn recover_swapchain(&mut self) -> bool { - // Get current window size - it may have changed (e.g., fullscreen toggle) - let current_size = self.window.inner_size(); - if current_size.width == 0 || current_size.height == 0 { - warn!("Cannot recover swapchain: window size is zero"); - return false; - } - - // Update size and reconfigure - self.size = current_size; - self.configure_surface(); - info!( - "Swapchain recovered: {}x{} @ {:?}", - self.size.width, self.size.height, self.config.present_mode - ); - true - } - - /// Toggle fullscreen with high refresh rate support - /// Uses exclusive fullscreen to bypass the desktop compositor (DWM) for lowest latency - /// and selects the highest available refresh rate for the current resolution - pub fn toggle_fullscreen(&mut self) { - self.fullscreen = !self.fullscreen; - - if self.fullscreen { - // On macOS, use Core Graphics to force 120Hz display mode - #[cfg(target_os = "macos")] - Self::set_macos_display_mode_120hz(); - - // Use borderless fullscreen on macOS (exclusive doesn't work well) - // The display mode is set separately via Core Graphics - #[cfg(target_os = "macos")] - { - info!("Entering borderless fullscreen with 120Hz display mode"); - self.window - .set_fullscreen(Some(Fullscreen::Borderless(None))); - Self::disable_macos_vsync(&self.window); - return; - } - - // On other platforms, try exclusive fullscreen - #[cfg(not(target_os = "macos"))] - { - // Wayland doesn't support exclusive fullscreen - use borderless instead - #[cfg(target_os = "linux")] - let is_wayland = std::env::var("WAYLAND_DISPLAY").is_ok(); - #[cfg(not(target_os = "linux"))] - let is_wayland = false; - - if is_wayland { - info!( - "Wayland detected - using borderless fullscreen (exclusive not supported)" - ); - self.window - .set_fullscreen(Some(Fullscreen::Borderless(None))); - return; - } - - let current_monitor = self.window.current_monitor(); - - if let Some(monitor) = current_monitor { - let current_size = self.window.inner_size(); - let mut best_mode: Option = None; - let mut best_refresh_rate: u32 = 0; - - info!("Searching for video modes on monitor: {:?}", monitor.name()); - info!( - "Current window size: {}x{}", - current_size.width, current_size.height - ); - - let mut mode_count = 0; - let mut high_refresh_modes = Vec::new(); - for mode in monitor.video_modes() { - let mode_size = mode.size(); - let refresh_rate = mode.refresh_rate_millihertz() / 1000; - - if refresh_rate >= 100 { - high_refresh_modes.push(format!( - "{}x{}@{}Hz", - mode_size.width, mode_size.height, refresh_rate - )); - } - mode_count += 1; - - if mode_size.width >= current_size.width - && mode_size.height >= current_size.height - { - if refresh_rate > best_refresh_rate { - best_refresh_rate = refresh_rate; - best_mode = Some(mode); - } - } - } - info!( - "Total video modes: {} (high refresh >=100Hz: {:?})", - mode_count, high_refresh_modes - ); - - if let Some(mode) = best_mode { - let refresh_hz = mode.refresh_rate_millihertz() / 1000; - info!( - "SELECTED exclusive fullscreen: {}x{} @ {}Hz", - mode.size().width, - mode.size().height, - refresh_hz - ); - self.window - .set_fullscreen(Some(Fullscreen::Exclusive(mode))); - return; - } else { - info!("No suitable exclusive fullscreen mode found"); - } - } else { - info!("No current monitor detected"); - } - - info!("Entering borderless fullscreen"); - self.window - .set_fullscreen(Some(Fullscreen::Borderless(None))); - } - } else { - info!("Exiting fullscreen"); - self.window.set_fullscreen(None); - } - } - - /// Enter fullscreen with a specific target refresh rate - /// Useful when the stream FPS is known (e.g., 120fps stream -> 120Hz mode) - pub fn set_fullscreen_with_refresh(&mut self, target_fps: u32) { - // Wayland doesn't support exclusive fullscreen - use borderless instead - #[cfg(target_os = "linux")] - let is_wayland = std::env::var("WAYLAND_DISPLAY").is_ok(); - #[cfg(not(target_os = "linux"))] - let is_wayland = false; - - if is_wayland { - info!( - "Wayland detected - using borderless fullscreen for {}fps stream", - target_fps - ); - self.fullscreen = true; - self.window - .set_fullscreen(Some(Fullscreen::Borderless(None))); - return; - } - - let current_monitor = self.window.current_monitor(); - - if let Some(monitor) = current_monitor { - let current_size = self.window.inner_size(); - let mut best_mode: Option = None; - let mut best_refresh_diff: i32 = i32::MAX; - - // Find mode closest to target FPS - for mode in monitor.video_modes() { - let mode_size = mode.size(); - let refresh_rate = mode.refresh_rate_millihertz() / 1000; - - if mode_size.width >= current_size.width && mode_size.height >= current_size.height - { - let diff = (refresh_rate as i32 - target_fps as i32).abs(); - // Prefer modes >= target FPS - let adjusted_diff = if refresh_rate >= target_fps { - diff - } else { - diff + 1000 - }; - - if adjusted_diff < best_refresh_diff { - best_refresh_diff = adjusted_diff; - best_mode = Some(mode); - } - } - } - - if let Some(mode) = best_mode { - let refresh_hz = mode.refresh_rate_millihertz() / 1000; - info!( - "Entering exclusive fullscreen for {}fps stream: {}x{} @ {}Hz", - target_fps, - mode.size().width, - mode.size().height, - refresh_hz - ); - self.fullscreen = true; - self.window - .set_fullscreen(Some(Fullscreen::Exclusive(mode))); - - #[cfg(target_os = "macos")] - Self::disable_macos_vsync(&self.window); - - return; - } - } - - // Fallback - self.fullscreen = true; - self.window - .set_fullscreen(Some(Fullscreen::Borderless(None))); - - #[cfg(target_os = "macos")] - Self::disable_macos_vsync(&self.window); - } - - /// Disable VSync on macOS Metal layer for unlimited FPS - /// This prevents the compositor from limiting frame rate - #[cfg(target_os = "macos")] - fn disable_macos_vsync(window: &Window) { - use cocoa::base::id; - use objc::{msg_send, sel, sel_impl}; - - // Get NSView from raw window handle - let ns_view = match window.window_handle() { - Ok(handle) => match handle.as_raw() { - RawWindowHandle::AppKit(appkit) => appkit.ns_view.as_ptr() as id, - _ => { - warn!("macOS: Unexpected window handle type"); - return; - } - }, - Err(e) => { - warn!("macOS: Could not get window handle: {:?}", e); - return; - } - }; - - unsafe { - // Get the layer from NSView - let layer: id = msg_send![ns_view, layer]; - if layer.is_null() { - warn!("macOS: Could not get layer for VSync disable"); - return; - } - - // Check if it's a CAMetalLayer by checking class name - let class: id = msg_send![layer, class]; - let class_name: id = msg_send![class, description]; - let name_cstr: *const i8 = msg_send![class_name, UTF8String]; - - if !name_cstr.is_null() { - let name = std::ffi::CStr::from_ptr(name_cstr).to_string_lossy(); - if name.contains("CAMetalLayer") { - // Set preferredFrameRateRange for ProMotion displays FIRST - // This tells macOS we want 120fps, preventing dynamic drop to 60Hz - #[repr(C)] - struct CAFrameRateRange { - minimum: f32, - maximum: f32, - preferred: f32, - } - - let frame_rate_range = CAFrameRateRange { - minimum: 60.0, // Allow 60fps minimum for flexibility - maximum: 120.0, - preferred: 120.0, - }; - - // Check if the layer responds to setPreferredFrameRateRange: (macOS 12+) - let responds: bool = - msg_send![layer, respondsToSelector: sel!(setPreferredFrameRateRange:)]; - if responds { - let _: () = msg_send![layer, setPreferredFrameRateRange: frame_rate_range]; - info!("macOS: Set preferredFrameRateRange to 60-120fps (ProMotion)"); - } - - // Enable displaySync for smooth presentation (no tearing) - // Latency is handled by decoder flags, not here - let _: () = msg_send![layer, setDisplaySyncEnabled: true]; - info!("macOS: Enabled displaySync on CAMetalLayer for tear-free rendering"); - } - } - } - } - - /// Set macOS display to 120Hz using Core Graphics - /// This bypasses winit's video mode selection which doesn't work well on macOS - #[cfg(target_os = "macos")] - fn set_macos_display_mode_120hz() { - use std::ffi::c_void; - - // Core Graphics FFI - #[link(name = "CoreGraphics", kind = "framework")] - extern "C" { - fn CGMainDisplayID() -> u32; - fn CGDisplayCopyAllDisplayModes(display: u32, options: *const c_void) -> *const c_void; - fn CFArrayGetCount(array: *const c_void) -> isize; - fn CFArrayGetValueAtIndex(array: *const c_void, idx: isize) -> *const c_void; - fn CGDisplayModeGetWidth(mode: *const c_void) -> usize; - fn CGDisplayModeGetHeight(mode: *const c_void) -> usize; - fn CGDisplayModeGetRefreshRate(mode: *const c_void) -> f64; - fn CGDisplaySetDisplayMode( - display: u32, - mode: *const c_void, - options: *const c_void, - ) -> i32; - fn CGDisplayPixelsWide(display: u32) -> usize; - fn CGDisplayPixelsHigh(display: u32) -> usize; - fn CFRelease(cf: *const c_void); - } - - unsafe { - let display_id = CGMainDisplayID(); - let current_width = CGDisplayPixelsWide(display_id); - let current_height = CGDisplayPixelsHigh(display_id); - - info!( - "macOS: Searching for 120Hz mode on display {} (current: {}x{})", - display_id, current_width, current_height - ); - - let modes = CGDisplayCopyAllDisplayModes(display_id, std::ptr::null()); - if modes.is_null() { - warn!("macOS: Could not enumerate display modes"); - return; - } - - let count = CFArrayGetCount(modes); - let mut best_mode: *const c_void = std::ptr::null(); - let mut best_refresh: f64 = 0.0; - - for i in 0..count { - let mode = CFArrayGetValueAtIndex(modes, i); - let width = CGDisplayModeGetWidth(mode); - let height = CGDisplayModeGetHeight(mode); - let refresh = CGDisplayModeGetRefreshRate(mode); - - // Look for modes matching current resolution with high refresh rate - if width == current_width && height == current_height { - if refresh > best_refresh { - best_refresh = refresh; - best_mode = mode; - } - if refresh >= 100.0 { - info!(" Found mode: {}x{} @ {:.1}Hz", width, height, refresh); - } - } - } - - if !best_mode.is_null() && best_refresh >= 119.0 { - let width = CGDisplayModeGetWidth(best_mode); - let height = CGDisplayModeGetHeight(best_mode); - info!( - "macOS: Setting display mode to {}x{} @ {:.1}Hz", - width, height, best_refresh - ); - - let result = CGDisplaySetDisplayMode(display_id, best_mode, std::ptr::null()); - if result == 0 { - info!("macOS: Successfully set 120Hz display mode!"); - } else { - warn!("macOS: Failed to set display mode, error: {}", result); - } - } else if best_refresh > 0.0 { - info!( - "macOS: No 120Hz mode found, best is {:.1}Hz - display may not support it", - best_refresh - ); - } else { - warn!("macOS: No matching display modes found"); - } - - CFRelease(modes); - } - } - - /// Enable high-performance mode on macOS - /// This disables App Nap and other power throttling that can limit FPS - #[cfg(target_os = "macos")] - fn enable_macos_high_performance() { - use cocoa::base::{id, nil}; - use objc::{class, msg_send, sel, sel_impl}; - - unsafe { - // Get NSProcessInfo - let process_info: id = msg_send![class!(NSProcessInfo), processInfo]; - if process_info == nil { - warn!("macOS: Could not get NSProcessInfo"); - return; - } - - // Activity options for high performance: - // NSActivityUserInitiated = 0x00FFFFFF (prevents App Nap, system sleep) - // NSActivityLatencyCritical = 0xFF00000000 (requests low latency scheduling) - let options: u64 = 0x00FFFFFF | 0xFF00000000; - - // Create reason string - let reason: id = msg_send![class!(NSString), stringWithUTF8String: b"Streaming requires consistent frame timing\0".as_ptr()]; - - // Begin activity - this returns an object we should retain - let activity: id = - msg_send![process_info, beginActivityWithOptions:options reason:reason]; - if activity != nil { - // Retain the activity object to keep it alive for the app lifetime - let _: id = msg_send![activity, retain]; - info!("macOS: High-performance mode enabled (App Nap disabled, latency-critical scheduling)"); - } else { - warn!("macOS: Failed to enable high-performance mode"); - } - - // Also try to disable automatic termination - let _: () = msg_send![process_info, disableAutomaticTermination: reason]; - - // Disable sudden termination - let _: () = msg_send![process_info, disableSuddenTermination]; - } - } - - /// Lock cursor for streaming (captures mouse) - pub fn lock_cursor(&self) { - // Try confined first, then locked mode - if let Err(e) = self.window.set_cursor_grab(CursorGrabMode::Confined) { - info!("Confined cursor grab failed ({}), trying locked mode", e); - if let Err(e) = self.window.set_cursor_grab(CursorGrabMode::Locked) { - log::warn!("Failed to lock cursor: {}", e); - } - } - self.window.set_cursor_visible(false); - info!("Cursor locked for streaming"); - } - - /// Unlock cursor - pub fn unlock_cursor(&self) { - let _ = self.window.set_cursor_grab(CursorGrabMode::None); - self.window.set_cursor_visible(true); - info!("Cursor unlocked"); - } - - /// Check if fullscreen - pub fn is_fullscreen(&self) -> bool { - self.fullscreen - } - - /// Set the shared frame buffer for direct frame access - /// This allows the renderer to pull frames directly from the decoder - pub fn set_shared_frame(&mut self, shared_frame: Arc) { - self.shared_frame = Some(shared_frame); - } - - /// Show racing wheel connection notification - /// Called when racing wheels are detected during streaming session - pub fn show_wheel_notification(&mut self, wheel_count: usize) { - if wheel_count > 0 && wheel_count != self.last_wheel_count { - info!( - "Racing wheel notification: {} wheel(s) detected", - wheel_count - ); - self.wheel_notification = Some(WheelNotification::new(wheel_count)); - self.last_wheel_count = wheel_count; - } - } - - /// Reset wheel notification state (call when streaming stops) - pub fn reset_wheel_notification(&mut self) { - self.wheel_notification = None; - self.last_wheel_count = 0; - } - - /// Update video textures from frame (GPU YUV->RGB conversion) - /// Supports both YUV420P (3 planes) and NV12 (2 planes) formats - /// On macOS, uses zero-copy path via CVPixelBuffer + Metal blit - /// On Windows, uses D3D11 shared textures - pub fn update_video(&mut self, frame: &VideoFrame) { - let uv_width = frame.width / 2; - let uv_height = frame.height / 2; - - // ZERO-COPY PATH: CVPixelBuffer + Metal blit (macOS VideoToolbox) - #[cfg(target_os = "macos")] - if let Some(ref gpu_frame) = frame.gpu_frame { - self.update_video_zero_copy(frame, gpu_frame, uv_width, uv_height); - return; - } - - // ZERO-COPY PATH: D3D11 texture sharing (Windows D3D11VA) - // TODO: Implement true GPU sharing via D3D11/DX12 interop with wgpu - // For now this still uses CPU staging - needs wgpu external memory support - #[cfg(target_os = "windows")] - if let Some(ref gpu_frame) = frame.gpu_frame { - self.update_video_d3d11(frame, gpu_frame, uv_width, uv_height); - self.last_uploaded_frame_id = frame.frame_id; - return; - } - - // ZERO-COPY PATH: VAAPI DMA-BUF import (Linux) - // Imports the DMA-BUF from VAAPI decoder into Vulkan via VK_EXT_external_memory_dma_buf - #[cfg(target_os = "linux")] - if let Some(ref gpu_frame) = frame.gpu_frame { - self.update_video_vaapi(frame, gpu_frame, uv_width, uv_height); - self.last_uploaded_frame_id = frame.frame_id; - return; - } - - // EXTERNAL TEXTURE PATH: Disabled for now - using NV12 shader path instead - // The external texture API on Windows DX12 may have issues with our frame lifecycle - // TODO: Re-enable once external texture path is debugged - // if self.external_texture_supported && frame.format == PixelFormat::NV12 && !frame.y_plane.is_empty() { - // self.update_video_external_texture(frame, uv_width, uv_height); - // return; - // } - - // Check if we need to recreate textures (size or format change) - let format_changed = self.current_format != frame.format; - let size_changed = self.video_size != (frame.width, frame.height); - - // Update transfer function (HDR detection) - can change during stream - if self.current_transfer_function != frame.transfer_function { - info!( - "Transfer function changed: {:?} -> {:?}", - self.current_transfer_function, frame.transfer_function - ); - self.current_transfer_function = frame.transfer_function; - } - - if size_changed || format_changed { - self.current_format = frame.format; - self.video_size = (frame.width, frame.height); - - // Y texture is same for both formats (full resolution, R8) - let y_texture = self.device.create_texture(&wgpu::TextureDescriptor { - label: Some("Y Texture"), - size: wgpu::Extent3d { - width: frame.width, - height: frame.height, - depth_or_array_layers: 1, - }, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: wgpu::TextureFormat::R8Unorm, - usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, - view_formats: &[], - }); - - match frame.format { - PixelFormat::NV12 => { - // NV12: UV plane is interleaved (Rg8, 2 bytes per pixel) - let uv_texture = self.device.create_texture(&wgpu::TextureDescriptor { - label: Some("UV Texture (NV12)"), - size: wgpu::Extent3d { - width: uv_width, - height: uv_height, - depth_or_array_layers: 1, - }, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: wgpu::TextureFormat::Rg8Unorm, // 2-channel for interleaved UV - usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, - view_formats: &[], - }); - - let y_view = y_texture.create_view(&wgpu::TextureViewDescriptor::default()); - let uv_view = uv_texture.create_view(&wgpu::TextureViewDescriptor::default()); - - let bind_group = self.device.create_bind_group(&wgpu::BindGroupDescriptor { - label: Some("NV12 Bind Group"), - layout: &self.nv12_bind_group_layout, - entries: &[ - wgpu::BindGroupEntry { - binding: 0, - resource: wgpu::BindingResource::TextureView(&y_view), - }, - wgpu::BindGroupEntry { - binding: 1, - resource: wgpu::BindingResource::TextureView(&uv_view), - }, - wgpu::BindGroupEntry { - binding: 2, - resource: wgpu::BindingResource::Sampler(&self.video_sampler), - }, - ], - }); - - self.y_texture = Some(y_texture); - self.uv_texture = Some(uv_texture); - self.nv12_bind_group = Some(bind_group); - // Clear YUV420P textures - self.u_texture = None; - self.v_texture = None; - self.video_bind_group = None; - - info!("NV12 textures created: {}x{} (UV: {}x{}) - GPU deinterleaving enabled (CPU scaler bypassed!)", - frame.width, frame.height, uv_width, uv_height); - } - PixelFormat::YUV420P => { - // YUV420P: Separate U and V planes (R8 each) - let u_texture = self.device.create_texture(&wgpu::TextureDescriptor { - label: Some("U Texture"), - size: wgpu::Extent3d { - width: uv_width, - height: uv_height, - depth_or_array_layers: 1, - }, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: wgpu::TextureFormat::R8Unorm, - usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, - view_formats: &[], - }); - - let v_texture = self.device.create_texture(&wgpu::TextureDescriptor { - label: Some("V Texture"), - size: wgpu::Extent3d { - width: uv_width, - height: uv_height, - depth_or_array_layers: 1, - }, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: wgpu::TextureFormat::R8Unorm, - usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, - view_formats: &[], - }); - - let y_view = y_texture.create_view(&wgpu::TextureViewDescriptor::default()); - let u_view = u_texture.create_view(&wgpu::TextureViewDescriptor::default()); - let v_view = v_texture.create_view(&wgpu::TextureViewDescriptor::default()); - - let bind_group = self.device.create_bind_group(&wgpu::BindGroupDescriptor { - label: Some("Video YUV Bind Group"), - layout: &self.video_bind_group_layout, - entries: &[ - wgpu::BindGroupEntry { - binding: 0, - resource: wgpu::BindingResource::TextureView(&y_view), - }, - wgpu::BindGroupEntry { - binding: 1, - resource: wgpu::BindingResource::TextureView(&u_view), - }, - wgpu::BindGroupEntry { - binding: 2, - resource: wgpu::BindingResource::TextureView(&v_view), - }, - wgpu::BindGroupEntry { - binding: 3, - resource: wgpu::BindingResource::Sampler(&self.video_sampler), - }, - ], - }); - - self.y_texture = Some(y_texture); - self.u_texture = Some(u_texture); - self.v_texture = Some(v_texture); - self.video_bind_group = Some(bind_group); - // Clear NV12 textures - self.uv_texture = None; - self.nv12_bind_group = None; - - info!("YUV420P textures created: {}x{} (UV: {}x{}) - GPU color conversion enabled", - frame.width, frame.height, uv_width, uv_height); - } - PixelFormat::P010 => { - // P010: 10-bit HDR with interleaved UV (similar to NV12 but 16-bit) - // For now, we treat it like NV12 - proper HDR support needs 16-bit textures - // TODO: Use Rg16Unorm for proper 10-bit support - let uv_texture = self.device.create_texture(&wgpu::TextureDescriptor { - label: Some("UV Texture (P010/HDR)"), - size: wgpu::Extent3d { - width: uv_width, - height: uv_height, - depth_or_array_layers: 1, - }, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: wgpu::TextureFormat::Rg8Unorm, // TODO: Rg16Unorm for true 10-bit - usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, - view_formats: &[], - }); - - let y_view = y_texture.create_view(&wgpu::TextureViewDescriptor::default()); - let uv_view = uv_texture.create_view(&wgpu::TextureViewDescriptor::default()); - - let bind_group = self.device.create_bind_group(&wgpu::BindGroupDescriptor { - label: Some("P010/HDR Bind Group"), - layout: &self.nv12_bind_group_layout, - entries: &[ - wgpu::BindGroupEntry { - binding: 0, - resource: wgpu::BindingResource::TextureView(&y_view), - }, - wgpu::BindGroupEntry { - binding: 1, - resource: wgpu::BindingResource::TextureView(&uv_view), - }, - wgpu::BindGroupEntry { - binding: 2, - resource: wgpu::BindingResource::Sampler(&self.video_sampler), - }, - ], - }); - - self.y_texture = Some(y_texture); - self.uv_texture = Some(uv_texture); - self.nv12_bind_group = Some(bind_group); - self.u_texture = None; - self.v_texture = None; - self.video_bind_group = None; - - info!( - "P010/HDR textures created: {}x{} (UV: {}x{}) - HDR mode (10-bit)", - frame.width, frame.height, uv_width, uv_height - ); - } - } - } - - // Upload Y plane (same for both formats) - if let Some(ref texture) = self.y_texture { - self.queue.write_texture( - wgpu::TexelCopyTextureInfo { - texture, - mip_level: 0, - origin: wgpu::Origin3d::ZERO, - aspect: wgpu::TextureAspect::All, - }, - &frame.y_plane, - wgpu::TexelCopyBufferLayout { - offset: 0, - bytes_per_row: Some(frame.y_stride), - rows_per_image: Some(frame.height), - }, - wgpu::Extent3d { - width: frame.width, - height: frame.height, - depth_or_array_layers: 1, - }, - ); - } - - match frame.format { - PixelFormat::NV12 => { - // Upload interleaved UV plane (Rg8) - if let Some(ref texture) = self.uv_texture { - self.queue.write_texture( - wgpu::TexelCopyTextureInfo { - texture, - mip_level: 0, - origin: wgpu::Origin3d::ZERO, - aspect: wgpu::TextureAspect::All, - }, - &frame.u_plane, // NV12: u_plane contains interleaved UV data - wgpu::TexelCopyBufferLayout { - offset: 0, - bytes_per_row: Some(frame.u_stride), // stride for interleaved UV - rows_per_image: Some(uv_height), - }, - wgpu::Extent3d { - width: uv_width, - height: uv_height, - depth_or_array_layers: 1, - }, - ); - } - } - PixelFormat::YUV420P => { - // Upload separate U and V planes - if let Some(ref texture) = self.u_texture { - self.queue.write_texture( - wgpu::TexelCopyTextureInfo { - texture, - mip_level: 0, - origin: wgpu::Origin3d::ZERO, - aspect: wgpu::TextureAspect::All, - }, - &frame.u_plane, - wgpu::TexelCopyBufferLayout { - offset: 0, - bytes_per_row: Some(frame.u_stride), - rows_per_image: Some(uv_height), - }, - wgpu::Extent3d { - width: uv_width, - height: uv_height, - depth_or_array_layers: 1, - }, - ); - } - - if let Some(ref texture) = self.v_texture { - self.queue.write_texture( - wgpu::TexelCopyTextureInfo { - texture, - mip_level: 0, - origin: wgpu::Origin3d::ZERO, - aspect: wgpu::TextureAspect::All, - }, - &frame.v_plane, - wgpu::TexelCopyBufferLayout { - offset: 0, - bytes_per_row: Some(frame.v_stride), - rows_per_image: Some(uv_height), - }, - wgpu::Extent3d { - width: uv_width, - height: uv_height, - depth_or_array_layers: 1, - }, - ); - } - } - PixelFormat::P010 => { - // P010: Similar to NV12 but with 10-bit data in 16-bit words - // For now, treat like NV12 (data truncated to 8-bit) - if let Some(ref texture) = self.uv_texture { - self.queue.write_texture( - wgpu::TexelCopyTextureInfo { - texture, - mip_level: 0, - origin: wgpu::Origin3d::ZERO, - aspect: wgpu::TextureAspect::All, - }, - &frame.u_plane, - wgpu::TexelCopyBufferLayout { - offset: 0, - bytes_per_row: Some(frame.u_stride), - rows_per_image: Some(uv_height), - }, - wgpu::Extent3d { - width: uv_width, - height: uv_height, - depth_or_array_layers: 1, - }, - ); - } - } - } - - // Mark this frame as uploaded to avoid redundant uploads - self.last_uploaded_frame_id = frame.frame_id; - } - - /// TRUE zero-copy video update using CVMetalTextureCache (macOS only) - /// Creates Metal textures that share GPU memory with CVPixelBuffer - NO CPU COPY! - /// Uses wgpu's hal layer to import Metal textures directly, avoiding all CPU involvement. - #[cfg(target_os = "macos")] - fn update_video_zero_copy( - &mut self, - frame: &VideoFrame, - gpu_frame: &std::sync::Arc, - uv_width: u32, - uv_height: u32, - ) { - use objc::runtime::Object; - use objc::{msg_send, sel, sel_impl}; - - // Use CVMetalTextureCache for true zero-copy (no CPU involvement) - if self.zero_copy_enabled { - if let Some(ref manager) = self.zero_copy_manager { - // Create Metal textures directly from CVPixelBuffer - TRUE ZERO-COPY! - // These textures share GPU memory with the decoded video frame - if let Some((y_metal, uv_metal)) = manager.create_textures_from_cv_buffer(gpu_frame) - { - // Check if we need to recreate wgpu textures (size changed) - let size_changed = self.video_size != (frame.width, frame.height); - - if size_changed { - self.current_format = frame.format; - self.video_size = (frame.width, frame.height); - - // Create wgpu textures that we'll blit into from Metal - let y_texture = self.device.create_texture(&wgpu::TextureDescriptor { - label: Some("Y Texture (Zero-Copy Target)"), - size: wgpu::Extent3d { - width: frame.width, - height: frame.height, - depth_or_array_layers: 1, - }, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: wgpu::TextureFormat::R8Unorm, - usage: wgpu::TextureUsages::TEXTURE_BINDING - | wgpu::TextureUsages::COPY_DST - | wgpu::TextureUsages::RENDER_ATTACHMENT, - view_formats: &[], - }); - - let uv_texture = self.device.create_texture(&wgpu::TextureDescriptor { - label: Some("UV Texture (Zero-Copy Target)"), - size: wgpu::Extent3d { - width: uv_width, - height: uv_height, - depth_or_array_layers: 1, - }, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: wgpu::TextureFormat::Rg8Unorm, - usage: wgpu::TextureUsages::TEXTURE_BINDING - | wgpu::TextureUsages::COPY_DST - | wgpu::TextureUsages::RENDER_ATTACHMENT, - view_formats: &[], - }); - - let y_view = y_texture.create_view(&wgpu::TextureViewDescriptor::default()); - let uv_view = - uv_texture.create_view(&wgpu::TextureViewDescriptor::default()); - - let bind_group = - self.device.create_bind_group(&wgpu::BindGroupDescriptor { - label: Some("NV12 Bind Group (Zero-Copy)"), - layout: &self.nv12_bind_group_layout, - entries: &[ - wgpu::BindGroupEntry { - binding: 0, - resource: wgpu::BindingResource::TextureView(&y_view), - }, - wgpu::BindGroupEntry { - binding: 1, - resource: wgpu::BindingResource::TextureView(&uv_view), - }, - wgpu::BindGroupEntry { - binding: 2, - resource: wgpu::BindingResource::Sampler( - &self.video_sampler, - ), - }, - ], - }); - - self.y_texture = Some(y_texture); - self.uv_texture = Some(uv_texture); - self.nv12_bind_group = Some(bind_group); - - log::info!( - "Zero-copy video textures created: {}x{} (UV: {}x{})", - frame.width, - frame.height, - uv_width, - uv_height - ); - } - - // GPU-to-GPU blit: Copy from CVMetalTexture to wgpu texture using Metal blit encoder - // This is entirely on GPU - no CPU involvement at all! - unsafe { - // Use the cached command queue from ZeroCopyTextureManager (created once, reused every frame) - let command_queue = manager.command_queue(); - - if !command_queue.is_null() { - let command_buffer: *mut Object = - msg_send![command_queue, commandBuffer]; - - if !command_buffer.is_null() { - let blit_encoder: *mut Object = - msg_send![command_buffer, blitCommandEncoder]; - - if !blit_encoder.is_null() { - // Get source Metal textures from CVMetalTexture - let y_src = y_metal.metal_texture_ptr(); - let uv_src = uv_metal.metal_texture_ptr(); - - // Get destination Metal textures from wgpu - // wgpu on Metal stores the underlying MTLTexture - if let (Some(ref y_dst_wgpu), Some(ref uv_dst_wgpu)) = - (&self.y_texture, &self.uv_texture) - { - // Use wgpu's hal API to get underlying Metal textures - let copied = self.blit_metal_textures( - blit_encoder, - y_src, - uv_src, - y_dst_wgpu, - uv_dst_wgpu, - frame.width, - frame.height, - uv_width, - uv_height, - ); - - if copied { - let _: () = msg_send![blit_encoder, endEncoding]; - let _: () = msg_send![command_buffer, commit]; - // NOTE: Not waiting for completion - GPU synchronization - // is handled by the fact that we're rendering immediately after - // and Metal will queue the operations correctly within the same frame - - // Store CVMetalTextures to keep them alive - self.current_y_cv_texture = Some(y_metal); - self.current_uv_cv_texture = Some(uv_metal); - - return; // Success! GPU-to-GPU copy complete - } - } - - let _: () = msg_send![blit_encoder, endEncoding]; - } - // Don't commit if blit failed - } - } - } - } - } - } - - // CPU fallback: Lock CVPixelBuffer and upload plane data to textures - // This is slower than zero-copy but works on legacy Macs (2015 and earlier) - // that don't support the Metal features required for zero-copy rendering - if let Some(locked) = gpu_frame.lock_and_get_planes() { - log::debug!("Using CPU fallback for video frame (legacy mode or zero-copy failed)"); - - // Ensure textures exist - let size_changed = self.video_size != (frame.width, frame.height); - if size_changed || self.y_texture.is_none() { - self.current_format = frame.format; - self.video_size = (frame.width, frame.height); - - let y_texture = self.device.create_texture(&wgpu::TextureDescriptor { - label: Some("Y Texture (CPU Fallback)"), - size: wgpu::Extent3d { - width: frame.width, - height: frame.height, - depth_or_array_layers: 1, - }, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: wgpu::TextureFormat::R8Unorm, - usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, - view_formats: &[], - }); - - let uv_texture = self.device.create_texture(&wgpu::TextureDescriptor { - label: Some("UV Texture (CPU Fallback)"), - size: wgpu::Extent3d { - width: uv_width, - height: uv_height, - depth_or_array_layers: 1, - }, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: wgpu::TextureFormat::Rg8Unorm, - usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, - view_formats: &[], - }); - - let y_view = y_texture.create_view(&wgpu::TextureViewDescriptor::default()); - let uv_view = uv_texture.create_view(&wgpu::TextureViewDescriptor::default()); - - let bind_group = self.device.create_bind_group(&wgpu::BindGroupDescriptor { - label: Some("NV12 Bind Group (CPU Fallback)"), - layout: &self.nv12_bind_group_layout, - entries: &[ - wgpu::BindGroupEntry { - binding: 0, - resource: wgpu::BindingResource::TextureView(&y_view), - }, - wgpu::BindGroupEntry { - binding: 1, - resource: wgpu::BindingResource::TextureView(&uv_view), - }, - wgpu::BindGroupEntry { - binding: 2, - resource: wgpu::BindingResource::Sampler(&self.video_sampler), - }, - ], - }); - - self.y_texture = Some(y_texture); - self.uv_texture = Some(uv_texture); - self.nv12_bind_group = Some(bind_group); - - log::info!( - "CPU fallback textures created: {}x{} (UV: {}x{})", - frame.width, - frame.height, - uv_width, - uv_height - ); - } - - // Upload Y plane data - if let Some(ref y_texture) = self.y_texture { - self.queue.write_texture( - wgpu::TexelCopyTextureInfo { - texture: y_texture, - mip_level: 0, - origin: wgpu::Origin3d::ZERO, - aspect: wgpu::TextureAspect::All, - }, - locked.y_data, - wgpu::TexelCopyBufferLayout { - offset: 0, - bytes_per_row: Some(locked.y_stride), - rows_per_image: Some(locked.y_height), - }, - wgpu::Extent3d { - width: frame.width, - height: frame.height, - depth_or_array_layers: 1, - }, - ); - } - - // Upload UV plane data - if let Some(ref uv_texture) = self.uv_texture { - self.queue.write_texture( - wgpu::TexelCopyTextureInfo { - texture: uv_texture, - mip_level: 0, - origin: wgpu::Origin3d::ZERO, - aspect: wgpu::TextureAspect::All, - }, - locked.uv_data, - wgpu::TexelCopyBufferLayout { - offset: 0, - bytes_per_row: Some(locked.uv_stride), - rows_per_image: Some(locked.uv_height), - }, - wgpu::Extent3d { - width: uv_width, - height: uv_height, - depth_or_array_layers: 1, - }, - ); - } - - return; // CPU fallback succeeded - } - - // If we get here, both zero-copy and CPU fallback failed - log::warn!( - "Both GPU blit and CPU fallback failed - frame dropped (zero_copy_enabled={}, manager={})", - self.zero_copy_enabled, - self.zero_copy_manager.is_some() - ); - } - - /// Update video textures from D3D11 hardware-decoded frame (Windows) - /// Copies from D3D11 staging texture to wgpu - faster than FFmpeg's av_hwframe_transfer_data - /// because we skip FFmpeg's intermediate copies and work directly with decoder output - #[cfg(target_os = "windows")] - fn update_video_d3d11( - &mut self, - frame: &VideoFrame, - gpu_frame: &std::sync::Arc, - uv_width: u32, - uv_height: u32, - ) { - log::info!( - "update_video_d3d11: {}x{}, array_index={}, is_texture_array={}", - frame.width, - frame.height, - gpu_frame.array_index(), - gpu_frame.is_texture_array() - ); - - // Skip zero-copy for texture arrays (array_index > 0 means it's part of an array) - // The zero-copy path doesn't properly handle texture array slices yet - // The CPU path correctly uses CopySubresourceRegion with the array_index - let is_texture_array = gpu_frame.array_index() > 0 || gpu_frame.is_texture_array(); - - if is_texture_array { - log::info!( - "D3D11: Using CPU path for texture array (array_index={})", - gpu_frame.array_index() - ); - } - - // Try zero-copy via Shared Handle first (only for non-array textures) - // This eliminates the CPU copy by importing the D3D11 texture directly into DX12 - if !is_texture_array { - if let Ok(handle) = gpu_frame.get_shared_handle() { - let mut handle_changed = false; - - // Check if we need to re-import (handle changed or texture missing) - let needs_import = match self.current_imported_handle { - Some(current) => current != handle, - None => true, - }; - - if needs_import || self.current_imported_texture.is_none() { - // Import the shared handle into DX12 - // We must use unsafe to access the raw DX12 device via wgpu-hal - let imported_texture = unsafe { - match self.device.as_hal::() { - Some(hal_device) => { - let d3d12_device: &ID3D12Device = hal_device.raw_device(); - - // Open the shared handle as a D3D12 resource - let mut resource: Option = None; - if let Err(e) = d3d12_device.OpenSharedHandle(handle, &mut resource) - { - warn!("Failed to OpenSharedHandle: {:?}", e); - return; // Fallback to CPU copy - } - let resource = resource.unwrap(); - - // Wrap it in a wgpu::Texture - let size = wgpu::Extent3d { - width: frame.width, - height: frame.height, - depth_or_array_layers: 1, - }; - - let format = wgpu::TextureFormat::NV12; - let usage = wgpu::TextureUsages::TEXTURE_BINDING - | wgpu::TextureUsages::COPY_DST; - - // Create wgpu-hal texture from raw resource - let hal_texture = wgpu_hal::dx12::Device::texture_from_raw( - resource, - format, - wgpu::TextureDimension::D2, - size, - 1, // mip_levels - 1, // sample_count - ); - - // Create wgpu Texture from HAL texture - let descriptor = wgpu::TextureDescriptor { - label: Some("Imported D3D11 Texture"), - size, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format, - usage, - view_formats: &[], - }; - - Some(self.device.create_texture_from_hal::( - hal_texture, - &descriptor, - )) - } - None => { - warn!("Failed to get DX12 HAL device"); - None - } - } - }; - - if let Some(texture) = imported_texture { - self.current_imported_texture = Some(texture); - self.current_imported_handle = Some(handle); - handle_changed = true; - // Log success once per session or on change - debug!( - "Zero-copy: Imported D3D11 texture handle {:?} -> DX12", - handle - ); - } else { - // Import failed - clear cache and fall through to CPU path - self.current_imported_handle = None; - self.current_imported_texture = None; - } - } - - // If we have a valid imported texture, use it! - if let Some(ref texture) = self.current_imported_texture { - // If the handle changed OR if we don't have an external texture bind group yet (e.g. resize) - // we need to recreate the bind group. - // Note: video_size check handles resolution changes - let size_changed = self.video_size != (frame.width, frame.height); - - if handle_changed || size_changed || self.external_texture_bind_group.is_none() - { - self.video_size = (frame.width, frame.height); - self.current_format = PixelFormat::NV12; - - // Create views for Y and UV planes - let y_view = texture.create_view(&wgpu::TextureViewDescriptor { - label: Some("Plane 0 View"), - aspect: wgpu::TextureAspect::Plane0, - ..Default::default() - }); - - let uv_view = texture.create_view(&wgpu::TextureViewDescriptor { - label: Some("Plane 1 View"), - aspect: wgpu::TextureAspect::Plane1, - ..Default::default() - }); - - // Create ExternalTexture with color space aware conversion - // Select YUV to RGB conversion matrix based on color space - let yuv_conversion_matrix: [f32; 16] = match frame.color_space { - ColorSpace::BT709 => [ - 1.0, 1.0, 1.0, 0.0, 0.0, -0.1873, 1.8556, 0.0, 1.5748, -0.4681, - 0.0, 0.0, -0.7874, 0.3277, -0.9278, 1.0, - ], - ColorSpace::BT601 => [ - 1.0, 1.0, 1.0, 0.0, 0.0, -0.344, 1.772, 0.0, 1.402, -0.714, 0.0, - 0.0, -0.701, 0.529, -0.886, 1.0, - ], - ColorSpace::BT2020 => [ - 1.0, 1.0, 1.0, 0.0, 0.0, -0.1646, 1.8814, 0.0, 1.4746, -0.5714, - 0.0, 0.0, -0.7373, 0.3680, -0.9407, 1.0, - ], - }; - - let gamut_conversion_matrix: [f32; 9] = match frame.color_space { - ColorSpace::BT2020 => [ - 1.6605, -0.5876, -0.0728, -0.1246, 1.1329, -0.0083, -0.0182, - -0.1006, 1.1187, - ], - _ => [1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0], - }; - - // HDR transfer function handling (same as CPU path) - // Based on NVIDIA GFN client TrueHDR analysis - let (src_transfer, dst_transfer) = match frame.transfer_function { - TransferFunction::PQ => { - // PQ (SMPTE ST 2084) HDR content - // Use moderate gamma to decode PQ and compress dynamic range - let pq_decode = wgpu::ExternalTextureTransferFunction { - a: 1.0, - b: 0.0, - g: 1.8, // Moderate gamma for PQ decode - k: 0.0, - }; - let sdr_encode = wgpu::ExternalTextureTransferFunction { - a: 1.0, - b: 0.0, - g: 0.55, // Re-encode to SDR gamma - k: 0.0, - }; - (pq_decode, sdr_encode) - } - TransferFunction::HLG => { - // HLG is backwards-compatible with SDR displays - let hlg_decode = wgpu::ExternalTextureTransferFunction { - a: 1.0, - b: 0.0, - g: 1.2, - k: 0.0, - }; - let sdr_encode = wgpu::ExternalTextureTransferFunction { - a: 1.0, - b: 0.0, - g: 0.85, - k: 0.0, - }; - (hlg_decode, sdr_encode) - } - TransferFunction::SDR => { - // SDR: identity transfer - let identity = wgpu::ExternalTextureTransferFunction { - a: 1.0, - b: 0.0, - g: 1.0, - k: 1.0, - }; - (identity.clone(), identity) - } - }; - - let identity_transform: [f32; 6] = [1.0, 0.0, 0.0, 0.0, 1.0, 0.0]; - - let external_texture = self.device.create_external_texture( - &wgpu::ExternalTextureDescriptor { - label: Some("Zero-Copy External Texture"), - width: frame.width, - height: frame.height, - format: wgpu::ExternalTextureFormat::Nv12, - yuv_conversion_matrix, - gamut_conversion_matrix, - src_transfer_function: src_transfer, - dst_transfer_function: dst_transfer, - sample_transform: identity_transform, - load_transform: identity_transform, - }, - &[&y_view, &uv_view], - ); - - if let Some(ref layout) = self.external_texture_bind_group_layout { - let bind_group = - self.device.create_bind_group(&wgpu::BindGroupDescriptor { - label: Some("Zero-Copy Bind Group"), - layout, - entries: &[ - wgpu::BindGroupEntry { - binding: 0, - resource: wgpu::BindingResource::ExternalTexture( - &external_texture, - ), - }, - wgpu::BindGroupEntry { - binding: 1, - resource: wgpu::BindingResource::Sampler( - &self.video_sampler, - ), - }, - ], - }); - - self.external_texture_bind_group = Some(bind_group); - self.external_texture = Some(external_texture); - log::info!( - "Zero-copy pipeline configured for {}x{}", - frame.width, - frame.height - ); - } - } - - // Success! We are set up for zero-copy rendering. - return; - } - } - } // end if !is_texture_array - - // Fallback: Lock the D3D11 texture and get plane data (CPU Copy) - log::info!("D3D11: Locking texture for CPU copy..."); - let planes = match gpu_frame.lock_and_get_planes() { - Ok(p) => { - log::info!( - "D3D11: Got planes - y_size={}, uv_size={}, stride={}", - p.y_plane.len(), - p.uv_plane.len(), - p.y_stride - ); - - // Debug: Check Y plane data content - if !p.y_plane.is_empty() { - // Sample first few rows to check if data is valid or all zeros/gray - let sample_size = std::cmp::min(256, p.y_plane.len()); - let sample = &p.y_plane[..sample_size]; - let min_y = sample.iter().min().copied().unwrap_or(0); - let max_y = sample.iter().max().copied().unwrap_or(0); - let avg_y: u32 = - sample.iter().map(|&x| x as u32).sum::() / sample.len() as u32; - log::info!( - "D3D11: Y plane stats (first {} bytes): min={}, max={}, avg={}", - sample_size, - min_y, - max_y, - avg_y - ); - - // Also sample middle of the frame - let mid_offset = p.y_plane.len() / 2; - if mid_offset + 256 <= p.y_plane.len() { - let mid_sample = &p.y_plane[mid_offset..mid_offset + 256]; - let mid_min = mid_sample.iter().min().copied().unwrap_or(0); - let mid_max = mid_sample.iter().max().copied().unwrap_or(0); - let mid_avg: u32 = mid_sample.iter().map(|&x| x as u32).sum::() / 256; - log::info!( - "D3D11: Y plane middle stats: min={}, max={}, avg={}", - mid_min, - mid_max, - mid_avg - ); - } - } - - p - } - Err(e) => { - log::warn!("Failed to lock D3D11 texture: {:?}", e); - return; - } - }; - - // Check if we need to recreate textures (size change) - let size_changed = self.video_size != (frame.width, frame.height); - log::info!( - "D3D11: size_changed={}, video_size={:?}, frame_size={}x{}", - size_changed, - self.video_size, - frame.width, - frame.height - ); - - if size_changed { - self.video_size = (frame.width, frame.height); - self.current_format = PixelFormat::NV12; - - // Create Y texture (full resolution, R8) - let y_texture = self.device.create_texture(&wgpu::TextureDescriptor { - label: Some("Y Texture (D3D11)"), - size: wgpu::Extent3d { - width: frame.width, - height: frame.height, - depth_or_array_layers: 1, - }, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: wgpu::TextureFormat::R8Unorm, - usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, - view_formats: &[], - }); - - // Create UV texture for NV12 (Rg8 interleaved) - let uv_texture = self.device.create_texture(&wgpu::TextureDescriptor { - label: Some("UV Texture (D3D11)"), - size: wgpu::Extent3d { - width: uv_width, - height: uv_height, - depth_or_array_layers: 1, - }, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: wgpu::TextureFormat::Rg8Unorm, - usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, - view_formats: &[], - }); - - let y_view = y_texture.create_view(&wgpu::TextureViewDescriptor::default()); - let uv_view = uv_texture.create_view(&wgpu::TextureViewDescriptor::default()); - - let bind_group = self.device.create_bind_group(&wgpu::BindGroupDescriptor { - label: Some("NV12 Bind Group (D3D11)"), - layout: &self.nv12_bind_group_layout, - entries: &[ - wgpu::BindGroupEntry { - binding: 0, - resource: wgpu::BindingResource::TextureView(&y_view), - }, - wgpu::BindGroupEntry { - binding: 1, - resource: wgpu::BindingResource::TextureView(&uv_view), - }, - wgpu::BindGroupEntry { - binding: 2, - resource: wgpu::BindingResource::Sampler(&self.video_sampler), - }, - ], - }); - - self.y_texture = Some(y_texture); - self.uv_texture = Some(uv_texture); - self.nv12_bind_group = Some(bind_group); - // Clear YUV420P textures - self.u_texture = None; - self.v_texture = None; - self.video_bind_group = None; - - log::info!( - "D3D11 video textures created: {}x{} (UV: {}x{})", - frame.width, - frame.height, - uv_width, - uv_height - ); - } - - // Upload Y plane from D3D11 staging texture - if let Some(ref texture) = self.y_texture { - self.queue.write_texture( - wgpu::TexelCopyTextureInfo { - texture, - mip_level: 0, - origin: wgpu::Origin3d::ZERO, - aspect: wgpu::TextureAspect::All, - }, - &planes.y_plane, - wgpu::TexelCopyBufferLayout { - offset: 0, - bytes_per_row: Some(planes.y_stride), - rows_per_image: Some(planes.height), - }, - wgpu::Extent3d { - width: frame.width, - height: frame.height, - depth_or_array_layers: 1, - }, - ); - } - - // Upload UV plane from D3D11 staging texture - if let Some(ref texture) = self.uv_texture { - self.queue.write_texture( - wgpu::TexelCopyTextureInfo { - texture, - mip_level: 0, - origin: wgpu::Origin3d::ZERO, - aspect: wgpu::TextureAspect::All, - }, - &planes.uv_plane, - wgpu::TexelCopyBufferLayout { - offset: 0, - bytes_per_row: Some(planes.uv_stride), - rows_per_image: Some(uv_height), - }, - wgpu::Extent3d { - width: uv_width, - height: uv_height, - depth_or_array_layers: 1, - }, - ); - } - } - - /// Update video from VAAPI surface via DMA-BUF (Linux) - /// - /// This provides zero-copy rendering by importing the DMA-BUF from VAAPI - /// directly into Vulkan via VK_EXT_external_memory_dma_buf. - /// - /// Flow: - /// 1. VAAPI decoder outputs VASurface in GPU VRAM - /// 2. We export VASurface as DMA-BUF fd via vaExportSurfaceHandle - /// 3. Import DMA-BUF into Vulkan via VK_EXT_external_memory_dma_buf - /// 4. Bind to wgpu texture for rendering - /// - /// Fallback: If DMA-BUF import fails, we mmap and copy to CPU (still faster - /// than FFmpeg's sw_transfer since we avoid the intermediate copy). - #[cfg(target_os = "linux")] - fn update_video_vaapi( - &mut self, - frame: &VideoFrame, - gpu_frame: &std::sync::Arc, - uv_width: u32, - uv_height: u32, - ) { - // TODO: Implement true zero-copy via VK_EXT_external_memory_dma_buf - // This requires: - // 1. Check for VK_EXT_external_memory_dma_buf extension - // 2. Export DMA-BUF fd from VAAPI surface - // 3. Create VkImage with VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT - // 4. Import into wgpu via hal layer - // - // For now, use the fallback path: mmap the DMA-BUF and upload to GPU - // This is still faster than FFmpeg's sw_transfer because: - // - We skip FFmpeg's intermediate buffer allocation - // - We read directly from the GPU-accessible DMA-BUF - // - The DMA-BUF may be in CPU-cached memory for faster reads - - // Try to get plane data from the VAAPI surface - let planes = match gpu_frame.lock_and_get_planes() { - Ok(p) => p, - Err(e) => { - log::warn!("Failed to lock VAAPI surface: {:?}", e); - return; - } - }; - - // Check if we need to recreate textures (size change) - let size_changed = self.video_size != (frame.width, frame.height); - - if size_changed { - self.video_size = (frame.width, frame.height); - self.current_format = PixelFormat::NV12; - - // Create Y texture (full resolution, R8) - let y_texture = self.device.create_texture(&wgpu::TextureDescriptor { - label: Some("Y Texture (VAAPI)"), - size: wgpu::Extent3d { - width: frame.width, - height: frame.height, - depth_or_array_layers: 1, - }, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: wgpu::TextureFormat::R8Unorm, - usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, - view_formats: &[], - }); - - // Create UV texture for NV12 (Rg8 interleaved) - let uv_texture = self.device.create_texture(&wgpu::TextureDescriptor { - label: Some("UV Texture (VAAPI)"), - size: wgpu::Extent3d { - width: uv_width, - height: uv_height, - depth_or_array_layers: 1, - }, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: wgpu::TextureFormat::Rg8Unorm, - usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, - view_formats: &[], - }); - - let y_view = y_texture.create_view(&wgpu::TextureViewDescriptor::default()); - let uv_view = uv_texture.create_view(&wgpu::TextureViewDescriptor::default()); - - let bind_group = self.device.create_bind_group(&wgpu::BindGroupDescriptor { - label: Some("NV12 Bind Group (VAAPI)"), - layout: &self.nv12_bind_group_layout, - entries: &[ - wgpu::BindGroupEntry { - binding: 0, - resource: wgpu::BindingResource::TextureView(&y_view), - }, - wgpu::BindGroupEntry { - binding: 1, - resource: wgpu::BindingResource::TextureView(&uv_view), - }, - wgpu::BindGroupEntry { - binding: 2, - resource: wgpu::BindingResource::Sampler(&self.video_sampler), - }, - ], - }); - - self.y_texture = Some(y_texture); - self.uv_texture = Some(uv_texture); - self.nv12_bind_group = Some(bind_group); - - log::info!( - "VAAPI video textures created: {}x{} (UV: {}x{})", - frame.width, - frame.height, - uv_width, - uv_height - ); - } - - // Upload Y plane from VAAPI DMA-BUF - if let Some(ref texture) = self.y_texture { - self.queue.write_texture( - wgpu::TexelCopyTextureInfo { - texture, - mip_level: 0, - origin: wgpu::Origin3d::ZERO, - aspect: wgpu::TextureAspect::All, - }, - &planes.y_plane, - wgpu::TexelCopyBufferLayout { - offset: 0, - bytes_per_row: Some(planes.y_stride), - rows_per_image: Some(planes.height), - }, - wgpu::Extent3d { - width: frame.width, - height: frame.height, - depth_or_array_layers: 1, - }, - ); - } - - // Upload UV plane from VAAPI DMA-BUF - if let Some(ref texture) = self.uv_texture { - self.queue.write_texture( - wgpu::TexelCopyTextureInfo { - texture, - mip_level: 0, - origin: wgpu::Origin3d::ZERO, - aspect: wgpu::TextureAspect::All, - }, - &planes.uv_plane, - wgpu::TexelCopyBufferLayout { - offset: 0, - bytes_per_row: Some(planes.uv_stride), - rows_per_image: Some(uv_height), - }, - wgpu::Extent3d { - width: uv_width, - height: uv_height, - depth_or_array_layers: 1, - }, - ); - } - } - - /// Update video using ExternalTexture for hardware YUV->RGB conversion - /// This uses wgpu's ExternalTexture API which provides hardware-accelerated - /// color space conversion on supported platforms (DX12, Metal, Vulkan) - fn update_video_external_texture(&mut self, frame: &VideoFrame, uv_width: u32, uv_height: u32) { - // Check if we need to recreate textures (size change) - let size_changed = self.video_size != (frame.width, frame.height); - - if size_changed { - self.video_size = (frame.width, frame.height); - self.current_format = PixelFormat::NV12; - - // Create Y texture (full resolution, R8) - let y_texture = self.device.create_texture(&wgpu::TextureDescriptor { - label: Some("Y Texture (External)"), - size: wgpu::Extent3d { - width: frame.width, - height: frame.height, - depth_or_array_layers: 1, - }, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: wgpu::TextureFormat::R8Unorm, - usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, - view_formats: &[], - }); - - // Create UV texture for NV12 (Rg8 interleaved) - let uv_texture = self.device.create_texture(&wgpu::TextureDescriptor { - label: Some("UV Texture (External)"), - size: wgpu::Extent3d { - width: uv_width, - height: uv_height, - depth_or_array_layers: 1, - }, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: wgpu::TextureFormat::Rg8Unorm, - usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, - view_formats: &[], - }); - - self.y_texture = Some(y_texture); - self.uv_texture = Some(uv_texture); - - log::info!( - "External Texture video created: {}x{} (hardware YUV->RGB)", - frame.width, - frame.height - ); - } - - // Upload Y plane - if let Some(ref texture) = self.y_texture { - self.queue.write_texture( - wgpu::TexelCopyTextureInfo { - texture, - mip_level: 0, - origin: wgpu::Origin3d::ZERO, - aspect: wgpu::TextureAspect::All, - }, - &frame.y_plane, - wgpu::TexelCopyBufferLayout { - offset: 0, - bytes_per_row: Some(frame.y_stride), - rows_per_image: Some(frame.height), - }, - wgpu::Extent3d { - width: frame.width, - height: frame.height, - depth_or_array_layers: 1, - }, - ); - } - - // Upload UV plane - if let Some(ref texture) = self.uv_texture { - self.queue.write_texture( - wgpu::TexelCopyTextureInfo { - texture, - mip_level: 0, - origin: wgpu::Origin3d::ZERO, - aspect: wgpu::TextureAspect::All, - }, - &frame.u_plane, // u_plane contains interleaved UV for NV12 - wgpu::TexelCopyBufferLayout { - offset: 0, - bytes_per_row: Some(frame.u_stride), - rows_per_image: Some(uv_height), - }, - wgpu::Extent3d { - width: uv_width, - height: uv_height, - depth_or_array_layers: 1, - }, - ); - } - - // Create texture views for ExternalTexture - let y_view = self - .y_texture - .as_ref() - .unwrap() - .create_view(&wgpu::TextureViewDescriptor::default()); - let uv_view = self - .uv_texture - .as_ref() - .unwrap() - .create_view(&wgpu::TextureViewDescriptor::default()); - - // Select YUV to RGB conversion matrix based on color space - // All matrices are for Full Range (PC levels: Y 0-255, UV 0-255) - // With UV offset of -0.5 baked into the matrix offsets - let yuv_conversion_matrix: [f32; 16] = match frame.color_space { - ColorSpace::BT709 => [ - // BT.709 Full Range: R = Y + 1.5748*V, G = Y - 0.1873*U - 0.4681*V, B = Y + 1.8556*U - 1.0, 1.0, 1.0, 0.0, // Column 0: Y coefficients - 0.0, -0.1873, 1.8556, 0.0, // Column 1: U coefficients - 1.5748, -0.4681, 0.0, 0.0, // Column 2: V coefficients - -0.7874, 0.3277, -0.9278, 1.0, // Column 3: Offsets - ], - ColorSpace::BT601 => [ - // BT.601 Full Range: R = Y + 1.402*V, G = Y - 0.344*U - 0.714*V, B = Y + 1.772*U - 1.0, 1.0, 1.0, 0.0, // Column 0: Y coefficients - 0.0, -0.344, 1.772, 0.0, // Column 1: U coefficients - 1.402, -0.714, 0.0, 0.0, // Column 2: V coefficients - -0.701, 0.529, -0.886, 1.0, // Column 3: Offsets - ], - ColorSpace::BT2020 => [ - // BT.2020 Full Range (NCL): R = Y + 1.4746*V, G = Y - 0.1646*U - 0.5714*V, B = Y + 1.8814*U - 1.0, 1.0, 1.0, 0.0, // Column 0: Y coefficients - 0.0, -0.1646, 1.8814, 0.0, // Column 1: U coefficients - 1.4746, -0.5714, 0.0, 0.0, // Column 2: V coefficients - -0.7373, 0.3680, -0.9407, 1.0, // Column 3: Offsets - ], - }; - - // For HDR (BT.2020), convert gamut from BT.2020 to sRGB/BT.709 primaries - let gamut_conversion_matrix: [f32; 9] = match frame.color_space { - ColorSpace::BT2020 => [ - // BT.2020 to BT.709 gamut conversion (row-major for wgpu) - 1.6605, -0.5876, -0.0728, -0.1246, 1.1329, -0.0083, -0.0182, -0.1006, 1.1187, - ], - _ => [ - // Identity (no gamut conversion for BT.709/BT.601) - 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, - ], - }; - - // Select transfer function based on HDR mode - // For SDR: identity (video is already gamma-corrected in BT.709) - // For HDR PQ: apply tone mapping to convert to SDR display range - // - // Based on NVIDIA GFN client analysis, they use proper TrueHDR processing - // with parameters like TrueHdrMiddleGrey, TrueHdrContrast, TrueHdrSaturation - // - // The wgpu ExternalTextureTransferFunction formula is: - // For E < k: L = a * E - // For E >= k: L = a * pow((E + b) / (1 + b), g) - // - // For PQ to SDR conversion, we need to: - // 1. Decode PQ to linear light (linearize) - // 2. Tone map from HDR range (0-10000 nits) to SDR range (0-100 nits) - // 3. Re-encode to sRGB gamma - let (src_transfer, dst_transfer) = match frame.transfer_function { - TransferFunction::PQ => { - // PQ (SMPTE ST 2084) HDR content - // The PQ EOTF is complex, but we can approximate with gamma - // PQ encoded values map roughly: 0.5 PQ ≈ 100 nits (SDR reference white) - // - // Simplified approach: Use a moderate gamma to decode PQ - // and compress the dynamic range while preserving detail - // A gamma of ~1.8 gives good results for tone mapping PQ to SDR - let pq_decode = wgpu::ExternalTextureTransferFunction { - a: 1.0, // Scale factor - b: 0.0, // Offset - g: 1.8, // Moderate gamma for PQ decode (was 2.4, too aggressive) - k: 0.0, // Threshold - }; - // Re-encode to SDR sRGB gamma - // sRGB uses gamma 2.2, but 0.45 (1/2.2) for encoding - let sdr_encode = wgpu::ExternalTextureTransferFunction { - a: 1.0, - b: 0.0, - g: 0.55, // Slightly stronger than 1/2.2 to brighten shadows - k: 0.0, - }; - (pq_decode, sdr_encode) - } - TransferFunction::HLG => { - // HLG (Hybrid Log-Gamma) is designed to be backwards-compatible - // with SDR displays, so less aggressive tone mapping is needed - let hlg_decode = wgpu::ExternalTextureTransferFunction { - a: 1.0, - b: 0.0, - g: 1.2, // Mild gamma adjustment for HLG - k: 0.0, - }; - // HLG is mostly compatible with gamma 2.4 displays - let sdr_encode = wgpu::ExternalTextureTransferFunction { - a: 1.0, - b: 0.0, - g: 0.85, // Slight adjustment for SDR display - k: 0.0, - }; - (hlg_decode, sdr_encode) - } - TransferFunction::SDR => { - // SDR: identity transfer (video is pre-gamma-corrected in BT.709) - let identity = wgpu::ExternalTextureTransferFunction { - a: 1.0, - b: 0.0, - g: 1.0, // Linear passthrough - k: 1.0, // k=1 makes everything use the linear path (a*E) - }; - (identity.clone(), identity) - } - }; - - // Identity transforms for texture coordinates - let identity_transform: [f32; 6] = [1.0, 0.0, 0.0, 0.0, 1.0, 0.0]; - - // Create ExternalTexture - let external_texture = self.device.create_external_texture( - &wgpu::ExternalTextureDescriptor { - label: Some("Video External Texture"), - width: frame.width, - height: frame.height, - format: wgpu::ExternalTextureFormat::Nv12, - yuv_conversion_matrix, - gamut_conversion_matrix, - src_transfer_function: src_transfer, - dst_transfer_function: dst_transfer, - sample_transform: identity_transform, - load_transform: identity_transform, - }, - &[&y_view, &uv_view], - ); - - // Create bind group with external texture and sampler - if let Some(ref layout) = self.external_texture_bind_group_layout { - let bind_group = self.device.create_bind_group(&wgpu::BindGroupDescriptor { - label: Some("External Texture Bind Group"), - layout, - entries: &[ - wgpu::BindGroupEntry { - binding: 0, - resource: wgpu::BindingResource::ExternalTexture(&external_texture), - }, - wgpu::BindGroupEntry { - binding: 1, - resource: wgpu::BindingResource::Sampler(&self.video_sampler), - }, - ], - }); - - self.external_texture_bind_group = Some(bind_group); - self.external_texture = Some(external_texture); - } - } - - /// Helper function to blit Metal textures using wgpu's hal layer - /// Returns true if the blit was successful - #[cfg(target_os = "macos")] - unsafe fn blit_metal_textures( - &self, - blit_encoder: *mut objc::runtime::Object, - y_src: *mut objc::runtime::Object, - uv_src: *mut objc::runtime::Object, - y_dst_wgpu: &wgpu::Texture, - uv_dst_wgpu: &wgpu::Texture, - y_width: u32, - y_height: u32, - uv_width: u32, - uv_height: u32, - ) -> bool { - use objc::{msg_send, sel, sel_impl}; - - // Define MTLOrigin and MTLSize structs for Metal API - #[repr(C)] - #[derive(Copy, Clone)] - struct MTLOrigin { - x: u64, - y: u64, - z: u64, - } - #[repr(C)] - #[derive(Copy, Clone)] - struct MTLSize { - width: u64, - height: u64, - depth: u64, - } - - let origin = MTLOrigin { x: 0, y: 0, z: 0 }; - - // wgpu 27 as_hal API: returns Option> - // IMPORTANT: as_hal holds a read lock - we must get one pointer and drop the result - // before getting the next, otherwise we get a recursive lock panic. - - // Get Y texture pointer and drop hal reference immediately - let y_dst: Option<*mut objc::runtime::Object> = { - let y_hal = y_dst_wgpu.as_hal::(); - y_hal.map(|y_hal_tex| { - let y_metal_tex = (*y_hal_tex).raw_handle(); - *(y_metal_tex as *const _ as *const *mut objc::runtime::Object) - }) - }; // y_hal dropped here, lock released - - // Get UV texture pointer (now safe - Y's lock is released) - let uv_dst: Option<*mut objc::runtime::Object> = { - let uv_hal = uv_dst_wgpu.as_hal::(); - uv_hal.map(|uv_hal_tex| { - let uv_metal_tex = (*uv_hal_tex).raw_handle(); - *(uv_metal_tex as *const _ as *const *mut objc::runtime::Object) - }) - }; // uv_hal dropped here - - if let (Some(y_dst), Some(uv_dst)) = (y_dst, uv_dst) { - // Blit Y texture (GPU-to-GPU copy) - let y_size = MTLSize { - width: y_width as u64, - height: y_height as u64, - depth: 1, - }; - let _: () = msg_send![blit_encoder, - copyFromTexture: y_src - sourceSlice: 0u64 - sourceLevel: 0u64 - sourceOrigin: origin - sourceSize: y_size - toTexture: y_dst as *mut objc::runtime::Object - destinationSlice: 0u64 - destinationLevel: 0u64 - destinationOrigin: origin - ]; - - // Blit UV texture (GPU-to-GPU copy) - let uv_size = MTLSize { - width: uv_width as u64, - height: uv_height as u64, - depth: 1, - }; - let uv_origin = MTLOrigin { x: 0, y: 0, z: 0 }; - let _: () = msg_send![blit_encoder, - copyFromTexture: uv_src - sourceSlice: 0u64 - sourceLevel: 0u64 - sourceOrigin: uv_origin - sourceSize: uv_size - toTexture: uv_dst as *mut objc::runtime::Object - destinationSlice: 0u64 - destinationLevel: 0u64 - destinationOrigin: uv_origin - ]; - - log::trace!( - "GPU blit: Y {}x{}, UV {}x{}", - y_width, - y_height, - uv_width, - uv_height - ); - return true; - } - - log::debug!("Could not get Metal textures from wgpu for GPU blit"); - false - } - - /// Render video frame to screen - /// Automatically selects the correct pipeline based on current pixel format - /// Priority: External Texture (true zero-copy) > NV12 > YUV420P - fn render_video(&self, encoder: &mut wgpu::CommandEncoder, view: &wgpu::TextureView) { - // Priority 1: Use External Texture pipeline if available (hardware YUV->RGB conversion) - // This is the true zero-copy path with automatic color space conversion - if let (Some(ref pipeline), Some(ref bind_group)) = ( - &self.external_texture_pipeline, - &self.external_texture_bind_group, - ) { - let mut render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { - label: Some("Video Pass (External Texture)"), - color_attachments: &[Some(wgpu::RenderPassColorAttachment { - view, - resolve_target: None, - ops: wgpu::Operations { - load: wgpu::LoadOp::Clear(wgpu::Color::BLACK), - store: wgpu::StoreOp::Store, - }, - depth_slice: None, - })], - depth_stencil_attachment: None, - ..Default::default() - }); - - render_pass.set_pipeline(pipeline); - render_pass.set_bind_group(0, bind_group, &[]); - render_pass.draw(0..6, 0..1); - return; - } - - // Priority 2: Fallback to format-specific pipelines (manual YUV->RGB in shader) - let (pipeline, bind_group) = match self.current_format { - PixelFormat::NV12 => { - if let Some(ref bg) = self.nv12_bind_group { - // Use HDR tone mapping pipeline for PQ content on SDR displays - let pipeline = if self.current_transfer_function == TransferFunction::PQ { - &self.nv12_hdr_pipeline - } else { - &self.nv12_pipeline - }; - (pipeline, bg) - } else { - return; // No bind group ready - } - } - PixelFormat::YUV420P => { - if let Some(ref bg) = self.video_bind_group { - (&self.video_pipeline, bg) - } else { - return; // No bind group ready - } - } - PixelFormat::P010 => { - // P010 is 10-bit HDR format - use HDR pipeline for PQ content - if let Some(ref bg) = self.nv12_bind_group { - let pipeline = if self.current_transfer_function == TransferFunction::PQ { - &self.nv12_hdr_pipeline - } else { - &self.nv12_pipeline - }; - (pipeline, bg) - } else { - return; // No bind group ready - } - } - }; - - let mut render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { - label: Some("Video Pass"), - color_attachments: &[Some(wgpu::RenderPassColorAttachment { - view, - resolve_target: None, - ops: wgpu::Operations { - load: wgpu::LoadOp::Clear(wgpu::Color::BLACK), - store: wgpu::StoreOp::Store, - }, - depth_slice: None, - })], - depth_stencil_attachment: None, - ..Default::default() - }); - - render_pass.set_pipeline(pipeline); - render_pass.set_bind_group(0, bind_group, &[]); - render_pass.draw(0..6, 0..1); // Draw 6 vertices (2 triangles = 1 quad) - } - - /// Render frame and return UI actions plus optional repaint delay - /// The Duration indicates when the next repaint should happen (for idle throttling) - pub fn render(&mut self, app: &App) -> Result<(Vec, Option)> { - profile_scope!("render"); - - // Get surface texture with SMART error recovery for swapchain issues - // Key insight: During fullscreen transitions, the window size updates AFTER - // the surface error occurs. If we immediately "recover" with the old size, - // we force DWM composition (scaling), causing 60Hz lock and input lag. - // Instead, we YIELD to the event loop to let the Resized event propagate. - let output = match self.surface.get_current_texture() { - Ok(texture) => { - // Success - reset error counter - self.consecutive_surface_errors = 0; - texture - } - Err(wgpu::SurfaceError::Outdated) | Err(wgpu::SurfaceError::Lost) => { - self.consecutive_surface_errors += 1; - - // Check if window size differs from our config (resize pending) - let current_window_size = self.window.inner_size(); - let config_matches_window = current_window_size.width == self.config.width - && current_window_size.height == self.config.height; - - if !config_matches_window { - // Window size changed - resize event should handle this - // Call resize directly to sync up - debug!( - "Swapchain outdated: window {}x{} != config {}x{} - resizing", - current_window_size.width, - current_window_size.height, - self.config.width, - self.config.height - ); - self.resize(current_window_size); - - // Retry after resize - match self.surface.get_current_texture() { - Ok(texture) => { - self.consecutive_surface_errors = 0; - info!( - "Swapchain recovered after resize to {}x{}", - current_window_size.width, current_window_size.height - ); - texture - } - Err(e) => { - debug!("Still failing after resize: {} - yielding", e); - return Ok((vec![], None)); - } - } - } else if self.consecutive_surface_errors < 10 { - // Sizes match but surface is outdated - likely a race condition - // YIELD to event loop to let Resized event arrive with correct size - debug!( - "Swapchain outdated (attempt {}/10): sizes match {}x{} - yielding to event loop", - self.consecutive_surface_errors, - self.config.width, self.config.height - ); - return Ok((vec![], None)); - } else { - // Persistent error (10+ frames) - force recovery as fallback - warn!( - "Swapchain persistently outdated ({} attempts) - forcing recovery", - self.consecutive_surface_errors - ); - if !self.recover_swapchain() { - return Ok((vec![], None)); - } - match self.surface.get_current_texture() { - Ok(texture) => { - self.consecutive_surface_errors = 0; - texture - } - Err(e) => { - warn!("Failed to get texture after forced recovery: {}", e); - return Ok((vec![], None)); - } - } - } - } - Err(wgpu::SurfaceError::Timeout) => { - // GPU is busy, skip this frame - debug!("Surface timeout - skipping frame"); - return Ok((vec![], None)); - } - Err(e) => { - // Fatal error (e.g., OutOfMemory) - return Err(anyhow::anyhow!("Surface error: {}", e)); - } - }; - - let view = output - .texture - .create_view(&wgpu::TextureViewDescriptor::default()); - - let mut encoder = self - .device - .create_command_encoder(&wgpu::CommandEncoderDescriptor { - label: Some("Render Encoder"), - }); - - // Update video texture if we have a frame - if let Some(ref frame) = app.current_frame { - profile_scope!("update_video"); - self.update_video(frame); - } - - // Render video or clear based on state - // Check for either YUV420P (video_bind_group) or NV12 (nv12_bind_group) - let has_video = self.video_bind_group.is_some() || self.nv12_bind_group.is_some(); - if app.state == AppState::Streaming && has_video { - profile_scope!("render_video"); - // Render video full-screen - self.render_video(&mut encoder, &view); - } else { - // Clear pass for non-streaming states - let _render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { - label: Some("Clear Pass"), - color_attachments: &[Some(wgpu::RenderPassColorAttachment { - view: &view, - resolve_target: None, - ops: wgpu::Operations { - load: wgpu::LoadOp::Clear(wgpu::Color { - r: 0.08, - g: 0.08, - b: 0.12, - a: 1.0, - }), - store: wgpu::StoreOp::Store, - }, - depth_slice: None, - })], - depth_stencil_attachment: None, - ..Default::default() - }); - } - - // Draw egui UI and collect actions - let raw_input = self.egui_state.take_egui_input(&self.window); - let mut actions: Vec = Vec::new(); - - // === UI Optimization: Throttle stats updates to 200ms === - // This dramatically reduces CPU usage from stats panel rendering - const STATS_UPDATE_INTERVAL: Duration = Duration::from_millis(200); - if self.stats_last_update.elapsed() >= STATS_UPDATE_INTERVAL { - self.cached_stats = Some(app.stats.clone()); - self.stats_last_update = Instant::now(); - - // Detect resolution changes and show notification - if !app.stats.resolution.is_empty() && app.stats.resolution != self.last_resolution { - if !self.last_resolution.is_empty() { - // Resolution changed - create notification - self.resolution_notification = Some(ResolutionNotification::new( - &self.last_resolution, - &app.stats.resolution, - )); - } - self.last_resolution = app.stats.resolution.clone(); - } - - // Detect racing wheel connection and show notification - if app.stats.wheel_count > 0 && app.stats.wheel_count != self.last_wheel_count { - self.show_wheel_notification(app.stats.wheel_count); - } else if app.stats.wheel_count == 0 && self.last_wheel_count > 0 { - // Wheels disconnected - reset state - self.last_wheel_count = 0; - } - } - - // Clean up expired notifications - if let Some(ref notif) = self.resolution_notification { - if notif.is_expired() { - self.resolution_notification = None; - } - } - if let Some(ref notif) = self.wheel_notification { - if notif.is_expired() { - self.wheel_notification = None; - } - } - - // Extract state needed for UI rendering - let app_state = app.state; - // Use cached stats for display (throttled to 200ms updates) - let stats = self - .cached_stats - .clone() - .unwrap_or_else(|| app.stats.clone()); - let show_stats = app.show_stats; - let status_message = app.status_message.clone(); - let error_message = app.error_message.clone(); - let selected_game = app.selected_game.clone(); - let stats_position = self.stats_panel.position; - let stats_visible = self.stats_panel.visible; - let show_settings = app.show_settings; - let settings = app.settings.clone(); - let login_providers = app.login_providers.clone(); - let selected_provider_index = app.selected_provider_index; - let is_loading = app.is_loading; - let login_url = app.login_url.clone(); - let show_welcome_popup = app.show_welcome_popup; - let mut search_query = app.search_query.clone(); - let runtime = app.runtime.clone(); - - // New state for tabs, subscription, library, popup - let current_tab = app.current_tab; - let subscription = app.subscription.clone(); - let selected_game_popup = app.selected_game_popup.clone(); - - // Server/region state - let servers = app.servers.clone(); - let selected_server_index = app.selected_server_index; - let auto_server_selection = app.auto_server_selection; - let ping_testing = app.ping_testing; - let show_settings_modal = app.show_settings_modal; - - // Resolution notification data (extracted for use in closure) - let resolution_notif = self.resolution_notification.as_ref().map(|n| { - ( - n.old_resolution.clone(), - n.new_resolution.clone(), - n.direction, - n.alpha(), - ) - }); - - // Wheel notification data (extracted for use in closure) - let wheel_notif = self - .wheel_notification - .as_ref() - .map(|n| (n.wheel_count, n.alpha())); - - // Queue times state - let mut queue_servers = app.queue_servers.clone(); - let queue_loading = app.queue_loading; - let queue_sort_mode = app.queue_sort_mode; - let queue_region_filter = app.queue_region_filter.clone(); - let show_server_selection = app.show_server_selection; - let selected_queue_server = app.selected_queue_server.clone(); - let pending_server_selection_game = app.pending_server_selection_game.clone(); - - // Ads state (free tier) - let ads_required = app.ads_required; - let ads_remaining_secs = app.ads_remaining_secs; - let ads_total_secs = app.ads_total_secs; - - // Get games based on current tab - // Optimization: Home tab uses game_sections, not games_list - avoid cloning games - let games_list: Vec<(usize, crate::app::GameInfo)> = match current_tab { - GamesTab::Home => { - // Home tab renders from game_sections, return empty to avoid clone - Vec::new() - } - GamesTab::AllGames | GamesTab::MyLibrary => { - // Only clone filtered games for tabs that need them - let query = app.search_query.to_lowercase(); - let source = if current_tab == GamesTab::MyLibrary { - &app.library_games - } else { - &app.games - }; - source - .iter() - .enumerate() - .filter(|(_, g)| query.is_empty() || g.title.to_lowercase().contains(&query)) - .map(|(i, g)| (i, g.clone())) - .collect() - } - GamesTab::QueueTimes => Vec::new(), - }; - - // Get game sections for Home tab - only clone if on Home tab - let game_sections = if current_tab == GamesTab::Home { - app.game_sections.clone() - } else { - Vec::new() - }; - - // Clone texture map for rendering (avoid borrow issues) - let game_textures = self.game_textures.clone(); - let mut new_textures: Vec<(String, egui::TextureHandle)> = Vec::new(); - - let full_output; - { - profile_scope!("egui_run"); - full_output = self.egui_ctx.run_ui(raw_input, |ctx| { - // Custom styling - let mut style = (*ctx.global_style()).clone(); - style.visuals.window_fill = egui::Color32::from_rgb(20, 20, 30); - style.visuals.panel_fill = egui::Color32::from_rgb(25, 25, 35); - style.visuals.widgets.noninteractive.bg_fill = egui::Color32::from_rgb(35, 35, 50); - style.visuals.widgets.inactive.bg_fill = egui::Color32::from_rgb(45, 45, 65); - style.visuals.widgets.hovered.bg_fill = egui::Color32::from_rgb(60, 60, 90); - style.visuals.widgets.active.bg_fill = egui::Color32::from_rgb(80, 180, 80); - style.visuals.selection.bg_fill = egui::Color32::from_rgb(60, 120, 60); - ctx.set_global_style(style); - - match app_state { - AppState::Login => { - // Reduce idle CPU usage on login screen - ctx.request_repaint_after(Duration::from_millis(100)); - - render_login_screen( - ctx, - &login_providers, - selected_provider_index, - &status_message, - is_loading, - login_url.as_deref(), - &mut actions, - ); - - // Show welcome popup for first-time users - if show_welcome_popup { - render_welcome_popup(ctx, &mut actions); - } - } - AppState::Games => { - // Update image cache for async loading - image_cache::update_cache(); - - // === UI Optimization: Reduce idle repaints === - // When in Games view with no user interaction, we only need to repaint - // occasionally to check for newly loaded images. This reduces CPU from - // 100% to ~5% when idle in the game library. - // Note: User interactions (mouse, keyboard) will trigger immediate repaints - // via the winit event system, so responsiveness is not affected. - ctx.request_repaint_after(Duration::from_millis(100)); - - self.render_games_screen( - ctx, - &games_list, - &game_sections, - &mut search_query, - &status_message, - show_settings, - &settings, - &runtime, - &game_textures, - &mut new_textures, - current_tab, - subscription.as_ref(), - selected_game_popup.as_ref(), - &servers, - selected_server_index, - auto_server_selection, - ping_testing, - show_settings_modal, - app.show_session_conflict, - app.show_av1_warning, - app.show_alliance_warning, - crate::auth::get_selected_provider() - .login_provider_display_name - .as_str(), - &app.active_sessions, - app.pending_game_launch.as_ref(), - &mut queue_servers, - queue_loading, - queue_sort_mode, - &queue_region_filter, - show_server_selection, - &selected_queue_server, - pending_server_selection_game.as_ref(), - &mut actions, - ); - } - AppState::Session => { - // Session screen shows loading spinner, update at 30fps for smooth animation - ctx.request_repaint_after(Duration::from_millis(33)); - - // Show ads screen if ads are required (free tier) - if ads_required { - render_ads_required_screen( - ctx, - &selected_game, - ads_remaining_secs, - ads_total_secs, - &mut actions, - ); - } else { - render_session_screen( - ctx, - &selected_game, - &status_message, - &error_message, - &mut actions, - ); - } - } - AppState::Streaming => { - // Render stats overlay - if show_stats && stats_visible { - render_stats_panel(ctx, &stats, stats_position); - } - - // Render resolution change notification - if let Some((old_res, new_res, direction, alpha)) = &resolution_notif { - render_resolution_notification( - ctx, old_res, new_res, *direction, *alpha, - ); - } - - // Render racing wheel connection notification - if let Some((wheel_count, alpha)) = wheel_notif { - render_wheel_notification(ctx, wheel_count, alpha); - } - - // Small overlay hint - egui::Area::new(egui::Id::new("stream_hint")) - .anchor(egui::Align2::CENTER_TOP, [0.0, 10.0]) - .interactable(false) - .show(ctx, |ui| { - ui.label( - egui::RichText::new( - "Ctrl+Shift+Q to stop • F3 stats • F11 fullscreen", - ) - .color(egui::Color32::from_rgba_unmultiplied( - 255, 255, 255, 100, - )) - .size(12.0), - ); - }); - } - } - }); - } // end profile_scope!("egui_run") - - // Check if search query changed - if search_query != app.search_query { - // If user starts typing a search and is on Home tab, switch to All Games tab - // so they can see the filtered results - if !search_query.is_empty() && current_tab == GamesTab::Home { - actions.push(UiAction::SwitchTab(GamesTab::AllGames)); - } - actions.push(UiAction::UpdateSearch(search_query)); - } - - // Apply newly loaded textures to the cache - for (url, texture) in new_textures { - self.game_textures.insert(url, texture); - } - - self.egui_state - .handle_platform_output(&self.window, full_output.platform_output); - - let clipped_primitives = self - .egui_ctx - .tessellate(full_output.shapes, full_output.pixels_per_point); - - // Update egui textures - for (id, image_delta) in &full_output.textures_delta.set { - self.egui_renderer - .update_texture(&self.device, &self.queue, *id, image_delta); - } - - // Render egui - let screen_descriptor = egui_wgpu::ScreenDescriptor { - size_in_pixels: [self.size.width, self.size.height], - pixels_per_point: self.window.scale_factor() as f32, - }; - - self.egui_renderer.update_buffers( - &self.device, - &self.queue, - &mut encoder, - &clipped_primitives, - &screen_descriptor, - ); - - { - let render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { - label: Some("Egui Pass"), - color_attachments: &[Some(wgpu::RenderPassColorAttachment { - view: &view, - resolve_target: None, - ops: wgpu::Operations { - load: wgpu::LoadOp::Load, - store: wgpu::StoreOp::Store, - }, - depth_slice: None, - })], - depth_stencil_attachment: None, - ..Default::default() - }); - - // forget_lifetime is safe here as render_pass is dropped before encoder.finish() - let mut render_pass = render_pass.forget_lifetime(); - self.egui_renderer - .render(&mut render_pass, &clipped_primitives, &screen_descriptor); - } - - // Free egui textures - for id in &full_output.textures_delta.free { - self.egui_renderer.free_texture(id); - } - - { - profile_scope!("gpu_submit"); - self.queue.submit(std::iter::once(encoder.finish())); - } - { - profile_scope!("present"); - output.present(); - } - - // Return repaint delay based on app state for idle throttling - // This is set by request_repaint_after() calls in the UI code - let repaint_delay = match app.state { - AppState::Login | AppState::Games => Some(Duration::from_millis(100)), - AppState::Session => Some(Duration::from_millis(33)), // 30fps for spinner - AppState::Streaming => None, // No delay when streaming - }; - - Ok((actions, repaint_delay)) - } - - // render_login_screen moved to screens/login.rs - - fn render_games_screen( - &self, - ctx: &egui::Context, - games: &[(usize, crate::app::GameInfo)], - game_sections: &[crate::app::GameSection], - search_query: &mut String, - _status_message: &str, - _show_settings: bool, - settings: &crate::app::Settings, - _runtime: &tokio::runtime::Handle, - game_textures: &HashMap, - new_textures: &mut Vec<(String, egui::TextureHandle)>, - current_tab: GamesTab, - subscription: Option<&crate::app::SubscriptionInfo>, - selected_game_popup: Option<&crate::app::GameInfo>, - servers: &[crate::app::ServerInfo], - selected_server_index: usize, - auto_server_selection: bool, - ping_testing: bool, - show_settings_modal: bool, - show_session_conflict: bool, - show_av1_warning: bool, - show_alliance_warning: bool, - alliance_provider_name: &str, - active_sessions: &[ActiveSessionInfo], - pending_game_launch: Option<&GameInfo>, - queue_servers: &mut Vec, - queue_loading: bool, - queue_sort_mode: crate::app::QueueSortMode, - queue_region_filter: &crate::app::QueueRegionFilter, - show_server_selection: bool, - selected_queue_server: &Option, - pending_server_selection_game: Option<&GameInfo>, - actions: &mut Vec, - ) { - // Top bar with tabs, search, and logout - subscription info moved to bottom - egui::Panel::top("top_bar") - .frame( - egui::Frame::new() - .fill(egui::Color32::from_rgb(22, 22, 30)) - .inner_margin(egui::Margin { - left: 0, - right: 0, - top: 10, - bottom: 10, - }), - ) - .show(ctx, |ui| { - ui.horizontal(|ui| { - ui.add_space(15.0); - - // Logo - ui.label( - egui::RichText::new("OpenNOW") - .size(24.0) - .color(egui::Color32::from_rgb(118, 185, 0)) - .strong(), - ); - - ui.add_space(20.0); - - // Tab buttons - solid style like login button - let home_selected = current_tab == GamesTab::Home; - let all_games_selected = current_tab == GamesTab::AllGames; - let library_selected = current_tab == GamesTab::MyLibrary; - let queue_times_selected = current_tab == GamesTab::QueueTimes; - - // Home tab button - let home_btn = egui::Button::new( - egui::RichText::new("Home") - .size(13.0) - .color(egui::Color32::WHITE) - .strong(), - ) - .fill(if home_selected { - egui::Color32::from_rgb(118, 185, 0) - } else { - egui::Color32::from_rgb(50, 50, 65) - }) - .corner_radius(6.0); - - if ui.add_sized([70.0, 32.0], home_btn).clicked() && !home_selected { - actions.push(UiAction::SwitchTab(GamesTab::Home)); - } - - ui.add_space(8.0); - - let all_games_btn = egui::Button::new( - egui::RichText::new("All Games") - .size(13.0) - .color(egui::Color32::WHITE) - .strong(), - ) - .fill(if all_games_selected { - egui::Color32::from_rgb(118, 185, 0) - } else { - egui::Color32::from_rgb(50, 50, 65) - }) - .corner_radius(6.0); - - if ui.add_sized([90.0, 32.0], all_games_btn).clicked() && !all_games_selected { - actions.push(UiAction::SwitchTab(GamesTab::AllGames)); - } - - ui.add_space(8.0); - - let library_btn = egui::Button::new( - egui::RichText::new("My Library") - .size(13.0) - .color(egui::Color32::WHITE) - .strong(), - ) - .fill(if library_selected { - egui::Color32::from_rgb(118, 185, 0) - } else { - egui::Color32::from_rgb(50, 50, 65) - }) - .corner_radius(6.0); - - if ui.add_sized([90.0, 32.0], library_btn).clicked() && !library_selected { - actions.push(UiAction::SwitchTab(GamesTab::MyLibrary)); - } - - ui.add_space(8.0); - - // Queue Times tab button (for free tier users) - let queue_times_btn = egui::Button::new( - egui::RichText::new("🕐 Queue Times") - .size(13.0) - .color(egui::Color32::WHITE) - .strong(), - ) - .fill(if queue_times_selected { - egui::Color32::from_rgb(118, 185, 0) - } else { - egui::Color32::from_rgb(50, 50, 65) - }) - .corner_radius(6.0); - - if ui - .add_sized([120.0, 32.0], queue_times_btn) - .on_hover_text("View queue times for servers") - .clicked() - && !queue_times_selected - { - actions.push(UiAction::SwitchTab(GamesTab::QueueTimes)); - } - - ui.add_space(20.0); - - // Search box in the middle - egui::Frame::new() - .fill(egui::Color32::from_rgb(35, 35, 45)) - .corner_radius(6.0) - .inner_margin(egui::Margin { - left: 10, - right: 10, - top: 6, - bottom: 6, - }) - .stroke(egui::Stroke::new(1.0, egui::Color32::from_rgb(60, 60, 75))) - .show(ui, |ui| { - ui.horizontal(|ui| { - ui.label( - egui::RichText::new("🔍") - .size(12.0) - .color(egui::Color32::from_rgb(120, 120, 140)), - ); - ui.add_space(6.0); - let search = egui::TextEdit::singleline(search_query) - .hint_text("Search games...") - .desired_width(200.0) - .frame(false) - .text_color(egui::Color32::WHITE); - ui.add(search); - }); - }); - - // Right side content - ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { - ui.add_space(15.0); - - // Logout button - solid style - let logout_btn = egui::Button::new( - egui::RichText::new("Logout") - .size(13.0) - .color(egui::Color32::WHITE), - ) - .fill(egui::Color32::from_rgb(50, 50, 65)) - .corner_radius(6.0); - - if ui.add_sized([80.0, 32.0], logout_btn).clicked() { - actions.push(UiAction::Logout); - } - - ui.add_space(10.0); - - // Settings button - between hours and logout - let settings_btn = - egui::Button::new(egui::RichText::new("⚙").size(16.0).color( - if show_settings_modal { - egui::Color32::from_rgb(118, 185, 0) - } else { - egui::Color32::WHITE - }, - )) - .fill(if show_settings_modal { - egui::Color32::from_rgb(50, 70, 50) - } else { - egui::Color32::from_rgb(50, 50, 65) - }) - .corner_radius(6.0); - - if ui.add_sized([36.0, 32.0], settings_btn).clicked() { - actions.push(UiAction::ToggleSettingsModal); - } - }); - }); - }); - - // Bottom bar with subscription stats - egui::Panel::bottom("bottom_bar") - .frame( - egui::Frame::new() - .fill(egui::Color32::from_rgb(22, 22, 30)) - .inner_margin(egui::Margin { - left: 15, - right: 15, - top: 8, - bottom: 8, - }), - ) - .show(ctx, |ui| { - ui.horizontal(|ui| { - if let Some(sub) = subscription { - // Membership tier badge - let (tier_bg, tier_fg) = match sub.membership_tier.as_str() { - // Ultimate: Gold/Bronze theme - "ULTIMATE" => ( - egui::Color32::from_rgb(80, 60, 10), - egui::Color32::from_rgb(255, 215, 0), - ), - // Priority/Performance: Brown theme - "PERFORMANCE" | "PRIORITY" => ( - egui::Color32::from_rgb(70, 40, 20), - egui::Color32::from_rgb(205, 175, 149), - ), - // Free: Gray theme - _ => ( - egui::Color32::from_rgb(45, 45, 45), - egui::Color32::from_rgb(180, 180, 180), - ), - }; - - egui::Frame::new() - .fill(tier_bg) - .corner_radius(4.0) - .inner_margin(egui::Margin { - left: 8, - right: 8, - top: 4, - bottom: 4, - }) - .show(ui, |ui| { - ui.label( - egui::RichText::new(&sub.membership_tier) - .size(11.0) - .color(tier_fg) - .strong(), - ); - }); - - // Alliance badge (if using an Alliance partner) - if crate::auth::get_selected_provider().is_alliance_partner() { - ui.add_space(8.0); - egui::Frame::new() - .fill(egui::Color32::from_rgb(30, 80, 130)) - .corner_radius(4.0) - .inner_margin(egui::Margin { - left: 8, - right: 8, - top: 4, - bottom: 4, - }) - .show(ui, |ui| { - ui.label( - egui::RichText::new("ALLIANCE") - .size(11.0) - .color(egui::Color32::from_rgb(100, 180, 255)) - .strong(), - ); - }); - } - - ui.add_space(20.0); - - // Hours icon and remaining - ui.label( - egui::RichText::new("⏱") - .size(14.0) - .color(egui::Color32::GRAY), - ); - ui.add_space(5.0); - - // Show ∞ for unlimited subscriptions, otherwise show hours - if sub.is_unlimited { - ui.label( - egui::RichText::new("∞") - .size(15.0) - .color(egui::Color32::from_rgb(118, 185, 0)) - .strong(), - ); - } else { - let hours_color = if sub.remaining_hours > 5.0 { - egui::Color32::from_rgb(118, 185, 0) - } else if sub.remaining_hours > 1.0 { - egui::Color32::from_rgb(255, 200, 50) - } else { - egui::Color32::from_rgb(255, 80, 80) - }; - - ui.label( - egui::RichText::new(format!("{:.1}h", sub.remaining_hours)) - .size(13.0) - .color(hours_color) - .strong(), - ); - ui.label( - egui::RichText::new(format!(" / {:.0}h", sub.total_hours)) - .size(12.0) - .color(egui::Color32::GRAY), - ); - } - - ui.add_space(20.0); - - // Storage icon and space (if available) - if sub.has_persistent_storage { - if let Some(storage_gb) = sub.storage_size_gb { - ui.label( - egui::RichText::new("💾") - .size(14.0) - .color(egui::Color32::GRAY), - ); - ui.add_space(5.0); - ui.label( - egui::RichText::new(format!("{} GB", storage_gb)) - .size(13.0) - .color(egui::Color32::from_rgb(100, 180, 255)), - ); - } - } - } else { - ui.label( - egui::RichText::new("Loading subscription info...") - .size(12.0) - .color(egui::Color32::GRAY), - ); - } - - // Right side: server info - ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { - // Show selected server - if auto_server_selection { - let best_server = servers - .iter() - .filter(|s| { - s.status == crate::app::ServerStatus::Online - && s.ping_ms.is_some() - }) - .min_by_key(|s| s.ping_ms.unwrap_or(9999)); - - if let Some(server) = best_server { - ui.label( - egui::RichText::new(format!( - "🌐 Auto: {} ({}ms)", - server.name, - server.ping_ms.unwrap_or(0) - )) - .size(12.0) - .color(egui::Color32::from_rgb(118, 185, 0)), - ); - } else if ping_testing { - ui.label( - egui::RichText::new("🌐 Testing servers...") - .size(12.0) - .color(egui::Color32::GRAY), - ); - } else { - ui.label( - egui::RichText::new("🌐 Auto (waiting for ping)") - .size(12.0) - .color(egui::Color32::GRAY), - ); - } - } else if let Some(server) = servers.get(selected_server_index) { - let ping_text = server - .ping_ms - .map(|p| format!(" ({}ms)", p)) - .unwrap_or_default(); - ui.label( - egui::RichText::new(format!("🌐 {}{}", server.name, ping_text)) - .size(12.0) - .color(egui::Color32::from_rgb(100, 180, 255)), - ); - } - }); - }); - }); - - // Main content area - egui::CentralPanel::default().show(ctx, |ui| { - ui.add_space(15.0); - - // Render based on current tab - match current_tab { - GamesTab::Home => { - // Home tab - horizontal scrolling sections - if game_sections.is_empty() { - ui.vertical_centered(|ui| { - ui.add_space(100.0); - ui.label( - egui::RichText::new("Loading sections...") - .size(14.0) - .color(egui::Color32::from_rgb(120, 120, 120)) - ); - }); - } else { - egui::ScrollArea::vertical() - .auto_shrink([false, false]) - .show(ui, |ui| { - ui.add_space(5.0); - - for section in game_sections { - // Section header - ui.horizontal(|ui| { - ui.add_space(10.0); - ui.label( - egui::RichText::new(§ion.title) - .size(18.0) - .strong() - .color(egui::Color32::WHITE) - ); - }); - - ui.add_space(10.0); - - // Horizontal scroll of game cards - ui.horizontal(|ui| { - ui.add_space(10.0); - egui::ScrollArea::horizontal() - .id_salt(§ion.title) - .auto_shrink([false, false]) - .show(ui, |ui| { - ui.horizontal(|ui| { - for (idx, game) in section.games.iter().enumerate() { - Self::render_game_card(ui, ctx, idx, game, _runtime, game_textures, new_textures, actions); - ui.add_space(12.0); - } - }); - }); - }); - - ui.add_space(20.0); - } - }); - } - } - GamesTab::QueueTimes => { - // Check if we have any ping data (to know if ping test is still running) - let has_ping_data = queue_servers.iter().any(|s| s.ping_ms.is_some()); - - // Get recommended server (only if we have ping data) - let recommended_server = if has_ping_data { - crate::api::get_auto_selected_server(queue_servers) - } else { - None - }; - - // Aggregated location data (grouped by display_name within a region) - #[derive(Debug, Clone)] - struct LocationInfo { - display_name: String, - avg_queue_position: i32, - avg_eta_seconds: Option, - best_ping_ms: Option, - server_count: usize, - has_5080: bool, - has_4080: bool, - } - - // Apply region filter to servers - let filtered_servers: Vec<&crate::api::QueueServerInfo> = queue_servers.iter() - .filter(|s| match queue_region_filter { - crate::app::QueueRegionFilter::All => true, - crate::app::QueueRegionFilter::Region(ref region) => &s.region == region, - }) - .collect(); - - // Group servers by region, then by location (display_name) - let mut regions: std::collections::HashMap>> = std::collections::HashMap::new(); - - for server in filtered_servers.iter() { - let region_entry = regions.entry(server.region.clone()).or_insert_with(std::collections::HashMap::new); - let location_entry = region_entry.entry(server.display_name.clone()).or_insert_with(Vec::new); - location_entry.push(*server); - } - - // Build aggregated location info for each region - let mut region_locations: std::collections::HashMap> = std::collections::HashMap::new(); - - for (region, locations) in ®ions { - let mut location_list: Vec = Vec::new(); - - for (display_name, servers) in locations { - let count = servers.len(); - let avg_queue = if count > 0 { - let sum: i64 = servers.iter().map(|s| s.queue_position as i64).sum(); - let avg_i64 = sum / count as i64; - avg_i64.clamp(i32::MIN as i64, i32::MAX as i64) as i32 - } else { - 0 - }; - let avg_eta = { - let eta_sum: i64 = servers.iter().filter_map(|s| s.eta_seconds).sum(); - let eta_count = servers.iter().filter(|s| s.eta_seconds.is_some()).count(); - if eta_count > 0 { Some(eta_sum / eta_count as i64) } else { None } - }; - let best_ping = servers.iter().filter_map(|s| s.ping_ms).min(); - let has_5080 = servers.iter().any(|s| s.is_5080_server); - let has_4080 = servers.iter().any(|s| s.is_4080_server); - - location_list.push(LocationInfo { - display_name: display_name.clone(), - avg_queue_position: avg_queue, - avg_eta_seconds: avg_eta, - best_ping_ms: best_ping, - server_count: count, - has_5080, - has_4080, - }); - } - - // Sort locations based on the selected sort mode - // All sorts use display_name as a tiebreaker for stable ordering - match queue_sort_mode { - crate::app::QueueSortMode::BestValue => { - // Sort by a combined score (lower is better) - location_list.sort_by(|a, b| { - let score_a = a.best_ping_ms.unwrap_or(500) as f64 - + (a.avg_eta_seconds.unwrap_or(0) as f64 / 60.0 * 0.5).min(100.0); - let score_b = b.best_ping_ms.unwrap_or(500) as f64 - + (b.avg_eta_seconds.unwrap_or(0) as f64 / 60.0 * 0.5).min(100.0); - score_a.partial_cmp(&score_b) - .unwrap_or(std::cmp::Ordering::Equal) - .then_with(|| a.display_name.cmp(&b.display_name)) - }); - } - crate::app::QueueSortMode::QueueTime => { - location_list.sort_by(|a, b| { - let eta_a = a.avg_eta_seconds.unwrap_or(i64::MAX); - let eta_b = b.avg_eta_seconds.unwrap_or(i64::MAX); - eta_a.cmp(&eta_b) - .then_with(|| a.display_name.cmp(&b.display_name)) - }); - } - crate::app::QueueSortMode::Ping => { - location_list.sort_by(|a, b| { - let ping_a = a.best_ping_ms.unwrap_or(u32::MAX); - let ping_b = b.best_ping_ms.unwrap_or(u32::MAX); - ping_a.cmp(&ping_b) - .then_with(|| a.display_name.cmp(&b.display_name)) - }); - } - crate::app::QueueSortMode::Alphabetical => { - location_list.sort_by(|a, b| a.display_name.cmp(&b.display_name)); - } - } - region_locations.insert(region.clone(), location_list); - } - - // Sort regions by priority - let mut region_keys: Vec<_> = regions.keys().cloned().collect(); - region_keys.sort_by(|a, b| { - let order = |r: &str| match r { - "US" => 0, "EU" => 1, "CA" => 2, "JP" => 3, "KR" => 4, "THAI" => 5, "MY" => 6, - "SG" => 7, "TW" => 8, "AU" => 9, "LATAM" => 10, "TR" => 11, "SA" => 12, - _ => 100 - }; - order(a).cmp(&order(b)).then(a.cmp(b)) - }); - - // Header with title and controls - ui.horizontal(|ui| { - ui.add_space(16.0); - - // Title - ui.label( - egui::RichText::new("Queue Times") - .size(22.0) - .strong() - .color(egui::Color32::WHITE) - ); - - ui.add_space(12.0); - - // Region count badge - if queue_loading { - ui.spinner(); - } else { - egui::Frame::new() - .fill(egui::Color32::from_rgb(40, 40, 55)) - .corner_radius(12.0) - .inner_margin(egui::Margin { left: 10, right: 10, top: 4, bottom: 4 }) - .show(ui, |ui| { - ui.label( - egui::RichText::new(format!("{} regions", region_keys.len())) - .size(12.0) - .color(egui::Color32::from_rgb(150, 150, 150)) - ); - }); - } - - ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { - ui.add_space(16.0); - - // Refresh button - let refresh_btn = egui::Button::new( - egui::RichText::new("↻ Refresh") - .size(12.0) - .color(egui::Color32::WHITE) - ) - .fill(egui::Color32::from_rgb(50, 50, 65)) - .corner_radius(6.0); - - if ui.add(refresh_btn).clicked() { - actions.push(UiAction::RefreshQueueTimes); - } - }); - }); - - ui.add_space(16.0); - - if queue_servers.is_empty() && !queue_loading { - // Empty state - ui.vertical_centered(|ui| { - ui.add_space(100.0); - ui.label( - egui::RichText::new("📊") - .size(48.0) - ); - ui.add_space(16.0); - ui.label( - egui::RichText::new("No Queue Data Available") - .size(18.0) - .strong() - .color(egui::Color32::from_rgb(180, 180, 180)) - ); - ui.add_space(8.0); - ui.label( - egui::RichText::new("Click Refresh to load queue times") - .size(14.0) - .color(egui::Color32::from_rgb(120, 120, 120)) - ); - }); - } else { - egui::ScrollArea::vertical() - .auto_shrink([false, false]) - .show(ui, |ui| { - ui.add_space(8.0); - - // Recommended server card - ui.horizontal(|ui| { - ui.add_space(16.0); - - if !has_ping_data { - // Ping test still running - show loading state - egui::Frame::new() - .fill(egui::Color32::from_rgb(35, 35, 50)) - .stroke(egui::Stroke::new(1.0, egui::Color32::from_rgb(80, 80, 100))) - .corner_radius(12.0) - .inner_margin(egui::Margin::same(16)) - .show(ui, |ui| { - ui.set_width(ui.available_width() - 32.0); - ui.horizontal(|ui| { - ui.spinner(); - ui.add_space(12.0); - ui.vertical(|ui| { - ui.label( - egui::RichText::new("⭐ RECOMMENDED") - .size(11.0) - .strong() - .color(egui::Color32::from_rgb(100, 100, 120)) - ); - ui.add_space(4.0); - ui.label( - egui::RichText::new("Waiting for ping test to finish...") - .size(14.0) - .color(egui::Color32::from_rgb(140, 140, 160)) - ); - }); - }); - }); - } else if let Some(best) = recommended_server { - egui::Frame::new() - .fill(egui::Color32::from_rgb(25, 45, 25)) - .stroke(egui::Stroke::new(1.5, egui::Color32::from_rgb(118, 185, 0))) - .corner_radius(12.0) - .inner_margin(egui::Margin::same(16)) - .show(ui, |ui| { - ui.set_width(ui.available_width() - 32.0); - ui.horizontal(|ui| { - // Star icon and recommended label - ui.label( - egui::RichText::new("⭐ RECOMMENDED") - .size(11.0) - .strong() - .color(egui::Color32::from_rgb(118, 185, 0)) - ); - - ui.add_space(12.0); - - // Server info - ui.label( - egui::RichText::new(&best.display_name) - .size(16.0) - .strong() - .color(egui::Color32::WHITE) - ); - - ui.add_space(8.0); - - // GPU badge - let gpu_text = if best.is_5080_server { - "5080" - } else if best.is_4080_server { - "4080" - } else { - "Unknown" - }; - egui::Frame::new() - .fill(egui::Color32::from_rgb(118, 185, 0).gamma_multiply(0.3)) - .corner_radius(4.0) - .inner_margin(egui::Margin { left: 6, right: 6, top: 2, bottom: 2 }) - .show(ui, |ui| { - ui.label( - egui::RichText::new(gpu_text) - .size(10.0) - .strong() - .color(egui::Color32::from_rgb(118, 185, 0)) - ); - }); - - ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { - // Wait time - let eta_text = crate::api::format_queue_eta(best.eta_seconds); - ui.label( - egui::RichText::new(format!("~{}", eta_text)) - .size(14.0) - .color(egui::Color32::from_rgb(118, 185, 0)) - ); - - ui.add_space(12.0); - - // Queue position in box - let position_color = if best.queue_position <= 0 { - egui::Color32::from_rgb(118, 185, 0) - } else if best.queue_position <= 5 { - // Low queue: green - egui::Color32::from_rgb(118, 185, 0) - } else if best.queue_position <= 15 { - // Medium queue: orange - egui::Color32::from_rgb(255, 165, 0) - } else { - // High queue: red - egui::Color32::from_rgb(230, 80, 80) - }; - let pos_text = if best.queue_position <= 0 { - "0".to_string() - } else { - format!("{}", best.queue_position) - }; - egui::Frame::new() - .fill(position_color.gamma_multiply(0.2)) - .corner_radius(4.0) - .inner_margin(egui::Margin { left: 8, right: 8, top: 3, bottom: 3 }) - .show(ui, |ui| { - ui.label( - egui::RichText::new(pos_text) - .size(13.0) - .strong() - .color(position_color) - ); - }); - - ui.add_space(12.0); - - // Ping - if let Some(ping) = best.ping_ms { - ui.label( - egui::RichText::new(format!("{}ms", ping)) - .size(13.0) - .color(egui::Color32::from_rgb(140, 180, 140)) - ); - } - }); - }); - }); - } - }); - - ui.add_space(20.0); - - // Region sections with locations (using CollapsingHeader) - for region in ®ion_keys { - if let Some(locations) = region_locations.get(region) { - let (flag, region_name) = match region.as_str() { - "US" => ("🇺🇸", "United States"), - "EU" => ("🇪🇺", "Europe"), - "CA" => ("🇨🇦", "Canada"), - "JP" => ("🇯🇵", "Japan"), - "THAI" => ("🇹🇭", "Thailand"), - "MY" => ("🇲🇾", "Malaysia"), - "KR" => ("🇰🇷", "South Korea"), - "SG" => ("🇸🇬", "Singapore"), - "TW" => ("🇹🇼", "Taiwan"), - "AU" => ("🇦🇺", "Australia"), - "LATAM" => ("🌎", "Latin America"), - "TR" => ("🇹🇷", "Turkey"), - "SA" => ("🇸🇦", "Saudi Arabia"), - "AF" => ("🌍", "Africa"), - "RU" => ("🇷🇺", "Russia"), - _ => ("🌐", region.as_str()), - }; - - // Region container with padding - ui.add_space(4.0); - ui.horizontal(|ui| { - ui.add_space(16.0); - ui.vertical(|ui| { - ui.set_width(ui.available_width() - 32.0); - - // Use CollapsingHeader for expandable regions - let header_text = format!("{} {} ({} locations)", flag, region_name, locations.len()); - egui::CollapsingHeader::new( - egui::RichText::new(header_text) - .size(15.0) - .strong() - .color(egui::Color32::WHITE) - ) - .default_open(true) - .show(ui, |ui| { - // Location rows within this region - for location in locations { - ui.horizontal(|ui| { - ui.add_space(20.0); // Indent under region - - egui::Frame::new() - .fill(egui::Color32::from_rgb(28, 28, 38)) - .corner_radius(6.0) - .inner_margin(egui::Margin { left: 12, right: 12, top: 8, bottom: 8 }) - .show(ui, |ui| { - ui.set_width(ui.available_width() - 36.0); - ui.horizontal(|ui| { - // Location name - ui.allocate_ui_with_layout( - egui::vec2(110.0, 20.0), - egui::Layout::left_to_right(egui::Align::Center), - |ui| { - ui.label( - egui::RichText::new(&location.display_name) - .size(13.0) - .color(egui::Color32::WHITE) - ); - } - ); - - // Server count if > 1 - if location.server_count > 1 { - egui::Frame::new() - .fill(egui::Color32::from_rgb(45, 45, 60)) - .corner_radius(4.0) - .inner_margin(egui::Margin { left: 5, right: 5, top: 2, bottom: 2 }) - .show(ui, |ui| { - ui.label( - egui::RichText::new(format!("x{}", location.server_count)) - .size(9.0) - .color(egui::Color32::from_rgb(120, 120, 140)) - ); - }); - ui.add_space(4.0); - } - - // GPU badges - if location.has_5080 { - let gpu_color = egui::Color32::from_rgb(118, 185, 0); - egui::Frame::new() - .fill(gpu_color.gamma_multiply(0.2)) - .corner_radius(4.0) - .inner_margin(egui::Margin { left: 5, right: 5, top: 2, bottom: 2 }) - .show(ui, |ui| { - ui.label( - egui::RichText::new("5080") - .size(9.0) - .strong() - .color(gpu_color) - ); - }); - ui.add_space(4.0); - } - if location.has_4080 { - let gpu_color = egui::Color32::from_rgb(100, 160, 220); - egui::Frame::new() - .fill(gpu_color.gamma_multiply(0.2)) - .corner_radius(4.0) - .inner_margin(egui::Margin { left: 5, right: 5, top: 2, bottom: 2 }) - .show(ui, |ui| { - ui.label( - egui::RichText::new("4080") - .size(9.0) - .strong() - .color(gpu_color) - ); - }); - } - - ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { - // ETA - let eta_text = crate::api::format_queue_eta(location.avg_eta_seconds); - let eta_color = if location.avg_eta_seconds.unwrap_or(0) <= 0 { - egui::Color32::from_rgb(118, 185, 0) - } else if location.avg_eta_seconds.unwrap_or(0) < 300 { - egui::Color32::from_rgb(255, 200, 50) - } else { - egui::Color32::from_rgb(150, 150, 150) - }; - - ui.label( - egui::RichText::new(format!("~{}", eta_text)) - .size(12.0) - .color(eta_color) - ); - - ui.add_space(12.0); - - // Queue position - let position_color = if location.avg_queue_position <= 0 { - egui::Color32::from_rgb(118, 185, 0) - } else if location.avg_queue_position < 20 { - egui::Color32::from_rgb(255, 200, 50) - } else if location.avg_queue_position < 100 { - egui::Color32::from_rgb(255, 150, 80) - } else { - egui::Color32::from_rgb(255, 100, 100) - }; - - egui::Frame::new() - .fill(position_color.gamma_multiply(0.2)) - .corner_radius(4.0) - .inner_margin(egui::Margin { left: 8, right: 8, top: 3, bottom: 3 }) - .show(ui, |ui| { - ui.label( - egui::RichText::new(format!("{}", location.avg_queue_position)) - .size(12.0) - .strong() - .color(position_color) - ); - }); - - ui.add_space(12.0); - - // Ping - if let Some(ping) = location.best_ping_ms { - let ping_color = if ping < 50 { - egui::Color32::from_rgb(118, 185, 0) - } else if ping < 100 { - egui::Color32::from_rgb(255, 200, 50) - } else { - egui::Color32::from_rgb(255, 150, 80) - }; - ui.label( - egui::RichText::new(format!("{}ms", ping)) - .size(12.0) - .color(ping_color) - ); - } else if !has_ping_data { - ui.spinner(); - } else { - ui.label( - egui::RichText::new("-") - .size(12.0) - .color(egui::Color32::from_rgb(80, 80, 100)) - ); - } - }); - }); - }); - }); - ui.add_space(4.0); - } - }); - }); - }); - } - } - - // Attribution footer - ui.add_space(16.0); - ui.vertical_centered(|ui| { - ui.horizontal(|ui| { - ui.label( - egui::RichText::new("Powered by") - .size(11.0) - .color(egui::Color32::from_rgb(80, 80, 80)) - ); - ui.add_space(4.0); - if ui.add( - egui::Label::new( - egui::RichText::new("PrintedWaste") - .size(11.0) - .color(egui::Color32::from_rgb(118, 185, 0)) - .underline() - ).sense(egui::Sense::click()) - ).on_hover_cursor(egui::CursorIcon::PointingHand).clicked() { - if let Err(e) = open::that("https://printedwaste.com/gfn/") { - warn!("Failed to open PrintedWaste link: {}", e); - } - } - }); - }); - ui.add_space(20.0); - }); - } - } - GamesTab::AllGames | GamesTab::MyLibrary => { - // Grid view for All Games and My Library tabs - let header_text = match current_tab { - GamesTab::AllGames => format!("All Games ({} available)", games.len()), - GamesTab::MyLibrary => format!("My Library ({} games)", games.len()), - _ => String::new(), - }; - - ui.horizontal(|ui| { - ui.add_space(10.0); - ui.label( - egui::RichText::new(header_text) - .size(20.0) - .strong() - .color(egui::Color32::WHITE) - ); - }); - - ui.add_space(20.0); - - if games.is_empty() { - ui.vertical_centered(|ui| { - ui.add_space(100.0); - let empty_text = match current_tab { - GamesTab::AllGames => "No games found", - GamesTab::MyLibrary => "Your library is empty.\nPurchase games from Steam, Epic, or other stores to see them here.", - _ => "", - }; - ui.label( - egui::RichText::new(empty_text) - .size(14.0) - .color(egui::Color32::from_rgb(120, 120, 120)) - ); - }); - } else { - // Games grid with VIRTUAL SCROLLING - only render visible rows - // This dramatically reduces CPU usage from rendering 648 games to ~20-30 - let available_width = ui.available_width(); - let card_width = 220.0; - let spacing = 16.0; - let num_columns = ((available_width + spacing) / (card_width + spacing)).floor() as usize; - let num_columns = num_columns.max(2).min(6); - - // Card height including image (124px) + title area (~60px) + spacing - let row_height = 124.0 + 60.0 + spacing; - let total_games = games.len(); - let total_rows = (total_games + num_columns - 1) / num_columns; - - egui::ScrollArea::vertical() - .auto_shrink([false, false]) - .show_viewport(ui, |ui, viewport| { - // Calculate which rows are visible - let first_visible_row = (viewport.min.y / row_height).floor() as usize; - let last_visible_row = ((viewport.max.y / row_height).ceil() as usize).min(total_rows); - - // Add buffer rows for smoother scrolling - let first_row = first_visible_row.saturating_sub(1); - let last_row = (last_visible_row + 1).min(total_rows); - - // Reserve space for rows before visible area - if first_row > 0 { - ui.allocate_space(egui::vec2(available_width, first_row as f32 * row_height)); - } - - ui.horizontal(|ui| { - ui.add_space(10.0); - ui.vertical(|ui| { - // Only render visible rows - for row in first_row..last_row { - let start_idx = row * num_columns; - let end_idx = (start_idx + num_columns).min(total_games); - - ui.horizontal(|ui| { - ui.spacing_mut().item_spacing.x = spacing; - for game_idx in start_idx..end_idx { - if let Some((idx, game)) = games.get(game_idx) { - Self::render_game_card(ui, ctx, *idx, game, _runtime, game_textures, new_textures, actions); - } - } - }); - ui.add_space(spacing); - } - }); - }); - - // Reserve space for rows after visible area - let remaining_rows = total_rows.saturating_sub(last_row); - if remaining_rows > 0 { - ui.allocate_space(egui::vec2(available_width, remaining_rows as f32 * row_height)); - } - }); - } - } - } - }); - - // Game detail popup - if let Some(game) = selected_game_popup { - Self::render_game_popup(ctx, game, game_textures, subscription, actions); - } - - // Server selection modal (for free tier users) - if show_server_selection { - if let Some(game) = pending_server_selection_game { - Self::render_server_selection_modal( - ctx, - game, - queue_servers, - queue_loading, - selected_queue_server, - actions, - ); - } - } - - // Settings modal - if show_settings_modal { - render_settings_modal( - ctx, - settings, - servers, - selected_server_index, - auto_server_selection, - ping_testing, - subscription, - actions, - ); - } - - // Session conflict dialog - if show_session_conflict { - render_session_conflict_dialog(ctx, active_sessions, pending_game_launch, actions); - } - - // AV1 hardware warning dialog - if show_av1_warning { - render_av1_warning_dialog(ctx, actions); - } - - // Alliance experimental warning dialog - if show_alliance_warning { - render_alliance_warning_dialog(ctx, alliance_provider_name, actions); - } - } - - // Note: render_settings_modal, render_session_conflict_dialog, render_av1_warning_dialog - // have been moved to src/gui/screens/dialogs.rs - // render_login_screen, render_session_screen moved to src/gui/screens/ - - /// Render the game detail popup - fn render_game_popup( - ctx: &egui::Context, - game: &crate::app::GameInfo, - game_textures: &HashMap, - subscription: Option<&crate::app::SubscriptionInfo>, - actions: &mut Vec, - ) { - // Check if user is free tier (show server selection modal instead of direct launch). - // If subscription info is not available, default to treating the user as non-free - // to avoid incorrectly restricting paid users when data hasn't loaded yet. - let is_free_tier = subscription - .map(|s| s.membership_tier == "FREE") - .unwrap_or(false); - let popup_width = 450.0; - let popup_height = 500.0; - - // Modal overlay (darkens background) - egui::Area::new(egui::Id::new("modal_overlay")) - .fixed_pos([0.0, 0.0]) - .interactable(true) - .order(egui::Order::Background) // Draw behind windows - .show(ctx, |ui| { - let screen_rect = ctx.input(|i| i.viewport_rect()); - ui.painter() - .rect_filled(screen_rect, 0.0, egui::Color32::from_black_alpha(200)); - // Close popup on background click - if ui - .allocate_rect(screen_rect, egui::Sense::click()) - .clicked() - { - actions.push(UiAction::CloseGamePopup); - } - }); - - egui::Window::new("Game Details") - .collapsible(false) - .resizable(false) - .fixed_size([popup_width, popup_height]) - .anchor(egui::Align2::CENTER_CENTER, [0.0, 0.0]) - .show(ctx, |ui| { - ui.vertical(|ui| { - // Game image - if let Some(ref image_url) = game.image_url { - if let Some(texture) = game_textures.get(image_url) { - let image_size = egui::vec2(popup_width - 40.0, 150.0); - ui.add( - egui::Image::new(texture) - .fit_to_exact_size(image_size) - .corner_radius(8.0), - ); - } else { - // Placeholder - let placeholder_size = egui::vec2(popup_width - 40.0, 150.0); - let (_, rect) = ui.allocate_space(placeholder_size); - ui.painter().rect_filled( - rect, - 8.0, - egui::Color32::from_rgb(50, 50, 70), - ); - let initial = game - .title - .chars() - .next() - .unwrap_or('?') - .to_uppercase() - .to_string(); - ui.painter().text( - rect.center(), - egui::Align2::CENTER_CENTER, - initial, - egui::FontId::proportional(48.0), - egui::Color32::from_rgb(100, 100, 130), - ); - } - } - - ui.add_space(15.0); - - // Game title - ui.label( - egui::RichText::new(&game.title) - .size(20.0) - .strong() - .color(egui::Color32::WHITE), - ); - - ui.add_space(8.0); - - // Platform selector (if multiple variants) or store badge (single variant) - if game.variants.len() > 1 { - ui.horizontal(|ui| { - ui.label( - egui::RichText::new("Platform:") - .size(12.0) - .color(egui::Color32::GRAY), - ); - - // Show platform buttons - for (idx, variant) in game.variants.iter().enumerate() { - let is_selected = idx == game.selected_variant_index; - let btn_color = if is_selected { - egui::Color32::from_rgb(100, 180, 255) // Bright blue for selected - } else { - egui::Color32::from_rgb(60, 60, 80) // Dark for unselected - }; - let text_color = if is_selected { - egui::Color32::WHITE - } else { - egui::Color32::LIGHT_GRAY - }; - - let btn = egui::Button::new( - egui::RichText::new(variant.store.to_uppercase()) - .size(11.0) - .color(text_color), - ) - .fill(btn_color) - .corner_radius(4.0) - .min_size(egui::vec2(60.0, 24.0)); - - if ui.add(btn).clicked() && !is_selected { - actions.push(UiAction::SelectVariant(idx)); - } - } - }); - } else { - // Single store badge (existing behavior) - ui.horizontal(|ui| { - ui.label( - egui::RichText::new("Store:") - .size(12.0) - .color(egui::Color32::GRAY), - ); - ui.label( - egui::RichText::new(&game.store.to_uppercase()) - .size(12.0) - .color(egui::Color32::from_rgb(100, 180, 255)) - .strong(), - ); - }); - } - - // Publisher if available - if let Some(ref publisher) = game.publisher { - ui.horizontal(|ui| { - ui.label( - egui::RichText::new("Publisher:") - .size(12.0) - .color(egui::Color32::GRAY), - ); - ui.label( - egui::RichText::new(publisher) - .size(12.0) - .color(egui::Color32::LIGHT_GRAY), - ); - }); - } - - ui.add_space(8.0); - - // GFN Status (Play Type and Membership) - if let Some(ref play_type) = game.play_type { - ui.horizontal(|ui| { - ui.label( - egui::RichText::new("Type:") - .size(12.0) - .color(egui::Color32::GRAY), - ); - let color = if play_type == "INSTALL_TO_PLAY" { - egui::Color32::from_rgb(255, 180, 50) // Orange - } else { - egui::Color32::from_rgb(100, 200, 100) // Green - }; - ui.label( - egui::RichText::new(play_type) - .size(12.0) - .color(color) - .strong(), - ); - }); - } - - if let Some(ref tier) = game.membership_tier_label { - ui.horizontal(|ui| { - ui.label( - egui::RichText::new("Requires:") - .size(12.0) - .color(egui::Color32::GRAY), - ); - ui.label( - egui::RichText::new(tier) - .size(12.0) - .color(egui::Color32::from_rgb(118, 185, 0)) // Nvidia Green - .strong(), - ); - }); - } - - if let Some(ref text) = game.playability_text { - ui.add_space(4.0); - ui.add( - egui::Label::new( - egui::RichText::new(text) - .size(11.0) - .color(egui::Color32::LIGHT_GRAY), - ) - .wrap(), - ); - } - - ui.add_space(20.0); - - // Description - if let Some(ref desc) = game.description { - ui.label( - egui::RichText::new("About this game:") - .size(14.0) - .strong() - .color(egui::Color32::WHITE), - ); - ui.add_space(4.0); - egui::ScrollArea::vertical() - .max_height(100.0) - .show(ui, |ui| { - ui.label( - egui::RichText::new(desc) - .size(13.0) - .color(egui::Color32::LIGHT_GRAY), - ); - }); - ui.add_space(15.0); - } else { - ui.add_space(20.0); - } - - // Buttons - ui.horizontal(|ui| { - // Play button (for free tier, show server selection first) - let play_btn = egui::Button::new( - egui::RichText::new(" Play Now ").size(16.0).strong(), - ) - .fill(egui::Color32::from_rgb(70, 180, 70)) - .min_size(egui::vec2(120.0, 40.0)); - - if ui.add(play_btn).clicked() { - if is_free_tier { - // Free tier: show server selection modal - actions.push(UiAction::ShowServerSelection(game.clone())); - actions.push(UiAction::CloseGamePopup); - } else { - // Paid tier: launch directly - actions.push(UiAction::LaunchGameDirect(game.clone())); - actions.push(UiAction::CloseGamePopup); - } - } - - ui.add_space(20.0); - - // Close button - let close_btn = - egui::Button::new(egui::RichText::new(" Close ").size(14.0)) - .fill(egui::Color32::from_rgb(60, 60, 80)) - .min_size(egui::vec2(80.0, 40.0)); - - if ui.add(close_btn).clicked() { - actions.push(UiAction::CloseGamePopup); - } - }); - }); - }); - } - - /// Render the server selection modal (for free tier users choosing a server before launching) - fn render_server_selection_modal( - ctx: &egui::Context, - game: &crate::app::GameInfo, - queue_servers: &[crate::api::QueueServerInfo], - queue_loading: bool, - selected_server: &Option, - actions: &mut Vec, - ) { - // Modal overlay - egui::Area::new(egui::Id::new("server_selection_overlay")) - .fixed_pos([0.0, 0.0]) - .interactable(true) - .order(egui::Order::Background) - .show(ctx, |ui| { - let screen_rect = ctx.input(|i| i.viewport_rect()); - ui.painter() - .rect_filled(screen_rect, 0.0, egui::Color32::from_black_alpha(200)); - if ui - .allocate_rect(screen_rect, egui::Sense::click()) - .clicked() - { - actions.push(UiAction::CloseServerSelection); - } - }); - - // Check if we have ping data - let has_ping_data = queue_servers.iter().any(|s| s.ping_ms.is_some()); - - // Get recommended server (best score across all servers) - let recommended = if has_ping_data { - crate::api::get_auto_selected_server(queue_servers) - } else { - None - }; - - // Group servers by region (already simple: "US", "EU", etc. from queue API) and find best server per region - let mut region_best: std::collections::HashMap = - std::collections::HashMap::new(); - - for server in queue_servers { - let entry = region_best.entry(server.region.clone()).or_insert(server); - // Replace if this server has better score - let current_score = crate::api::calculate_server_score(entry); - let new_score = crate::api::calculate_server_score(server); - if new_score < current_score { - *entry = server; - } - } - - // Sort regions by priority - let mut region_keys: Vec<_> = region_best.keys().cloned().collect(); - region_keys.sort_by(|a, b| { - let order = |r: &str| match r { - "US" => 0, - "EU" => 1, - "CA" => 2, - "JP" => 3, - "KR" => 4, - "THAI" => 5, - "MY" => 6, - "SG" => 7, - "TW" => 8, - "AU" => 9, - "LATAM" => 10, - "TR" => 11, - "SA" => 12, - _ => 100, - }; - order(a).cmp(&order(b)).then(a.cmp(b)) - }); - - egui::Window::new("Choose Server") - .collapsible(false) - .resizable(false) - .fixed_size([520.0, 480.0]) - .anchor(egui::Align2::CENTER_CENTER, [0.0, 0.0]) - .show(ctx, |ui| { - ui.vertical(|ui| { - // Game title - ui.horizontal(|ui| { - ui.label( - egui::RichText::new("Launching:") - .size(14.0) - .color(egui::Color32::GRAY), - ); - ui.label( - egui::RichText::new(&game.title) - .size(14.0) - .strong() - .color(egui::Color32::WHITE), - ); - }); - - ui.add_space(12.0); - ui.separator(); - ui.add_space(10.0); - - // Auto/Recommended option - let auto_selected = selected_server.is_none(); - let auto_frame_fill = if auto_selected { - egui::Color32::from_rgb(30, 50, 30) - } else { - egui::Color32::from_rgb(35, 35, 50) - }; - let auto_stroke = if auto_selected { - egui::Stroke::new(2.0, egui::Color32::from_rgb(118, 185, 0)) - } else { - egui::Stroke::new(1.0, egui::Color32::from_rgb(60, 60, 80)) - }; - - let auto_response = egui::Frame::new() - .fill(auto_frame_fill) - .stroke(auto_stroke) - .corner_radius(8.0) - .inner_margin(egui::Margin::same(12)) - .show(ui, |ui| { - ui.set_width(ui.available_width()); - ui.horizontal(|ui| { - ui.label(egui::RichText::new("⭐").size(18.0)); - ui.add_space(8.0); - ui.vertical(|ui| { - ui.label( - egui::RichText::new("Auto (Recommended)") - .size(15.0) - .strong() - .color(egui::Color32::WHITE), - ); - if !has_ping_data { - ui.horizontal(|ui| { - ui.spinner(); - ui.add_space(8.0); - ui.label( - egui::RichText::new("Testing ping to servers...") - .size(12.0) - .color(egui::Color32::from_rgb(140, 140, 160)), - ); - }); - } else if let Some(best) = recommended { - ui.label( - egui::RichText::new(format!( - "{} • {}ms • ~{}", - best.display_name, - best.ping_ms.unwrap_or(0), - crate::api::format_queue_eta(best.eta_seconds) - )) - .size(12.0) - .color(egui::Color32::from_rgb(118, 185, 0)), - ); - } - }); - }); - }) - .response; - - if auto_response.interact(egui::Sense::click()).clicked() { - actions.push(UiAction::SelectQueueServer(None)); - } - - ui.add_space(12.0); - - // Region list - ui.label( - egui::RichText::new("Or choose a region:") - .size(12.0) - .color(egui::Color32::from_rgb(120, 120, 120)), - ); - - ui.add_space(8.0); - - if queue_loading && queue_servers.is_empty() { - ui.horizontal(|ui| { - ui.spinner(); - ui.add_space(8.0); - ui.label( - egui::RichText::new("Loading servers...") - .size(13.0) - .color(egui::Color32::GRAY), - ); - }); - } else { - egui::ScrollArea::vertical() - .max_height(220.0) - .show(ui, |ui| { - // Show regions in sorted order - for region in ®ion_keys { - if let Some(best_server) = region_best.get(region) { - let (flag, region_name) = match region.as_str() { - "US" => ("🇺🇸", "United States"), - "EU" => ("🇪🇺", "Europe"), - "CA" => ("🇨🇦", "Canada"), - "JP" => ("🇯🇵", "Japan"), - "THAI" => ("🇹🇭", "Thailand"), - "MY" => ("🇲🇾", "Malaysia"), - "KR" => ("🇰🇷", "South Korea"), - "SG" => ("🇸🇬", "Singapore"), - "TW" => ("🇹🇼", "Taiwan"), - "AU" => ("🇦🇺", "Australia"), - "LATAM" => ("🌎", "Latin America"), - "TR" => ("🇹🇷", "Turkey"), - "SA" => ("🇸🇦", "Saudi Arabia"), - "AF" => ("🌍", "Africa"), - "RU" => ("🇷🇺", "Russia"), - _ => ("🌐", region.as_str()), - }; - - let is_selected = selected_server.as_ref() - == Some(&best_server.server_id); - let frame_fill = if is_selected { - egui::Color32::from_rgb(40, 50, 70) - } else { - egui::Color32::from_rgb(30, 30, 42) - }; - let frame_stroke = if is_selected { - egui::Stroke::new( - 1.5, - egui::Color32::from_rgb(100, 160, 220), - ) - } else { - egui::Stroke::NONE - }; - - let server_response = egui::Frame::new() - .fill(frame_fill) - .stroke(frame_stroke) - .corner_radius(6.0) - .inner_margin(egui::Margin::symmetric(12, 10)) - .show(ui, |ui| { - ui.set_width(ui.available_width()); - ui.horizontal(|ui| { - // Flag and region name - ui.label(egui::RichText::new(flag).size(18.0)); - ui.add_space(8.0); - ui.allocate_ui_with_layout( - egui::vec2(130.0, 20.0), - egui::Layout::left_to_right( - egui::Align::Center, - ), - |ui| { - ui.label( - egui::RichText::new(region_name) - .size(14.0) - .strong() - .color(egui::Color32::WHITE), - ); - }, - ); - - // Best server location in this region - ui.label( - egui::RichText::new( - &best_server.display_name, - ) - .size(11.0) - .color(egui::Color32::from_rgb( - 120, 120, 140, - )), - ); - - ui.with_layout( - egui::Layout::right_to_left( - egui::Align::Center, - ), - |ui| { - // ETA - let eta_text = - crate::api::format_queue_eta( - best_server.eta_seconds, - ); - let eta_color = if best_server - .eta_seconds - .unwrap_or(0) - <= 0 - { - egui::Color32::from_rgb(118, 185, 0) - } else if best_server - .eta_seconds - .unwrap_or(0) - < 300 - { - egui::Color32::from_rgb( - 255, 200, 50, - ) - } else { - egui::Color32::from_rgb( - 150, 150, 150, - ) - }; - ui.label( - egui::RichText::new(format!( - "~{}", - eta_text - )) - .size(11.0) - .color(eta_color), - ); - - ui.add_space(10.0); - - // Queue position in box - let queue_color = if best_server - .queue_position - <= 0 - { - egui::Color32::from_rgb(118, 185, 0) - } else if best_server.queue_position - < 20 - { - egui::Color32::from_rgb( - 255, 200, 50, - ) - } else if best_server.queue_position - < 100 - { - egui::Color32::from_rgb( - 255, 150, 80, - ) - } else { - egui::Color32::from_rgb( - 255, 100, 100, - ) - }; - egui::Frame::new() - .fill( - queue_color.gamma_multiply(0.2), - ) - .corner_radius(3.0) - .inner_margin( - egui::Margin::symmetric(6, 2), - ) - .show(ui, |ui| { - ui.label( - egui::RichText::new( - format!( - "{}", - best_server - .queue_position - ), - ) - .size(11.0) - .strong() - .color(queue_color), - ); - }); - - ui.add_space(10.0); - - // Ping - if let Some(ping) = best_server.ping_ms - { - let ping_color = if ping < 50 { - egui::Color32::from_rgb( - 118, 185, 0, - ) - } else if ping < 100 { - egui::Color32::from_rgb( - 255, 200, 50, - ) - } else { - egui::Color32::from_rgb( - 255, 150, 80, - ) - }; - ui.label( - egui::RichText::new(format!( - "{}ms", - ping - )) - .size(11.0) - .color(ping_color), - ); - } else { - ui.spinner(); - } - }, - ); - }); - }) - .response; - - if server_response.interact(egui::Sense::click()).clicked() - { - actions.push(UiAction::SelectQueueServer(Some( - best_server.server_id.clone(), - ))); - } - - ui.add_space(4.0); - } - } - }); - } - - ui.add_space(16.0); - - // Buttons - ui.horizontal(|ui| { - // Launch button - let launch_btn = egui::Button::new( - egui::RichText::new(" Launch Game ").size(15.0).strong(), - ) - .fill(egui::Color32::from_rgb(70, 180, 70)) - .min_size(egui::vec2(140.0, 38.0)); - - if ui.add(launch_btn).clicked() { - actions.push(UiAction::LaunchWithServer( - game.clone(), - selected_server.clone(), - )); - } - - ui.add_space(12.0); - - // Cancel button - let cancel_btn = - egui::Button::new(egui::RichText::new(" Cancel ").size(14.0)) - .fill(egui::Color32::from_rgb(60, 60, 80)) - .min_size(egui::vec2(90.0, 38.0)); - - if ui.add(cancel_btn).clicked() { - actions.push(UiAction::CloseServerSelection); - } - }); - - // Attribution footer - ui.add_space(10.0); - ui.horizontal(|ui| { - ui.label( - egui::RichText::new("Powered by") - .size(10.0) - .color(egui::Color32::from_rgb(80, 80, 80)), - ); - ui.add_space(4.0); - if ui - .add( - egui::Label::new( - egui::RichText::new("PrintedWaste") - .size(10.0) - .color(egui::Color32::from_rgb(118, 185, 0)) - .underline(), - ) - .sense(egui::Sense::click()), - ) - .on_hover_cursor(egui::CursorIcon::PointingHand) - .clicked() - { - if let Err(e) = open::that("https://printedwaste.com/gfn/") { - warn!("Failed to open PrintedWaste link: {}", e); - } - } - }); - }); - }); - } - - fn render_game_card( - ui: &mut egui::Ui, - ctx: &egui::Context, - _idx: usize, - game: &crate::app::GameInfo, - runtime: &tokio::runtime::Handle, - game_textures: &HashMap, - new_textures: &mut Vec<(String, egui::TextureHandle)>, - actions: &mut Vec, - ) { - // Card dimensions - larger for better visibility - let card_width = 220.0; - let image_height = 124.0; // 16:9 aspect ratio - - // Make the entire card clickable - let game_for_click = game.clone(); - - let response = egui::Frame::new() - .fill(egui::Color32::from_rgb(28, 28, 36)) - .corner_radius(8.0) - .inner_margin(0.0) - .show(ui, |ui| { - ui.set_min_width(card_width); - - ui.vertical(|ui| { - // Game box art image - full width, no padding - if let Some(ref image_url) = game.image_url { - // Check if texture is already loaded - if let Some(texture) = game_textures.get(image_url) { - // Display the image with rounded top corners - let size = egui::vec2(card_width, image_height); - ui.add( - egui::Image::new(texture) - .fit_to_exact_size(size) - .corner_radius(egui::CornerRadius { - nw: 8, - ne: 8, - sw: 0, - se: 0, - }), - ); - } else { - // Check if image data is available in cache - if let Some((pixels, width, height)) = image_cache::get_image(image_url) - { - // Create egui texture from pixels - let color_image = egui::ColorImage::from_rgba_unmultiplied( - [width as usize, height as usize], - &pixels, - ); - let texture = ctx.load_texture( - image_url, - color_image, - egui::TextureOptions::LINEAR, - ); - new_textures.push((image_url.clone(), texture.clone())); - - // Display immediately - let size = egui::vec2(card_width, image_height); - ui.add( - egui::Image::new(&texture) - .fit_to_exact_size(size) - .corner_radius(egui::CornerRadius { - nw: 8, - ne: 8, - sw: 0, - se: 0, - }), - ); - } else { - // Request loading - image_cache::request_image(image_url, runtime); - - // Show placeholder - let placeholder_rect = - ui.allocate_space(egui::vec2(card_width, image_height)); - ui.painter().rect_filled( - placeholder_rect.1, - egui::CornerRadius { - nw: 8, - ne: 8, - sw: 0, - se: 0, - }, - egui::Color32::from_rgb(40, 40, 55), - ); - // Loading spinner effect - ui.painter().text( - placeholder_rect.1.center(), - egui::Align2::CENTER_CENTER, - "...", - egui::FontId::proportional(16.0), - egui::Color32::from_rgb(80, 80, 100), - ); - } - } - } else { - // No image URL - show placeholder with game initial - let placeholder_rect = - ui.allocate_space(egui::vec2(card_width, image_height)); - ui.painter().rect_filled( - placeholder_rect.1, - egui::CornerRadius { - nw: 8, - ne: 8, - sw: 0, - se: 0, - }, - egui::Color32::from_rgb(45, 45, 65), - ); - // Show first letter of game title - let initial = game - .title - .chars() - .next() - .unwrap_or('?') - .to_uppercase() - .to_string(); - ui.painter().text( - placeholder_rect.1.center(), - egui::Align2::CENTER_CENTER, - initial, - egui::FontId::proportional(40.0), - egui::Color32::from_rgb(80, 80, 110), - ); - } - - // Text content area with padding - ui.add_space(10.0); - ui.horizontal(|ui| { - ui.add_space(12.0); - ui.vertical(|ui| { - // Game title (truncated if too long) - let title = if game.title.chars().count() > 24 { - let truncated: String = game.title.chars().take(21).collect(); - format!("{}...", truncated) - } else { - game.title.clone() - }; - ui.label( - egui::RichText::new(title) - .size(13.0) - .strong() - .color(egui::Color32::WHITE), - ); - - // Store badge - ui.label( - egui::RichText::new(game.store.to_uppercase()) - .size(10.0) - .color(egui::Color32::from_rgb(100, 180, 255)), - ); - }); - }); - ui.add_space(8.0); - }); - }); - - // Hover effect - green glow - let card_rect = response.response.rect; - if ui.rect_contains_pointer(card_rect) { - ui.ctx().set_cursor_icon(egui::CursorIcon::PointingHand); - ui.painter().rect_stroke( - card_rect, - 8.0, - egui::Stroke::new(2.0, egui::Color32::from_rgb(118, 185, 0)), - egui::StrokeKind::Outside, - ); - } - - if response.response.interact(egui::Sense::click()).clicked() { - actions.push(UiAction::OpenGamePopup(game_for_click)); - } - } - - // Note: render_session_conflict_dialog, render_av1_warning_dialog, render_session_screen - // have been moved to src/gui/screens/dialogs.rs and screens/session.rs -} - -// End of impl Renderer block -// Below is the standalone render_stats_panel function - -/// Render stats panel (standalone function) -fn render_stats_panel( - ctx: &egui::Context, - stats: &crate::media::StreamStats, - position: crate::app::StatsPosition, -) { - use egui::{Align2, Color32, FontId, RichText}; - - let (anchor, offset) = match position { - crate::app::StatsPosition::BottomLeft => (Align2::LEFT_BOTTOM, [10.0, -10.0]), - crate::app::StatsPosition::BottomRight => (Align2::RIGHT_BOTTOM, [-10.0, -10.0]), - crate::app::StatsPosition::TopLeft => (Align2::LEFT_TOP, [10.0, 10.0]), - crate::app::StatsPosition::TopRight => (Align2::RIGHT_TOP, [-10.0, 10.0]), - }; - - egui::Area::new(egui::Id::new("stats_panel")) - .anchor(anchor, offset) - .interactable(false) - .show(ctx, |ui| { - egui::Frame::new() - .fill(Color32::from_rgba_unmultiplied(0, 0, 0, 200)) - .corner_radius(4.0) - .inner_margin(8.0) - .show(ui, |ui| { - ui.set_min_width(200.0); - - // Resolution and HDR status - let res_text = if stats.resolution.is_empty() { - "Connecting...".to_string() - } else { - stats.resolution.clone() - }; - - ui.horizontal(|ui| { - ui.label( - RichText::new(res_text) - .font(FontId::monospace(13.0)) - .color(Color32::WHITE), - ); - - // HDR indicator - if stats.is_hdr { - ui.label( - RichText::new(" HDR") - .font(FontId::monospace(13.0)) - .color(Color32::from_rgb(255, 180, 0)), // Orange/gold for HDR - ); - } - }); - - // Decoded FPS vs Render FPS (shows if renderer is bottlenecked) - let decode_fps = stats.fps; - let render_fps = stats.render_fps; - let target_fps = stats.target_fps as f32; - - // Decode FPS color - let decode_color = if target_fps > 0.0 { - let ratio = decode_fps / target_fps; - if ratio >= 0.8 { - Color32::GREEN - } else if ratio >= 0.5 { - Color32::YELLOW - } else { - Color32::from_rgb(255, 100, 100) - } - } else { - Color32::WHITE - }; - - // Render FPS color (critical - this is what you actually see) - let render_color = if target_fps > 0.0 { - let ratio = render_fps / target_fps; - if ratio >= 0.8 { - Color32::GREEN - } else if ratio >= 0.5 { - Color32::YELLOW - } else { - Color32::from_rgb(255, 100, 100) - } - } else { - Color32::WHITE - }; - - // Show both FPS values - ui.horizontal(|ui| { - ui.label( - RichText::new(format!("Decode: {:.0}", decode_fps)) - .font(FontId::monospace(11.0)) - .color(decode_color), - ); - ui.label( - RichText::new(format!(" | Render: {:.0}", render_fps)) - .font(FontId::monospace(11.0)) - .color(render_color), - ); - if stats.target_fps > 0 { - ui.label( - RichText::new(format!(" / {} fps", stats.target_fps)) - .font(FontId::monospace(11.0)) - .color(Color32::GRAY), - ); - } - }); - - // Codec and bitrate - if !stats.codec.is_empty() { - ui.label( - RichText::new(format!( - "{} | {:.1} Mbps", - stats.codec, stats.bitrate_mbps - )) - .font(FontId::monospace(11.0)) - .color(Color32::LIGHT_GRAY), - ); - } - - // Latency (decode pipeline) - let latency_color = if stats.latency_ms < 30.0 { - Color32::GREEN - } else if stats.latency_ms < 60.0 { - Color32::YELLOW - } else { - Color32::RED - }; - - ui.label( - RichText::new(format!("Decode: {:.0} ms", stats.latency_ms)) - .font(FontId::monospace(11.0)) - .color(latency_color), - ); - - // Network RTT (round-trip time from ICE) - if stats.rtt_ms > 0.0 { - let rtt_color = if stats.rtt_ms < 30.0 { - Color32::GREEN - } else if stats.rtt_ms < 60.0 { - Color32::YELLOW - } else { - Color32::RED - }; - - ui.label( - RichText::new(format!("RTT: {:.0} ms", stats.rtt_ms)) - .font(FontId::monospace(11.0)) - .color(rtt_color), - ); - } else { - ui.label( - RichText::new("RTT: N/A") - .font(FontId::monospace(11.0)) - .color(Color32::GRAY), - ); - } - - // Estimated end-to-end latency (motion-to-photon) - if stats.estimated_e2e_ms > 0.0 { - let e2e_color = if stats.estimated_e2e_ms < 80.0 { - Color32::GREEN - } else if stats.estimated_e2e_ms < 150.0 { - Color32::YELLOW - } else { - Color32::RED - }; - - ui.label( - RichText::new(format!("E2E: ~{:.0} ms", stats.estimated_e2e_ms)) - .font(FontId::monospace(11.0)) - .color(e2e_color), - ); - } - - // Input rate and client-side latency - if stats.input_rate > 0.0 || stats.input_latency_ms > 0.0 { - let rate_str = if stats.input_rate > 0.0 { - format!("{:.0}/s", stats.input_rate) - } else { - "0/s".to_string() - }; - let latency_str = if stats.input_latency_ms > 0.001 { - format!("{:.2}ms", stats.input_latency_ms) - } else { - "<0.01ms".to_string() - }; - ui.label( - RichText::new(format!("Input: {} ({})", rate_str, latency_str)) - .font(FontId::monospace(10.0)) - .color(Color32::GRAY), - ); - } - - // Frame delivery latency (RTP to decode) - if stats.frame_delivery_ms > 0.0 { - let delivery_color = if stats.frame_delivery_ms < 10.0 { - Color32::GREEN - } else if stats.frame_delivery_ms < 20.0 { - Color32::YELLOW - } else { - Color32::RED - }; - ui.label( - RichText::new(format!( - "Frame delivery: {:.1} ms", - stats.frame_delivery_ms - )) - .font(FontId::monospace(10.0)) - .color(delivery_color), - ); - } - - if stats.packet_loss > 0.0 { - let loss_color = if stats.packet_loss < 1.0 { - Color32::YELLOW - } else { - Color32::RED - }; - - ui.label( - RichText::new(format!("Packet Loss: {:.1}%", stats.packet_loss)) - .font(FontId::monospace(11.0)) - .color(loss_color), - ); - } - - // Decode and render times - if stats.decode_time_ms > 0.0 || stats.render_time_ms > 0.0 { - ui.label( - RichText::new(format!( - "Decode: {:.1} ms | Render: {:.1} ms", - stats.decode_time_ms, stats.render_time_ms - )) - .font(FontId::monospace(10.0)) - .color(Color32::GRAY), - ); - } - - // Frame stats - if stats.frames_received > 0 { - ui.label( - RichText::new(format!( - "Frames: {} rx, {} dec, {} drop", - stats.frames_received, stats.frames_decoded, stats.frames_dropped - )) - .font(FontId::monospace(10.0)) - .color(Color32::DARK_GRAY), - ); - } - - // GPU and server info - if !stats.gpu_type.is_empty() || !stats.server_region.is_empty() { - let info = format!( - "{}{}{}", - stats.gpu_type, - if !stats.gpu_type.is_empty() && !stats.server_region.is_empty() { - " | " - } else { - "" - }, - stats.server_region - ); - - ui.label( - RichText::new(info) - .font(FontId::monospace(10.0)) - .color(Color32::DARK_GRAY), - ); - } - }); - }); -} - -/// Render resolution change notification popup (animated, center-top) -fn render_resolution_notification( - ctx: &egui::Context, - old_res: &str, - new_res: &str, - direction: ResolutionDirection, - alpha: f32, -) { - use egui::{Align2, Color32, FontId, RichText}; - - // Calculate alpha for animation (0-255) - let alpha_u8 = (alpha * 255.0) as u8; - - // Colors based on direction (using ASCII-compatible symbols) - let (arrow, color, label) = match direction { - ResolutionDirection::Up => ( - "+", - Color32::from_rgba_unmultiplied(100, 255, 100, alpha_u8), - "Quality Increased", - ), - ResolutionDirection::Down => ( - "-", - Color32::from_rgba_unmultiplied(255, 150, 100, alpha_u8), - "Quality Decreased", - ), - ResolutionDirection::Same => ( - "=", - Color32::from_rgba_unmultiplied(200, 200, 200, alpha_u8), - "Quality Changed", - ), - }; - - // Slide-in animation: start 20px above, slide down to final position - let slide_offset = (1.0 - alpha.min(1.0)) * -20.0; - - egui::Area::new(egui::Id::new("resolution_notification")) - .anchor(Align2::CENTER_TOP, [0.0, 40.0 + slide_offset]) - .interactable(false) - .order(egui::Order::Foreground) - .show(ctx, |ui| { - egui::Frame::new() - .fill(Color32::from_rgba_unmultiplied( - 20, - 20, - 25, - (alpha * 230.0) as u8, - )) - .corner_radius(8.0) - .inner_margin(egui::Margin::symmetric(16, 12)) - .stroke(egui::Stroke::new( - 1.0, - Color32::from_rgba_unmultiplied(80, 80, 90, alpha_u8), - )) - .show(ui, |ui| { - ui.vertical(|ui| { - ui.spacing_mut().item_spacing.y = 6.0; - - // Title with arrow - ui.horizontal(|ui| { - ui.label( - RichText::new(arrow) - .font(FontId::proportional(18.0)) - .color(color), - ); - ui.label( - RichText::new(label).font(FontId::proportional(14.0)).color( - Color32::from_rgba_unmultiplied(255, 255, 255, alpha_u8), - ), - ); - }); - - // Resolution change details - ui.horizontal(|ui| { - // Old resolution (strikethrough effect with dim color) - ui.label( - RichText::new(old_res).font(FontId::monospace(13.0)).color( - Color32::from_rgba_unmultiplied(150, 150, 150, alpha_u8), - ), - ); - ui.label( - RichText::new("->").font(FontId::monospace(13.0)).color( - Color32::from_rgba_unmultiplied(200, 200, 200, alpha_u8), - ), - ); - // New resolution (bright) - ui.label( - RichText::new(new_res) - .font(FontId::monospace(13.0)) - .strong() - .color(color), - ); - }); - }); - }); - }); - - // Request repaint for smooth animation - ctx.request_repaint(); -} - -/// Render racing wheel connection notification popup (animated, center-top) -/// Shows when a racing wheel is detected during streaming session -fn render_wheel_notification(ctx: &egui::Context, wheel_count: usize, alpha: f32) { - use egui::{Align2, Color32, FontId, RichText}; - - // Calculate alpha for animation (0-255) - let alpha_u8 = (alpha * 255.0) as u8; - - // Green color for wheel detection (positive feedback) - let accent_color = Color32::from_rgba_unmultiplied(100, 200, 100, alpha_u8); - - // Slide-in animation: start 20px above, slide down to final position - let slide_offset = (1.0 - alpha.min(1.0)) * -20.0; - - egui::Area::new(egui::Id::new("wheel_notification")) - .anchor(Align2::CENTER_TOP, [0.0, 100.0 + slide_offset]) // Below resolution notification - .interactable(false) - .order(egui::Order::Foreground) - .show(ctx, |ui| { - egui::Frame::new() - .fill(Color32::from_rgba_unmultiplied( - 20, - 30, - 25, - (alpha * 230.0) as u8, - )) - .corner_radius(8.0) - .inner_margin(egui::Margin::symmetric(16, 12)) - .stroke(egui::Stroke::new( - 1.0, - Color32::from_rgba_unmultiplied(60, 100, 70, alpha_u8), - )) - .show(ui, |ui| { - ui.vertical(|ui| { - ui.spacing_mut().item_spacing.y = 6.0; - - // Title with steering wheel icon (using ASCII-compatible symbol) - ui.horizontal(|ui| { - // Use a circle/wheel-like character that's cross-platform compatible - ui.label( - RichText::new("(O)") - .font(FontId::monospace(16.0)) - .color(accent_color), - ); - ui.label( - RichText::new("Racing Wheel Detected") - .font(FontId::proportional(14.0)) - .strong() - .color(Color32::from_rgba_unmultiplied( - 255, 255, 255, alpha_u8, - )), - ); - }); - - // Wheel count and info - ui.horizontal(|ui| { - let wheel_text = if wheel_count == 1 { - "1 wheel connected".to_string() - } else { - format!("{} wheels connected", wheel_count) - }; - ui.label( - RichText::new(wheel_text) - .font(FontId::monospace(12.0)) - .color(Color32::from_rgba_unmultiplied( - 180, 180, 180, alpha_u8, - )), - ); - }); - - // Supported features hint - ui.horizontal(|ui| { - ui.label( - RichText::new("Wheel, pedals, and shifter input active") - .font(FontId::proportional(11.0)) - .color(Color32::from_rgba_unmultiplied( - 140, 160, 140, alpha_u8, - )), - ); - }); - }); - }); - }); - - // Request repaint for smooth animation - ctx.request_repaint(); -} diff --git a/opennow-streamer/src/gui/screens/login.rs b/opennow-streamer/src/gui/screens/login.rs deleted file mode 100644 index a2deb3a..0000000 --- a/opennow-streamer/src/gui/screens/login.rs +++ /dev/null @@ -1,212 +0,0 @@ -//! Login Screen -//! -//! Renders the login/provider selection screen. - -use crate::app::UiAction; -use crate::auth::LoginProvider; - -/// Render the login screen with provider selection -pub fn render_login_screen( - ctx: &egui::Context, - login_providers: &[LoginProvider], - selected_provider_index: usize, - status_message: &str, - is_loading: bool, - login_url: Option<&str>, - actions: &mut Vec, -) { - egui::CentralPanel::default().show(ctx, |ui| { - let available_height = ui.available_height(); - let content_height = 400.0; - let top_padding = ((available_height - content_height) / 2.0).max(40.0); - - ui.vertical_centered(|ui| { - ui.add_space(top_padding); - - // Logo/Title with gradient-like effect - ui.label( - egui::RichText::new("OpenNOW") - .size(48.0) - .color(egui::Color32::from_rgb(118, 185, 0)) // NVIDIA green - .strong(), - ); - - ui.add_space(8.0); - ui.label( - egui::RichText::new("GeForce NOW Client") - .size(14.0) - .color(egui::Color32::from_rgb(150, 150, 150)), - ); - - ui.add_space(60.0); - - // Login card container - egui::Frame::new() - .fill(egui::Color32::from_rgb(30, 30, 40)) - .corner_radius(12.0) - .inner_margin(egui::Margin { - left: 40, - right: 40, - top: 30, - bottom: 30, - }) - .show(ui, |ui| { - ui.set_min_width(320.0); - - ui.vertical(|ui| { - // Region selection label - centered - ui.with_layout(egui::Layout::top_down(egui::Align::Center), |ui| { - ui.label( - egui::RichText::new("Select Region") - .size(13.0) - .color(egui::Color32::from_rgb(180, 180, 180)), - ); - }); - - ui.add_space(10.0); - - // Provider dropdown - centered using horizontal with spacing - ui.horizontal(|ui| { - let available_width = ui.available_width(); - let combo_width = 240.0; - let padding = (available_width - combo_width) / 2.0; - ui.add_space(padding.max(0.0)); - - let selected_name = login_providers - .get(selected_provider_index) - .map(|p| p.login_provider_display_name.as_str()) - .unwrap_or("NVIDIA (Global)"); - - egui::ComboBox::from_id_salt("provider_select") - .selected_text(selected_name) - .width(combo_width) - .show_ui(ui, |ui| { - for (i, provider) in login_providers.iter().enumerate() { - let is_selected = i == selected_provider_index; - if ui - .selectable_label( - is_selected, - &provider.login_provider_display_name, - ) - .clicked() - { - actions.push(UiAction::SelectProvider(i)); - } - } - }); - }); - - ui.add_space(25.0); - - // Login button or loading state - centered - ui.with_layout(egui::Layout::top_down(egui::Align::Center), |ui| { - if is_loading { - ui.add_space(10.0); - ui.spinner(); - ui.add_space(12.0); - ui.label( - egui::RichText::new("Waiting for login...") - .size(13.0) - .color(egui::Color32::from_rgb(118, 185, 0)), - ); - ui.add_space(5.0); - ui.label( - egui::RichText::new("Complete login in your browser") - .size(11.0) - .color(egui::Color32::GRAY), - ); - - // Show login URL with copy button if browser didn't open - if let Some(url) = login_url { - ui.add_space(15.0); - ui.separator(); - ui.add_space(10.0); - - ui.label( - egui::RichText::new("Browser didn't open? Copy this link:") - .size(11.0) - .color(egui::Color32::from_rgb(180, 180, 180)), - ); - - ui.add_space(8.0); - - // URL text box (read-only, selectable) - let mut url_text = url.to_string(); - let text_edit = egui::TextEdit::singleline(&mut url_text) - .font(egui::TextStyle::Small) - .desired_width(280.0) - .interactive(true); - ui.add(text_edit); - - ui.add_space(8.0); - - // Copy button - let copy_btn = egui::Button::new( - egui::RichText::new("Copy URL") - .size(12.0) - .color(egui::Color32::WHITE), - ) - .fill(egui::Color32::from_rgb(60, 60, 80)) - .corner_radius(4.0); - - if ui.add_sized([120.0, 28.0], copy_btn).clicked() { - ui.ctx().copy_text(url.to_string()); - } - - ui.add_space(5.0); - ui.label( - egui::RichText::new( - "Paste this URL in your browser to login", - ) - .size(10.0) - .color(egui::Color32::from_rgb(120, 120, 120)), - ); - } - } else { - let login_btn = egui::Button::new( - egui::RichText::new("Sign In") - .size(15.0) - .color(egui::Color32::WHITE) - .strong(), - ) - .fill(egui::Color32::from_rgb(118, 185, 0)) - .corner_radius(6.0); - - if ui.add_sized([240.0, 42.0], login_btn).clicked() { - actions.push(UiAction::StartLogin); - } - - ui.add_space(15.0); - - ui.label( - egui::RichText::new("Sign in with your NVIDIA account") - .size(11.0) - .color(egui::Color32::from_rgb(120, 120, 120)), - ); - } - }); - }); - }); - - ui.add_space(20.0); - - // Status message (if any) - if !status_message.is_empty() && status_message != "Welcome to OpenNOW" { - ui.label( - egui::RichText::new(status_message) - .size(11.0) - .color(egui::Color32::from_rgb(150, 150, 150)), - ); - } - - ui.add_space(40.0); - - // Footer info - ui.label( - egui::RichText::new("Alliance Partners can select their region above") - .size(10.0) - .color(egui::Color32::from_rgb(80, 80, 80)), - ); - }); - }); -} diff --git a/opennow-streamer/src/gui/screens/mod.rs b/opennow-streamer/src/gui/screens/mod.rs deleted file mode 100644 index bea620f..0000000 --- a/opennow-streamer/src/gui/screens/mod.rs +++ /dev/null @@ -1,866 +0,0 @@ -//! Screen Components -//! -//! UI screens and dialogs for the application. - -mod login; -mod session; - -pub use login::render_login_screen; -pub use session::render_session_screen; - -use crate::app::config::{ColorQuality, FPS_OPTIONS, RESOLUTIONS}; -use crate::app::session::ActiveSessionInfo; -use crate::app::{GameInfo, ServerInfo, SettingChange, Settings, UiAction}; - -/// Render the settings modal with bitrate slider and other options -/// Render the settings modal with bitrate slider and other options -pub fn render_settings_modal( - ctx: &egui::Context, - settings: &Settings, - servers: &[ServerInfo], - selected_server_index: usize, - auto_server_selection: bool, - ping_testing: bool, - subscription: Option<&crate::app::SubscriptionInfo>, - actions: &mut Vec, -) { - egui::Window::new("Settings") - .collapsible(false) - .resizable(false) - .fixed_size([500.0, 450.0]) // Increased size for cleaner layout - .anchor(egui::Align2::CENTER_CENTER, [0.0, 0.0]) - .show(ctx, |ui| { - egui::ScrollArea::vertical().show(ui, |ui| { - ui.add_space(8.0); - - // === Video Settings Section === - ui.heading(egui::RichText::new("Video").color(egui::Color32::from_rgb(118, 185, 0))); - ui.add_space(8.0); - - egui::Grid::new("video_settings_grid") - .num_columns(2) - .spacing([24.0, 16.0]) - .show(ui, |ui| { - // Max Bitrate - ui.label("Max Bitrate") - .on_hover_text("Controls the maximum bandwidth usage for video streaming.\nHigher values improve quality but require a stable, fast internet connection."); - ui.vertical(|ui| { - ui.horizontal(|ui| { - let mut bitrate = settings.max_bitrate_mbps as f32; - let slider = egui::Slider::new(&mut bitrate, 10.0..=200.0) - .show_value(false) - .step_by(5.0); - if ui.add(slider).changed() { - actions.push(UiAction::UpdateSetting(SettingChange::MaxBitrate(bitrate as u32))); - } - ui.label(egui::RichText::new(format!("{} Mbps", settings.max_bitrate_mbps)).strong()); - }); - ui.label(egui::RichText::new("Recommend: 50-75 Mbps for most users").size(10.0).weak()); - }); - ui.end_row(); - - // Resolution - ui.label("Resolution") - .on_hover_text("The resolution of the video stream."); - ui.with_layout(egui::Layout::left_to_right(egui::Align::Center), |ui| { - let current_display = RESOLUTIONS.iter() - .find(|(res, _)| *res == settings.resolution) - .map(|(_, name)| *name) - .unwrap_or(&settings.resolution); - - egui::ComboBox::from_id_salt("resolution_combo") - .selected_text(current_display) - .show_ui(ui, |ui| { - // Use entitled resolutions if available - if let Some(sub) = subscription { - if !sub.entitled_resolutions.is_empty() { - // 1. Deduplicate unique resolutions - let mut unique_resolutions = std::collections::HashSet::new(); - let mut resolutions = Vec::new(); - - // Sort by width then height descending first - let mut sorted_res = sub.entitled_resolutions.clone(); - sorted_res.sort_by(|a, b| b.width.cmp(&a.width).then(b.height.cmp(&a.height))); - - for res in sorted_res { - let key = (res.width, res.height); - if unique_resolutions.contains(&key) { - continue; - } - unique_resolutions.insert(key); - resolutions.push(res); - } - - // 2. Group by Aspect Ratio - let mut groups: std::collections::BTreeMap> = std::collections::BTreeMap::new(); - - for res in resolutions { - let ratio = res.width as f32 / res.height as f32; - let category = if (ratio - 16.0/9.0).abs() < 0.05 { - "16:9 Standard" - } else if (ratio - 16.0/10.0).abs() < 0.05 { - "16:10 Widescreen" - } else if (ratio - 21.0/9.0).abs() < 0.05 { - "21:9 Ultrawide" - } else if (ratio - 32.0/9.0).abs() < 0.05 { - "32:9 Super Ultrawide" - } else if (ratio - 4.0/3.0).abs() < 0.05 { - "4:3 Legacy" - } else { - "Other" - }; - - groups.entry(category.to_string()).or_default().push(res); - } - - // Define preferred order of categories - let order = ["16:9 Standard", "16:10 Widescreen", "21:9 Ultrawide", "32:9 Super Ultrawide", "4:3 Legacy", "Other"]; - - for category in order.iter() { - if let Some(res_list) = groups.get(*category) { - ui.heading(*category); - for res in res_list { - let res_str = format!("{}x{}", res.width, res.height); - - // Friendly name logic - let name = match (res.width, res.height) { - (1280, 720) => "720p (HD)".to_string(), - (1920, 1080) => "1080p (FHD)".to_string(), - (2560, 1440) => "1440p (QHD)".to_string(), - (3840, 2160) => "4K (UHD)".to_string(), - (2560, 1080) => "2560x1080 (Ultrawide)".to_string(), - (3440, 1440) => "3440x1440 (Ultrawide)".to_string(), - (w, h) => format!("{}x{}", w, h), - }; - - if ui.selectable_label(settings.resolution == res_str, name).clicked() { - actions.push(UiAction::UpdateSetting(SettingChange::Resolution(res_str))); - } - } - ui.separator(); - } - } - return; - } - } - - // Fallback to static list - for (res, name) in RESOLUTIONS { - if ui.selectable_label(settings.resolution == *res, *name).clicked() { - actions.push(UiAction::UpdateSetting(SettingChange::Resolution(res.to_string()))); - } - } - }); - }); - ui.end_row(); - - // Frame Rate - ui.label("Frame Rate") - .on_hover_text("Target frame rate for the stream.\nHigh FPS requires more bandwidth and decoder power."); - ui.with_layout(egui::Layout::left_to_right(egui::Align::Center), |ui| { - egui::ComboBox::from_id_salt("fps_combo") - .selected_text(format!("{} FPS", settings.fps)) - .show_ui(ui, |ui| { - // Use entitled FPS for the current resolution if available - if let Some(sub) = subscription { - if !sub.entitled_resolutions.is_empty() { - let (w, h) = crate::app::types::parse_resolution(&settings.resolution); - - // Find max FPS for this resolution - let mut available_fps = Vec::new(); - for res in &sub.entitled_resolutions { - if res.width == w && res.height == h { - available_fps.push(res.fps); - } - } - - // Also include global max FPS just in case resolution match fails - // or if we want to allow users to force lower FPS - if available_fps.is_empty() { - // Fallback to all entitled FPS - for res in &sub.entitled_resolutions { - available_fps.push(res.fps); - } - } - - available_fps.sort(); - available_fps.dedup(); - - if !available_fps.is_empty() { - for fps in available_fps { - if ui.selectable_label(settings.fps == fps, format!("{} FPS", fps)).clicked() { - actions.push(UiAction::UpdateSetting(SettingChange::Fps(fps))); - } - } - return; - } - } - } - - // Fallback to static list - for &fps in FPS_OPTIONS { - if ui.selectable_label(settings.fps == fps, format!("{} FPS", fps)).clicked() { - actions.push(UiAction::UpdateSetting(SettingChange::Fps(fps))); - } - } - }); - }); - ui.end_row(); - - // Video Codec - ui.label("Video Codec") - .on_hover_text("Compression standard used for video.\nAV1 and H.265 (HEVC) offer better quality than H.264 at the same bitrate, but require compatible hardware."); - ui.with_layout(egui::Layout::left_to_right(egui::Align::Center), |ui| { - let codec_text = match settings.codec { - crate::app::VideoCodec::H264 => "H.264", - crate::app::VideoCodec::H265 => "H.265 (HEVC)", - crate::app::VideoCodec::AV1 => "AV1", - }; - egui::ComboBox::from_id_salt("codec_combo") - .selected_text(codec_text) - .show_ui(ui, |ui| { - if ui.selectable_label(matches!(settings.codec, crate::app::VideoCodec::H264), "H.264").clicked() { - actions.push(UiAction::UpdateSetting(SettingChange::Codec(crate::app::VideoCodec::H264))); - } - if ui.selectable_label(matches!(settings.codec, crate::app::VideoCodec::H265), "H.265 (HEVC)").clicked() { - actions.push(UiAction::UpdateSetting(SettingChange::Codec(crate::app::VideoCodec::H265))); - } - if ui.selectable_label(matches!(settings.codec, crate::app::VideoCodec::AV1), "AV1").clicked() { - actions.push(UiAction::UpdateSetting(SettingChange::Codec(crate::app::VideoCodec::AV1))); - } - }); - }); - ui.end_row(); - - // Video Decoder - ui.label("Video Decoder") - .on_hover_text(settings.decoder_backend.description()); - ui.with_layout(egui::Layout::left_to_right(egui::Align::Center), |ui| { - // Show backend info next to selected decoder - let selected_backend = format!("{} ({})", - settings.decoder_backend.as_str(), - settings.decoder_backend.backend_name() - ); - egui::ComboBox::from_id_salt("decoder_combo") - .selected_text(selected_backend) - .show_ui(ui, |ui| { - for backend in crate::media::get_supported_decoder_backends() { - let label = format!("{} ({})", backend.as_str(), backend.backend_name()); - if ui.selectable_label(settings.decoder_backend == backend, &label) - .on_hover_ui_at_pointer(|ui| { - ui.label(backend.description()); - }) - .clicked() - { - actions.push(UiAction::UpdateSetting(SettingChange::DecoderBackend(backend))); - } - } - }); - }); - ui.end_row(); - - // Color Quality - ui.label("Color Quality") - .on_hover_text("Color bit depth and chroma subsampling.\n\n• 4:2:0 - Standard chroma, lower bandwidth\n• 4:4:4 - Full chroma, better for text/UI (requires HEVC)\n• 8-bit - Standard dynamic range\n• 10-bit - HDR capable, smoother gradients"); - ui.with_layout(egui::Layout::left_to_right(egui::Align::Center), |ui| { - egui::ComboBox::from_id_salt("color_quality_combo") - .selected_text(settings.color_quality.display_name()) - .show_ui(ui, |ui| { - for &quality in ColorQuality::all() { - let label = format!("{}", quality.display_name()); - let tooltip = quality.description(); - if ui.selectable_label(settings.color_quality == quality, &label) - .on_hover_text(tooltip) - .clicked() - { - actions.push(UiAction::UpdateSetting(SettingChange::ColorQuality(quality))); - } - } - }); - }); - ui.end_row(); - - // HDR Mode - ui.label("HDR Mode") - .on_hover_text("Enable High Dynamic Range for supported displays.\nRequires 10-bit color and HEVC/AV1 codec.\nWill auto-switch settings when enabled."); - ui.with_layout(egui::Layout::left_to_right(egui::Align::Center), |ui| { - let mut hdr_enabled = settings.hdr_enabled; - if ui.add(egui::Checkbox::new(&mut hdr_enabled, "Enable HDR")).changed() { - actions.push(UiAction::UpdateSetting(SettingChange::Hdr(hdr_enabled))); - } - if settings.hdr_enabled { - ui.label(egui::RichText::new("(10-bit + HEVC required)").size(10.0).weak()); - } - }); - ui.end_row(); - }); - - ui.add_space(20.0); - ui.separator(); - ui.add_space(8.0); - - // === Server Settings Section === - ui.heading(egui::RichText::new("Server & Network").color(egui::Color32::from_rgb(118, 185, 0))); - ui.add_space(8.0); - - egui::Grid::new("server_settings_grid") - .num_columns(2) - .spacing([24.0, 16.0]) - .show(ui, |ui| { - // Auto Selection - ui.label("Server Selection") - .on_hover_text("Choose a specific GeForce NOW server or let the client automatically pick the best one."); - - ui.vertical(|ui| { - let mut auto_select = auto_server_selection; - if ui.checkbox(&mut auto_select, "Auto-select best server").on_hover_text("Automatically selects the server with the lowest ping.").changed() { - actions.push(UiAction::SetAutoServerSelection(auto_select)); - } - - if !auto_server_selection && !servers.is_empty() { - ui.add_space(4.0); - let current_server = servers.get(selected_server_index) - .map(|s| format!("{} ({}ms)", s.name, s.ping_ms.unwrap_or(0))) - .unwrap_or_else(|| "Select server".to_string()); - - egui::ComboBox::from_id_salt("server_combo") - .selected_text(current_server) - .width(250.0) - .show_ui(ui, |ui| { - for (i, server) in servers.iter().enumerate() { - let ping_str = server.ping_ms - .map(|p| format!(" ({}ms)", p)) - .unwrap_or_default(); - let label = format!("{}{}", server.name, ping_str); - if ui.selectable_label(i == selected_server_index, label).clicked() { - actions.push(UiAction::SelectServer(i)); - } - } - }); - } - }); - ui.end_row(); - - // Network Test - if !auto_server_selection && !servers.is_empty() { - ui.label("Network Test") - .on_hover_text("Measure latency to available servers."); - ui.horizontal(|ui| { - if ping_testing { - ui.spinner(); - ui.label("Testing ping..."); - } else if ui.button("Test Ping").clicked() { - actions.push(UiAction::StartPingTest); - } - }); - ui.end_row(); - } - }); - - ui.add_space(20.0); - ui.separator(); - ui.add_space(8.0); - - // === Input Settings Section === - ui.heading(egui::RichText::new("Input").color(egui::Color32::from_rgb(118, 185, 0))); - ui.add_space(8.0); - - egui::Grid::new("input_settings_grid") - .num_columns(2) - .spacing([24.0, 16.0]) - .show(ui, |ui| { - // Clipboard Paste - ui.label("Clipboard Paste") - .on_hover_text("Enable Ctrl+V to paste clipboard text into the remote session.\nText is typed character-by-character (max 64KB).\nUseful for pasting passwords, URLs, or codes."); - ui.horizontal(|ui| { - let mut clipboard_enabled = settings.clipboard_paste_enabled; - if ui.checkbox(&mut clipboard_enabled, "Enable clipboard paste (Ctrl+V)").changed() { - actions.push(UiAction::UpdateSetting(SettingChange::ClipboardPasteEnabled(clipboard_enabled))); - } - }); - ui.end_row(); - }); - - ui.add_space(24.0); - - // Buttons row - ui.horizontal(|ui| { - // Reset button on the left - if ui.button(egui::RichText::new("Reset to Defaults").size(14.0).color(egui::Color32::from_rgb(200, 80, 80))).clicked() { - actions.push(UiAction::ResetSettings); - } - - ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { - if ui.button(egui::RichText::new("Close").size(16.0)).clicked() { - actions.push(UiAction::ToggleSettingsModal); - } - }); - }); - - ui.add_space(8.0); - }); - }); -} - -/// Render session conflict dialog when user has active sessions -pub fn render_session_conflict_dialog( - ctx: &egui::Context, - active_sessions: &[ActiveSessionInfo], - pending_game: Option<&GameInfo>, - actions: &mut Vec, -) { - egui::Window::new("Active Session") - .collapsible(false) - .resizable(false) - .fixed_size([400.0, 250.0]) - .anchor(egui::Align2::CENTER_CENTER, [0.0, 0.0]) - .show(ctx, |ui| { - ui.vertical_centered(|ui| { - ui.add_space(10.0); - - ui.label( - egui::RichText::new("You have an active session") - .size(18.0) - .strong() - .color(egui::Color32::WHITE), - ); - - ui.add_space(15.0); - - // Show active session info - if let Some(session) = active_sessions.first() { - ui.label( - egui::RichText::new(format!("Session ID: {}", &session.session_id)) - .size(14.0) - .color(egui::Color32::from_rgb(118, 185, 0)), - ); - - ui.add_space(5.0); - - if let Some(ref server_ip) = session.server_ip { - ui.label( - egui::RichText::new(format!("Server: {}", server_ip)) - .size(12.0) - .color(egui::Color32::GRAY), - ); - } - } - - ui.add_space(25.0); - - ui.horizontal(|ui| { - // Resume existing session - let resume_btn = - egui::Button::new(egui::RichText::new("Resume Session").size(14.0)) - .fill(egui::Color32::from_rgb(70, 130, 70)) - .min_size(egui::vec2(130.0, 35.0)); - - if ui.add(resume_btn).clicked() { - if let Some(session) = active_sessions.first() { - actions.push(UiAction::ResumeSession(session.clone())); - } - actions.push(UiAction::CloseSessionConflict); - } - - ui.add_space(10.0); - - // Terminate and start new - if let Some(game) = pending_game { - let new_btn = - egui::Button::new(egui::RichText::new("Start New Game").size(14.0)) - .fill(egui::Color32::from_rgb(130, 70, 70)) - .min_size(egui::vec2(130.0, 35.0)); - - if ui.add(new_btn).clicked() { - if let Some(session) = active_sessions.first() { - actions.push(UiAction::TerminateAndLaunch( - session.session_id.clone(), - game.clone(), - )); - } - actions.push(UiAction::CloseSessionConflict); - } - } - }); - - ui.add_space(15.0); - - // Cancel - if ui.button("Cancel").clicked() { - actions.push(UiAction::CloseSessionConflict); - } - }); - }); -} - -/// Render AV1 hardware warning dialog -pub fn render_av1_warning_dialog(ctx: &egui::Context, actions: &mut Vec) { - egui::Window::new("AV1 Not Supported") - .collapsible(false) - .resizable(false) - .fixed_size([400.0, 180.0]) - .anchor(egui::Align2::CENTER_CENTER, [0.0, 0.0]) - .show(ctx, |ui| { - ui.vertical_centered(|ui| { - ui.add_space(15.0); - - ui.label( - egui::RichText::new("⚠ AV1 Hardware Decoding Not Available") - .size(16.0) - .strong() - .color(egui::Color32::from_rgb(255, 180, 50)) - ); - - ui.add_space(15.0); - - ui.label( - egui::RichText::new("Your GPU does not support AV1 hardware decoding.\nAV1 requires an NVIDIA RTX 30 series or newer GPU.") - .size(13.0) - .color(egui::Color32::LIGHT_GRAY) - ); - - ui.add_space(20.0); - - ui.horizontal(|ui| { - if ui.button("Switch to H.265").clicked() { - actions.push(UiAction::UpdateSetting(SettingChange::Codec(crate::app::VideoCodec::H265))); - actions.push(UiAction::CloseAV1Warning); - } - - ui.add_space(10.0); - - if ui.button("Close").clicked() { - actions.push(UiAction::CloseAV1Warning); - } - }); - }); - }); -} - -/// Render Alliance experimental warning dialog -pub fn render_alliance_warning_dialog( - ctx: &egui::Context, - provider_name: &str, - actions: &mut Vec, -) { - egui::Window::new("Alliance Partner") - .collapsible(false) - .resizable(false) - .fixed_size([420.0, 200.0]) - .anchor(egui::Align2::CENTER_CENTER, [0.0, 0.0]) - .show(ctx, |ui| { - ui.vertical_centered(|ui| { - ui.add_space(10.0); - - // Alliance badge - centered - egui::Frame::new() - .fill(egui::Color32::from_rgb(30, 80, 130)) - .corner_radius(6.0) - .inner_margin(egui::Margin { - left: 14, - right: 14, - top: 6, - bottom: 6, - }) - .show(ui, |ui| { - ui.label( - egui::RichText::new("ALLIANCE") - .size(14.0) - .color(egui::Color32::from_rgb(100, 180, 255)) - .strong(), - ); - }); - - ui.add_space(12.0); - - ui.label( - egui::RichText::new(format!("Welcome to {} via Alliance!", provider_name)) - .size(17.0) - .strong() - .color(egui::Color32::WHITE), - ); - - ui.add_space(10.0); - - ui.label( - egui::RichText::new("Alliance support is still experimental.") - .size(14.0) - .color(egui::Color32::from_rgb(255, 200, 80)), - ); - - ui.add_space(6.0); - - ui.label( - egui::RichText::new( - "Please report issues: github.com/zortos293/OpenNOW/issues", - ) - .size(13.0) - .color(egui::Color32::LIGHT_GRAY), - ); - - ui.add_space(6.0); - - ui.label( - egui::RichText::new( - "Note: Feedback from Alliance users is especially valuable!", - ) - .size(12.0) - .color(egui::Color32::GRAY) - .italics(), - ); - - ui.add_space(12.0); - - let got_it_btn = - egui::Button::new(egui::RichText::new("Got it!").size(14.0).strong()) - .fill(egui::Color32::from_rgb(70, 130, 70)) - .min_size(egui::vec2(100.0, 32.0)); - - if ui.add(got_it_btn).clicked() { - actions.push(UiAction::CloseAllianceWarning); - } - }); - }); -} - -/// Render the ads required screen for free tier users -/// -/// This shows an informational screen explaining that ads are required -/// but cannot be displayed in this client. -pub fn render_ads_required_screen( - ctx: &egui::Context, - selected_game: &Option, - ads_remaining_secs: u32, - ads_total_secs: u32, - actions: &mut Vec, -) { - egui::CentralPanel::default().show(ctx, |ui| { - ui.vertical_centered(|ui| { - ui.add_space(80.0); - - // Game title - if let Some(ref game) = selected_game { - ui.label( - egui::RichText::new(&game.title) - .size(28.0) - .strong() - .color(egui::Color32::WHITE), - ); - ui.add_space(30.0); - } - - // Warning icon and header - egui::Frame::new() - .fill(egui::Color32::from_rgb(60, 50, 20)) - .corner_radius(8.0) - .inner_margin(egui::Margin::same(20)) - .show(ui, |ui| { - ui.vertical_centered(|ui| { - ui.label( - egui::RichText::new("FREE TIER - ADS REQUIRED") - .size(20.0) - .strong() - .color(egui::Color32::from_rgb(255, 200, 80)), - ); - - ui.add_space(15.0); - - ui.label( - egui::RichText::new( - "GeForce NOW free tier requires watching video ads\nbefore your gaming session can start.", - ) - .size(14.0) - .color(egui::Color32::LIGHT_GRAY), - ); - - ui.add_space(15.0); - - // Progress indicator (simulated) - let progress = if ads_total_secs > 0 { - 1.0 - (ads_remaining_secs as f32 / ads_total_secs as f32) - } else { - 0.0 - }; - - ui.add( - egui::ProgressBar::new(progress) - .desired_width(300.0) - .text(format!( - "Waiting for ads... (~{} seconds remaining)", - ads_remaining_secs - )), - ); - - ui.add_space(20.0); - - ui.label( - egui::RichText::new( - "OpenNOW cannot display ads from NVIDIA's ad partner.\nYour session will timeout if ads are not watched.", - ) - .size(12.0) - .color(egui::Color32::from_rgb(255, 150, 100)), - ); - - ui.add_space(15.0); - - ui.separator(); - - ui.add_space(10.0); - - ui.label( - egui::RichText::new("Options:") - .size(14.0) - .strong() - .color(egui::Color32::WHITE), - ); - - ui.add_space(8.0); - - ui.label( - egui::RichText::new( - "1. Subscribe to GeForce NOW Priority or Ultimate to skip ads\n2. Use the official GFN client for free tier sessions\n3. Wait - session may proceed if ads timeout (not guaranteed)", - ) - .size(12.0) - .color(egui::Color32::LIGHT_GRAY), - ); - }); - }); - - ui.add_space(30.0); - - // Buttons - ui.horizontal(|ui| { - ui.add_space(ui.available_width() / 2.0 - 150.0); - - // Continue anyway button (session may work after timeout) - let continue_btn = egui::Button::new( - egui::RichText::new("Continue Waiting").size(14.0), - ) - .fill(egui::Color32::from_rgb(60, 80, 60)) - .min_size(egui::vec2(140.0, 35.0)); - - if ui.add(continue_btn).on_hover_text("Wait for the session to proceed (may timeout)").clicked() { - // Just continue - the session poll loop will handle state changes - } - - ui.add_space(20.0); - - // Cancel button - let cancel_btn = egui::Button::new( - egui::RichText::new("Cancel Session").size(14.0), - ) - .fill(egui::Color32::from_rgb(100, 50, 50)) - .min_size(egui::vec2(140.0, 35.0)); - - if ui.add(cancel_btn).clicked() { - actions.push(UiAction::StopStreaming); - } - }); - - ui.add_space(20.0); - - // Link to subscription page - ui.hyperlink_to( - egui::RichText::new("Learn about GeForce NOW subscriptions") - .size(12.0) - .color(egui::Color32::from_rgb(100, 180, 255)), - "https://www.nvidia.com/en-us/geforce-now/memberships/", - ); - }); - }); -} - -/// Render first-time welcome popup -pub fn render_welcome_popup(ctx: &egui::Context, actions: &mut Vec) { - egui::Window::new("Welcome to OpenNOW") - .collapsible(false) - .resizable(false) - .fixed_size([450.0, 280.0]) - .anchor(egui::Align2::CENTER_CENTER, [0.0, 0.0]) - .show(ctx, |ui| { - ui.vertical_centered(|ui| { - ui.add_space(15.0); - - // Logo - ui.label( - egui::RichText::new("OpenNOW") - .size(32.0) - .color(egui::Color32::from_rgb(118, 185, 0)) - .strong(), - ); - - ui.add_space(8.0); - - ui.label( - egui::RichText::new("Open Source GeForce NOW Client") - .size(14.0) - .color(egui::Color32::from_rgb(180, 180, 180)), - ); - - ui.add_space(20.0); - - // Beta warning badge - egui::Frame::new() - .fill(egui::Color32::from_rgb(80, 60, 20)) - .corner_radius(6.0) - .inner_margin(egui::Margin { - left: 14, - right: 14, - top: 6, - bottom: 6, - }) - .show(ui, |ui| { - ui.label( - egui::RichText::new("BETA") - .size(14.0) - .color(egui::Color32::from_rgb(255, 200, 80)) - .strong(), - ); - }); - - ui.add_space(15.0); - - ui.label( - egui::RichText::new("This software is still in beta.") - .size(14.0) - .color(egui::Color32::from_rgb(255, 200, 80)), - ); - - ui.add_space(8.0); - - ui.label( - egui::RichText::new("You may encounter bugs and issues.") - .size(13.0) - .color(egui::Color32::from_rgb(180, 180, 180)), - ); - - ui.add_space(5.0); - - ui.label( - egui::RichText::new("Please report any problems to our GitHub:") - .size(12.0) - .color(egui::Color32::GRAY), - ); - - ui.add_space(3.0); - - ui.hyperlink_to( - egui::RichText::new("github.com/zortos293/OpenNOW") - .size(12.0) - .color(egui::Color32::from_rgb(100, 180, 255)), - "https://github.com/zortos293/OpenNOW", - ); - - ui.add_space(20.0); - - let continue_btn = - egui::Button::new(egui::RichText::new("Continue").size(14.0).strong()) - .fill(egui::Color32::from_rgb(118, 185, 0)) - .min_size(egui::vec2(120.0, 36.0)); - - if ui.add(continue_btn).clicked() { - actions.push(UiAction::CloseWelcomePopup); - } - }); - }); -} diff --git a/opennow-streamer/src/gui/screens/session.rs b/opennow-streamer/src/gui/screens/session.rs deleted file mode 100644 index e980cae..0000000 --- a/opennow-streamer/src/gui/screens/session.rs +++ /dev/null @@ -1,61 +0,0 @@ -//! Session Screen -//! -//! Renders the session loading/connecting screen. - -use crate::app::{GameInfo, UiAction}; - -/// Render the session screen (loading/connecting state) -pub fn render_session_screen( - ctx: &egui::Context, - selected_game: &Option, - status_message: &str, - error_message: &Option, - actions: &mut Vec -) { - egui::CentralPanel::default().show(ctx, |ui| { - ui.vertical_centered(|ui| { - ui.add_space(120.0); - - // Game title - if let Some(ref game) = selected_game { - ui.label( - egui::RichText::new(&game.title) - .size(28.0) - .strong() - .color(egui::Color32::WHITE) - ); - } - - ui.add_space(40.0); - - // Spinner - ui.spinner(); - - ui.add_space(20.0); - - // Status - ui.label( - egui::RichText::new(status_message) - .size(16.0) - .color(egui::Color32::LIGHT_GRAY) - ); - - // Error message - if let Some(ref error) = error_message { - ui.add_space(20.0); - ui.label( - egui::RichText::new(error) - .size(14.0) - .color(egui::Color32::from_rgb(255, 100, 100)) - ); - } - - ui.add_space(40.0); - - // Cancel button - if ui.button("Cancel").clicked() { - actions.push(UiAction::StopStreaming); - } - }); - }); -} diff --git a/opennow-streamer/src/gui/shaders.rs b/opennow-streamer/src/gui/shaders.rs deleted file mode 100644 index fce1a7f..0000000 --- a/opennow-streamer/src/gui/shaders.rs +++ /dev/null @@ -1,328 +0,0 @@ -//! GPU Shaders for video rendering -//! -//! WGSL shaders for YUV to RGB conversion on the GPU. -//! Supports SDR (BT.709) and HDR (BT.2020 + PQ) with automatic tone mapping. -//! -//! HDR Detection: -//! The VideoFrame struct carries color_space and transfer_function metadata. -//! When HDR content is detected (BT.2020 + PQ), the shader will apply: -//! 1. PQ EOTF to convert to linear light -//! 2. BT.2020 to BT.709 color space conversion -//! 3. ACES tone mapping to compress HDR to SDR range -//! 4. SDR gamma for display - -/// WGSL shader for YUV420P format (3 separate planes) -/// Uses BT.709 Limited range conversion (standard for video) -pub const VIDEO_SHADER: &str = r#" -struct VertexOutput { - @builtin(position) position: vec4, - @location(0) tex_coord: vec2, -}; - -@vertex -fn vs_main(@builtin(vertex_index) vertex_index: u32) -> VertexOutput { - // Full-screen quad (2 triangles = 6 vertices) - var positions = array, 6>( - vec2(-1.0, -1.0), // bottom-left - vec2( 1.0, -1.0), // bottom-right - vec2(-1.0, 1.0), // top-left - vec2(-1.0, 1.0), // top-left - vec2( 1.0, -1.0), // bottom-right - vec2( 1.0, 1.0), // top-right - ); - - var tex_coords = array, 6>( - vec2(0.0, 1.0), // bottom-left - vec2(1.0, 1.0), // bottom-right - vec2(0.0, 0.0), // top-left - vec2(0.0, 0.0), // top-left - vec2(1.0, 1.0), // bottom-right - vec2(1.0, 0.0), // top-right - ); - - var output: VertexOutput; - output.position = vec4(positions[vertex_index], 0.0, 1.0); - output.tex_coord = tex_coords[vertex_index]; - return output; -} - -// YUV planar textures (Y = full res, U/V = half res) -@group(0) @binding(0) -var y_texture: texture_2d; -@group(0) @binding(1) -var u_texture: texture_2d; -@group(0) @binding(2) -var v_texture: texture_2d; -@group(0) @binding(3) -var video_sampler: sampler; - -@fragment -fn fs_main(input: VertexOutput) -> @location(0) vec4 { - // Sample Y, U, V planes - let y_raw = textureSample(y_texture, video_sampler, input.tex_coord).r; - let u_raw = textureSample(u_texture, video_sampler, input.tex_coord).r; - let v_raw = textureSample(v_texture, video_sampler, input.tex_coord).r; - - // BT.709 Limited Range (TV range: Y 16-235, UV 16-240) - // Scale from limited to full range - let y = (y_raw - 16.0/255.0) * (255.0/219.0); - let u = (u_raw - 16.0/255.0) * (255.0/224.0) - 0.5; - let v = (v_raw - 16.0/255.0) * (255.0/224.0) - 0.5; - - // BT.709 color matrix - let r = y + 1.5748 * v; - let g = y - 0.1873 * u - 0.4681 * v; - let b = y + 1.8556 * u; - - return vec4(clamp(r, 0.0, 1.0), clamp(g, 0.0, 1.0), clamp(b, 0.0, 1.0), 1.0); -} -"#; - -/// WGSL shader for NV12 format (D3D11 on Windows, VideoToolbox on macOS) -/// Primary GPU path - Y plane (R8) + interleaved UV plane (Rg8) -/// Uses BT.709 Limited range conversion (standard for video) -pub const NV12_SHADER: &str = r#" -struct VertexOutput { - @builtin(position) position: vec4, - @location(0) tex_coord: vec2, -}; - -@vertex -fn vs_main(@builtin(vertex_index) vertex_index: u32) -> VertexOutput { - var positions = array, 6>( - vec2(-1.0, -1.0), - vec2( 1.0, -1.0), - vec2(-1.0, 1.0), - vec2(-1.0, 1.0), - vec2( 1.0, -1.0), - vec2( 1.0, 1.0), - ); - - var tex_coords = array, 6>( - vec2(0.0, 1.0), - vec2(1.0, 1.0), - vec2(0.0, 0.0), - vec2(0.0, 0.0), - vec2(1.0, 1.0), - vec2(1.0, 0.0), - ); - - var output: VertexOutput; - output.position = vec4(positions[vertex_index], 0.0, 1.0); - output.tex_coord = tex_coords[vertex_index]; - return output; -} - -// NV12 textures: Y (R8, full res) and UV (Rg8, half res, interleaved) -@group(0) @binding(0) -var y_texture: texture_2d; -@group(0) @binding(1) -var uv_texture: texture_2d; -@group(0) @binding(2) -var video_sampler: sampler; - -@fragment -fn fs_main(input: VertexOutput) -> @location(0) vec4 { - // Sample Y (full res) and UV (half res, interleaved) - let y_raw = textureSample(y_texture, video_sampler, input.tex_coord).r; - let uv = textureSample(uv_texture, video_sampler, input.tex_coord); - - // NV12 format: U in R channel, V in G channel - let u_raw = uv.r; - let v_raw = uv.g; - - // BT.709 Limited Range (TV range: Y 16-235, UV 16-240) - // Scale from limited to full range - let y = (y_raw - 16.0/255.0) * (255.0/219.0); - let u = (u_raw - 16.0/255.0) * (255.0/224.0) - 0.5; - let v = (v_raw - 16.0/255.0) * (255.0/224.0) - 0.5; - - // BT.709 color matrix - let r = y + 1.5748 * v; - let g = y - 0.1873 * u - 0.4681 * v; - let b = y + 1.8556 * u; - - return vec4(clamp(r, 0.0, 1.0), clamp(g, 0.0, 1.0), clamp(b, 0.0, 1.0), 1.0); -} -"#; - -/// WGSL shader for NV12 HDR content (BT.2020 + PQ to SDR tone mapping) -/// Used when HDR content is detected and display is in SDR mode -pub const NV12_HDR_TONEMAP_SHADER: &str = r#" -struct VertexOutput { - @builtin(position) position: vec4, - @location(0) tex_coord: vec2, -}; - -@vertex -fn vs_main(@builtin(vertex_index) vertex_index: u32) -> VertexOutput { - var positions = array, 6>( - vec2(-1.0, -1.0), - vec2( 1.0, -1.0), - vec2(-1.0, 1.0), - vec2(-1.0, 1.0), - vec2( 1.0, -1.0), - vec2( 1.0, 1.0), - ); - - var tex_coords = array, 6>( - vec2(0.0, 1.0), - vec2(1.0, 1.0), - vec2(0.0, 0.0), - vec2(0.0, 0.0), - vec2(1.0, 1.0), - vec2(1.0, 0.0), - ); - - var output: VertexOutput; - output.position = vec4(positions[vertex_index], 0.0, 1.0); - output.tex_coord = tex_coords[vertex_index]; - return output; -} - -// NV12 textures: Y (R8/R16, full res) and UV (Rg8/Rg16, half res, interleaved) -@group(0) @binding(0) -var y_texture: texture_2d; -@group(0) @binding(1) -var uv_texture: texture_2d; -@group(0) @binding(2) -var video_sampler: sampler; - -// PQ EOTF (SMPTE ST 2084) - converts PQ signal to linear light (nits) -fn pq_eotf(pq: vec3) -> vec3 { - let m1 = 0.1593017578125; // 2610/16384 - let m2 = 78.84375; // 2523/32 * 128 - let c1 = 0.8359375; // 3424/4096 - let c2 = 18.8515625; // 2413/128 - let c3 = 18.6875; // 2392/128 - - let pq_pow = pow(max(pq, vec3(0.0)), vec3(1.0 / m2)); - let num = max(pq_pow - c1, vec3(0.0)); - let den = c2 - c3 * pq_pow; - - // Output is in units of 10000 nits (PQ reference white) - return pow(num / max(den, vec3(0.0001)), vec3(1.0 / m1)) * 10000.0; -} - -// BT.2020 to BT.709 color gamut conversion -fn bt2020_to_bt709(color: vec3) -> vec3 { - // 3x3 matrix for BT.2020 to BT.709 conversion - let r = color.r * 1.6605 + color.g * -0.5876 + color.b * -0.0728; - let g = color.r * -0.1246 + color.g * 1.1329 + color.b * -0.0083; - let b = color.r * -0.0182 + color.g * -0.1006 + color.b * 1.1187; - return vec3(r, g, b); -} - -// ACES-inspired filmic tone mapping -fn tonemap_aces(hdr: vec3) -> vec3 { - // Normalize from nits to [0,1] range - // Assume 203 nits as SDR reference white (BT.2408) - let x = hdr / 203.0; - - // ACES filmic curve approximation (Narkowicz 2015) - let a = 2.51; - let b = 0.03; - let c = 2.43; - let d = 0.59; - let e = 0.14; - return clamp((x * (a * x + b)) / (x * (c * x + d) + e), vec3(0.0), vec3(1.0)); -} - -// Apply SDR gamma curve (sRGB approximation) -fn linear_to_srgb(linear: vec3) -> vec3 { - let cutoff = vec3(0.0031308); - let low = linear * 12.92; - let high = 1.055 * pow(linear, vec3(1.0/2.4)) - 0.055; - return select(high, low, linear <= cutoff); -} - -@fragment -fn fs_main(input: VertexOutput) -> @location(0) vec4 { - // Sample Y (full res) and UV (half res, interleaved) - let y_raw = textureSample(y_texture, video_sampler, input.tex_coord).r; - let uv = textureSample(uv_texture, video_sampler, input.tex_coord); - - // NV12 format: U in R channel, V in G channel - let u_raw = uv.r; - let v_raw = uv.g; - - // BT.2020 Limited Range (TV range: Y 64-940, UV 64-960 for 10-bit) - // For 8-bit textures, this is normalized to 0-1 range - // Scale from limited to full range - let y = (y_raw - 16.0/255.0) * (255.0/219.0); - let u = (u_raw - 16.0/255.0) * (255.0/224.0) - 0.5; - let v = (v_raw - 16.0/255.0) * (255.0/224.0) - 0.5; - - // BT.2020 YCbCr to RGB matrix (non-constant luminance) - let r = y + 1.4746 * v; - let g = y - 0.1646 * u - 0.5714 * v; - let b = y + 1.8814 * u; - - var rgb = vec3(clamp(r, 0.0, 1.0), clamp(g, 0.0, 1.0), clamp(b, 0.0, 1.0)); - - // Apply PQ EOTF to get linear light in nits - let linear_hdr = pq_eotf(rgb); - - // Convert from BT.2020 to BT.709 color gamut - let bt709_linear = bt2020_to_bt709(linear_hdr); - - // Tone map HDR to SDR range - let tonemapped = tonemap_aces(max(bt709_linear, vec3(0.0))); - - // Apply sRGB gamma for SDR display - let sdr = linear_to_srgb(tonemapped); - - return vec4(sdr, 1.0); -} -"#; - -/// WGSL shader for ExternalTexture (wgpu 28+ zero-copy video) -/// Uses texture_external which provides hardware-accelerated YUV->RGB conversion -/// This is the fastest path - no manual color conversion needed -pub const EXTERNAL_TEXTURE_SHADER: &str = r#" -struct VertexOutput { - @builtin(position) position: vec4, - @location(0) tex_coord: vec2, -}; - -@vertex -fn vs_main(@builtin(vertex_index) vertex_index: u32) -> VertexOutput { - var positions = array, 6>( - vec2(-1.0, -1.0), - vec2( 1.0, -1.0), - vec2(-1.0, 1.0), - vec2(-1.0, 1.0), - vec2( 1.0, -1.0), - vec2( 1.0, 1.0), - ); - - var tex_coords = array, 6>( - vec2(0.0, 1.0), - vec2(1.0, 1.0), - vec2(0.0, 0.0), - vec2(0.0, 0.0), - vec2(1.0, 1.0), - vec2(1.0, 0.0), - ); - - var output: VertexOutput; - output.position = vec4(positions[vertex_index], 0.0, 1.0); - output.tex_coord = tex_coords[vertex_index]; - return output; -} - -// External texture - hardware-accelerated YUV->RGB conversion -@group(0) @binding(0) -var video_texture: texture_external; - -// Sampler for external texture -@group(0) @binding(1) -var video_sampler: sampler; - -@fragment -fn fs_main(input: VertexOutput) -> @location(0) vec4 { - // textureSampleBaseClampToEdge automatically converts YUV to RGB - // using the color space information from the ExternalTexture descriptor - return textureSampleBaseClampToEdge(video_texture, video_sampler, input.tex_coord); -} -"#; diff --git a/opennow-streamer/src/gui/stats_panel.rs b/opennow-streamer/src/gui/stats_panel.rs deleted file mode 100644 index f7cf0ad..0000000 --- a/opennow-streamer/src/gui/stats_panel.rs +++ /dev/null @@ -1,248 +0,0 @@ -//! Stats Panel Overlay -//! -//! Bottom-left stats display matching the web client style. -//! Includes throttling to reduce CPU usage - stats update every 200ms instead of every frame. - -use egui::{Align2, Color32, FontId, RichText}; -use crate::media::StreamStats; -use crate::app::StatsPosition; -use std::time::{Duration, Instant}; - -/// Interval between stats updates (200ms = 5 updates per second) -/// This dramatically reduces CPU usage while still providing responsive feedback -const STATS_UPDATE_INTERVAL: Duration = Duration::from_millis(200); - -/// Stats panel overlay with throttled updates -pub struct StatsPanel { - pub visible: bool, - pub position: StatsPosition, - /// Cached stats for throttled rendering - cached_stats: Option, - /// Last time stats were updated - last_update: Instant, -} - -impl StatsPanel { - pub fn new() -> Self { - Self { - visible: true, - position: StatsPosition::BottomLeft, - cached_stats: None, - last_update: Instant::now(), - } - } - - /// Check if stats need to be updated (throttled to STATS_UPDATE_INTERVAL) - fn should_update(&self) -> bool { - self.last_update.elapsed() >= STATS_UPDATE_INTERVAL - } - - /// Update cached stats if the throttle interval has passed - /// Returns true if stats were updated (UI needs repaint) - pub fn update_stats(&mut self, stats: &StreamStats) -> bool { - if self.should_update() { - self.cached_stats = Some(stats.clone()); - self.last_update = Instant::now(); - true - } else { - false - } - } - - /// Render the stats panel using cached stats - /// This avoids recalculating the display every frame - pub fn render(&self, ctx: &egui::Context, stats: &StreamStats) { - if !self.visible { - return; - } - - // Use cached stats if available, otherwise use provided stats - let display_stats = self.cached_stats.as_ref().unwrap_or(stats); - - let (anchor, offset) = match self.position { - StatsPosition::BottomLeft => (Align2::LEFT_BOTTOM, [10.0, -10.0]), - StatsPosition::BottomRight => (Align2::RIGHT_BOTTOM, [-10.0, -10.0]), - StatsPosition::TopLeft => (Align2::LEFT_TOP, [10.0, 10.0]), - StatsPosition::TopRight => (Align2::RIGHT_TOP, [-10.0, 10.0]), - }; - - egui::Area::new(egui::Id::new("stats_panel")) - .anchor(anchor, offset) - .interactable(false) - .show(ctx, |ui| { - egui::Frame::new() - .fill(Color32::from_rgba_unmultiplied(0, 0, 0, 200)) - .corner_radius(4.0) - .inner_margin(8.0) - .show(ui, |ui| { - ui.set_min_width(200.0); - - // Resolution and FPS - let res_text = if display_stats.resolution.is_empty() { - "Connecting...".to_string() - } else { - format!("{} @ {} fps", display_stats.resolution, display_stats.fps as u32) - }; - - ui.label( - RichText::new(res_text) - .font(FontId::monospace(13.0)) - .color(Color32::WHITE) - ); - - // Codec, HDR status, and bitrate - if !display_stats.codec.is_empty() { - let hdr_indicator = if display_stats.is_hdr { - " • HDR" - } else { - "" - }; - let hdr_color = if display_stats.is_hdr { - Color32::from_rgb(255, 180, 0) // Orange/gold for HDR - } else { - Color32::LIGHT_GRAY - }; - - ui.horizontal(|ui| { - ui.label( - RichText::new(format!( - "{} • {:.1} Mbps", - display_stats.codec, - display_stats.bitrate_mbps - )) - .font(FontId::monospace(11.0)) - .color(Color32::LIGHT_GRAY) - ); - if display_stats.is_hdr { - ui.label( - RichText::new("HDR") - .font(FontId::monospace(11.0)) - .color(hdr_color) - ); - } - }); - } - - // Network RTT (round-trip time) - if display_stats.rtt_ms > 0.0 { - let rtt_color = if display_stats.rtt_ms < 30.0 { - Color32::GREEN - } else if display_stats.rtt_ms < 60.0 { - Color32::YELLOW - } else { - Color32::RED - }; - - ui.label( - RichText::new(format!("RTT: {:.0}ms", display_stats.rtt_ms)) - .font(FontId::monospace(11.0)) - .color(rtt_color) - ); - } else { - ui.label( - RichText::new("RTT: N/A") - .font(FontId::monospace(11.0)) - .color(Color32::GRAY) - ); - } - - // Packet loss - if display_stats.packet_loss > 0.1 { - let loss_color = if display_stats.packet_loss < 1.0 { - Color32::YELLOW - } else { - Color32::RED - }; - - ui.label( - RichText::new(format!( - "Packet Loss: {:.2}%", - display_stats.packet_loss - )) - .font(FontId::monospace(11.0)) - .color(loss_color) - ); - } - - // Decode, render, and input latency - if display_stats.decode_time_ms > 0.0 || display_stats.render_time_ms > 0.0 { - ui.label( - RichText::new(format!( - "Decode: {:.1}ms • Render: {:.1}ms", - display_stats.decode_time_ms, - display_stats.render_time_ms - )) - .font(FontId::monospace(10.0)) - .color(Color32::GRAY) - ); - } - - // Input latency (client-side only) - if display_stats.input_latency_ms > 0.0 { - let input_color = if display_stats.input_latency_ms < 5.0 { - Color32::GREEN - } else if display_stats.input_latency_ms < 10.0 { - Color32::YELLOW - } else { - Color32::RED - }; - - ui.label( - RichText::new(format!( - "Input: {:.1}ms", - display_stats.input_latency_ms - )) - .font(FontId::monospace(10.0)) - .color(input_color) - ); - } - - // Frame stats - if display_stats.frames_received > 0 { - ui.label( - RichText::new(format!( - "Frames: {} rx, {} dec, {} drop", - display_stats.frames_received, - display_stats.frames_decoded, - display_stats.frames_dropped - )) - .font(FontId::monospace(10.0)) - .color(Color32::DARK_GRAY) - ); - } - - // GPU and server info - if !display_stats.gpu_type.is_empty() || !display_stats.server_region.is_empty() { - let info = format!( - "{}{}{}", - display_stats.gpu_type, - if !display_stats.gpu_type.is_empty() && !display_stats.server_region.is_empty() { " • " } else { "" }, - display_stats.server_region - ); - - ui.label( - RichText::new(info) - .font(FontId::monospace(10.0)) - .color(Color32::DARK_GRAY) - ); - } - }); - }); - } - - /// Toggle visibility - pub fn toggle(&mut self) { - self.visible = !self.visible; - } - - /// Set position - pub fn set_position(&mut self, position: StatsPosition) { - self.position = position; - } -} - -impl Default for StatsPanel { - fn default() -> Self { - Self::new() - } -} diff --git a/opennow-streamer/src/input/controller.rs b/opennow-streamer/src/input/controller.rs deleted file mode 100644 index b756edb..0000000 --- a/opennow-streamer/src/input/controller.rs +++ /dev/null @@ -1,733 +0,0 @@ -use gilrs::{Axis, Button, Event, EventType, GamepadId, GilrsBuilder}; -use log::{debug, error, info, trace, warn}; -use parking_lot::Mutex; -use std::collections::HashMap; -use std::sync::{ - atomic::{AtomicBool, Ordering}, - Arc, -}; -use std::time::Duration; -use tokio::sync::mpsc; - -use super::get_timestamp_us; -use crate::webrtc::InputEvent; - -/// Pending rumble effect to be applied -#[derive(Debug, Clone)] -pub struct RumbleEffect { - /// Left motor intensity (0-255, low frequency / strong) - pub left_motor: u8, - /// Right motor intensity (0-255, high frequency / weak) - pub right_motor: u8, - /// Duration in milliseconds (0 = stop) - pub duration_ms: u16, - /// When this effect was queued - pub queued_at: std::time::Instant, -} - -impl RumbleEffect { - pub fn new(left_motor: u8, right_motor: u8, duration_ms: u16) -> Self { - Self { - left_motor, - right_motor, - duration_ms, - queued_at: std::time::Instant::now(), - } - } - - /// Check if this is a stop command - pub fn is_stop(&self) -> bool { - self.left_motor == 0 && self.right_motor == 0 - } -} - -/// XInput button format (confirmed from web client analysis) -/// This is the standard XInput wButtons format used by GFN: -/// -/// 0x0001 = DPad Up -/// 0x0002 = DPad Down -/// 0x0004 = DPad Left -/// 0x0008 = DPad Right -/// 0x0010 = Start -/// 0x0020 = Back/Select -/// 0x0040 = L3 (Left Stick Click) -/// 0x0080 = R3 (Right Stick Click) -/// 0x0100 = LB (Left Bumper) -/// 0x0200 = RB (Right Bumper) -/// 0x1000 = A -/// 0x2000 = B -/// 0x4000 = X -/// 0x8000 = Y -const XINPUT_DPAD_UP: u16 = 0x0001; -const XINPUT_DPAD_DOWN: u16 = 0x0002; -const XINPUT_DPAD_LEFT: u16 = 0x0004; -const XINPUT_DPAD_RIGHT: u16 = 0x0008; -const XINPUT_START: u16 = 0x0010; -const XINPUT_BACK: u16 = 0x0020; -const XINPUT_L3: u16 = 0x0040; -const XINPUT_R3: u16 = 0x0080; -const XINPUT_LB: u16 = 0x0100; -const XINPUT_RB: u16 = 0x0200; -const XINPUT_A: u16 = 0x1000; -const XINPUT_B: u16 = 0x2000; -const XINPUT_X: u16 = 0x4000; -const XINPUT_Y: u16 = 0x8000; - -/// Deadzone for analog sticks (15% as per GFN docs) -const STICK_DEADZONE: f32 = 0.15; - -/// Controller manager to handle gamepad input and rumble feedback -pub struct ControllerManager { - running: Arc, - event_tx: Mutex>>, - /// Pending rumble effects per controller (keyed by controller ID) - rumble_queue: Arc>>, - /// Active rumble effects with expiry times - active_rumble: Arc>>, -} - -impl ControllerManager { - pub fn new() -> Self { - Self { - running: Arc::new(AtomicBool::new(false)), - event_tx: Mutex::new(None), - rumble_queue: Arc::new(Mutex::new(HashMap::new())), - active_rumble: Arc::new(Mutex::new(HashMap::new())), - } - } - - /// Set the input event sender - pub fn set_event_sender(&self, tx: mpsc::Sender) { - *self.event_tx.lock() = Some(tx); - } - - /// Start the controller input loop - pub fn start(&self) { - if self.running.load(Ordering::SeqCst) { - return; - } - - self.running.store(true, Ordering::SeqCst); - let running = self.running.clone(); - - let tx_opt = self.event_tx.lock().clone(); - - if tx_opt.is_none() { - warn!("ControllerManager started without event sender!"); - return; - } - let tx = tx_opt.unwrap(); - - std::thread::spawn(move || { - info!("Controller input thread starting..."); - - // Initialize gilrs WITHOUT built-in axis filtering - // This gives us raw axis values so our radial deadzone works correctly - // on all controller types (Xbox, PS5, etc.) - let mut gilrs = match GilrsBuilder::new() - .with_default_filters(false) // Disable all default filters - .set_axis_to_btn(0.5, 0.4) // Only used for D-pad on some controllers - .build() - { - Ok(g) => { - info!("gilrs initialized (raw mode - no built-in filtering)"); - g - } - Err(e) => { - error!("Failed to initialize gilrs: {}", e); - return; - } - }; - - // Report connected gamepads and detect racing wheels - // Racing wheels need special axis mapping (wheel rotation, pedals) - let mut gamepad_count = 0; - let mut excluded_devices: Vec = Vec::new(); - let mut wheel_devices: Vec = Vec::new(); - - for (id, gamepad) in gilrs.gamepads() { - let name = gamepad.name().to_lowercase(); - - // Detect and EXCLUDE racing wheels for now - // TODO: Racing wheel support is disabled until axis mapping is finalized - let is_logitech = name.contains("logitech"); - let is_wheel = name.contains("g29") - || name.contains("g27") - || name.contains("g920") - || name.contains("g923") - || name.contains("g25") - || name.contains("driving force") - || name.contains("racing wheel") - || name.contains("fanatec") - || name.contains("thrustmaster") - || name.contains("t150") - || name.contains("t300") - || name.contains("t500") - || (is_logitech && (name.contains("steering") || name.contains("pedal"))); - - if is_wheel { - info!( - "Racing wheel excluded (support disabled): '{}' (id={})", - gamepad.name(), - id - ); - excluded_devices.push(id); - continue; - } - - gamepad_count += 1; - info!( - "Gamepad {} detected: '{}' (UUID: {:?})", - id, - gamepad.name(), - gamepad.uuid() - ); - - // Log supported features - debug!(" Power info: {:?}", gamepad.power_info()); - debug!(" Is connected: {}", gamepad.is_connected()); - } - - if gamepad_count == 0 && wheel_devices.is_empty() { - warn!( - "No gamepads or wheels detected at startup. Connect a controller to use input." - ); - } else { - if gamepad_count > 0 { - info!("Found {} gamepad(s)", gamepad_count); - } - if !wheel_devices.is_empty() { - info!( - "Found {} racing wheel(s) - using wheel axis mapping", - wheel_devices.len() - ); - } - } - - let mut last_button_flags: u16 = 0; - let mut event_count: u64 = 0; - - while running.load(Ordering::Relaxed) { - // Poll events - while let Some(Event { - id, event, time, .. - }) = gilrs.next_event() - { - // Skip events from excluded devices - if excluded_devices.contains(&id) { - continue; - } - - let gamepad = gilrs.gamepad(id); - let is_wheel_device = wheel_devices.contains(&id); - event_count += 1; - - // Log first few events for debugging - if event_count <= 10 { - debug!( - "Controller event #{}: {:?} from '{}' at {:?}{}", - event_count, - event, - gamepad.name(), - time, - if is_wheel_device { " [WHEEL]" } else { "" } - ); - } - - // Use gamepad index as controller ID (0-3) - // GamepadId is opaque, but we can use usize conversion - let controller_id: u8 = usize::from(id) as u8; - - match event { - EventType::Connected => { - // Check if newly connected device is a wheel (excluded) - let name = gamepad.name().to_lowercase(); - let is_logitech = name.contains("logitech"); - let is_wheel = name.contains("g29") - || name.contains("g27") - || name.contains("g920") - || name.contains("g923") - || name.contains("g25") - || name.contains("driving force") - || name.contains("racing wheel") - || name.contains("fanatec") - || name.contains("thrustmaster") - || name.contains("t150") - || name.contains("t300") - || name.contains("t500") - || (is_logitech - && (name.contains("steering") || name.contains("pedal"))); - - if is_wheel { - info!( - "Racing wheel connected (excluded): {} (id={})", - gamepad.name(), - controller_id - ); - excluded_devices.push(id); - } else { - info!( - "Gamepad connected: {} (id={})", - gamepad.name(), - controller_id - ); - } - } - EventType::Disconnected => { - // Remove from wheel/excluded lists if it was there - excluded_devices.retain(|&x| x != id); - wheel_devices.retain(|&x| x != id); - info!( - "Device disconnected: {} (id={})", - gamepad.name(), - controller_id - ); - } - _ => { - let ( - button_flags, - left_trigger, - right_trigger, - left_stick_x, - left_stick_y, - right_stick_x, - right_stick_y, - ) = if is_wheel_device { - // RACING WHEEL MAPPING (G29/G27/G920/etc) - // Debug: Read all axes to understand the mapping - let lsx = gamepad.value(Axis::LeftStickX); - let lsy = gamepad.value(Axis::LeftStickY); - let rsx = gamepad.value(Axis::RightStickX); - let rsy = gamepad.value(Axis::RightStickY); - let lz = gamepad.value(Axis::LeftZ); - let rz = gamepad.value(Axis::RightZ); - - // Log axis values periodically - static DEBUG_COUNTER: std::sync::atomic::AtomicU64 = - std::sync::atomic::AtomicU64::new(0); - let count = DEBUG_COUNTER - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - if count % 500 == 0 { - info!( - "G29 axes: LSX={:.3} LSY={:.3} RSX={:.3} RSY={:.3} LZ={:.3} RZ={:.3}", - lsx, lsy, rsx, rsy, lz, rz - ); - } - - let mut flags: u16 = 0; - - // D-Pad - enable for G29 - if gamepad.is_pressed(Button::DPadUp) { - flags |= XINPUT_DPAD_UP; - } - if gamepad.is_pressed(Button::DPadDown) { - flags |= XINPUT_DPAD_DOWN; - } - if gamepad.is_pressed(Button::DPadLeft) { - flags |= XINPUT_DPAD_LEFT; - } - if gamepad.is_pressed(Button::DPadRight) { - flags |= XINPUT_DPAD_RIGHT; - } - - // Face buttons - if gamepad.is_pressed(Button::South) { - flags |= XINPUT_A; - } - if gamepad.is_pressed(Button::East) { - flags |= XINPUT_B; - } - if gamepad.is_pressed(Button::West) { - flags |= XINPUT_X; - } - if gamepad.is_pressed(Button::North) { - flags |= XINPUT_Y; - } - - // Paddle shifters -> LB/RB - if gamepad.is_pressed(Button::LeftTrigger) { - flags |= XINPUT_LB; - } - if gamepad.is_pressed(Button::RightTrigger) { - flags |= XINPUT_RB; - } - - // Start/Select - if gamepad.is_pressed(Button::Start) { - flags |= XINPUT_START; - } - if gamepad.is_pressed(Button::Select) { - flags |= XINPUT_BACK; - } - - // Wheel rotation -> Left Stick X - let wheel_x = (lsx * 32767.0).clamp(-32768.0, 32767.0) as i16; - - // G29 Pedals: The axes report 1.0 when released, -1.0 when fully pressed - // We need to convert: 1.0 (released) -> 0, -1.0 (pressed) -> 255 - - // Gas pedal (try multiple axes) - let gas = { - // Try RightZ first, then RightStickY, then button data - let axis_val = if rz.abs() > 0.01 { - rz - } else if rsy.abs() > 0.01 { - rsy - } else { - 1.0 - }; // Default to released - - // Convert: 1.0 -> 0, -1.0 -> 255 - let normalized = ((1.0 - axis_val) / 2.0).clamp(0.0, 1.0); - (normalized * 255.0) as u8 - }; - - // Brake pedal - let brake = { - let axis_val = if lz.abs() > 0.01 { - lz - } else if lsy.abs() > 0.01 { - lsy - } else { - 1.0 - }; - let normalized = ((1.0 - axis_val) / 2.0).clamp(0.0, 1.0); - (normalized * 255.0) as u8 - }; - - // Clutch (usually on a separate axis) - let clutch_y = { - let axis_val = if rsx.abs() > 0.01 { rsx } else { 1.0 }; - let normalized = ((1.0 - axis_val) / 2.0).clamp(0.0, 1.0); - (normalized * 32767.0) as i16 - }; - - // Log when buttons pressed - if flags != 0 { - debug!("G29 buttons: 0x{:04X}", flags); - } - - (flags, brake, gas, wheel_x, clutch_y, 0i16, 0i16) - } else { - // STANDARD GAMEPAD MAPPING - let mut flags: u16 = 0; - - // D-Pad (bits 0-3) - if gamepad.is_pressed(Button::DPadUp) { - flags |= XINPUT_DPAD_UP; - } - if gamepad.is_pressed(Button::DPadDown) { - flags |= XINPUT_DPAD_DOWN; - } - if gamepad.is_pressed(Button::DPadLeft) { - flags |= XINPUT_DPAD_LEFT; - } - if gamepad.is_pressed(Button::DPadRight) { - flags |= XINPUT_DPAD_RIGHT; - } - - // Center buttons (bits 4-5) - if gamepad.is_pressed(Button::Start) { - flags |= XINPUT_START; - } - if gamepad.is_pressed(Button::Select) { - flags |= XINPUT_BACK; - } - - // Stick clicks (bits 6-7) - if gamepad.is_pressed(Button::LeftThumb) { - flags |= XINPUT_L3; - } - if gamepad.is_pressed(Button::RightThumb) { - flags |= XINPUT_R3; - } - - // Shoulder buttons / bumpers (bits 8-9) - if gamepad.is_pressed(Button::LeftTrigger) { - flags |= XINPUT_LB; - } - if gamepad.is_pressed(Button::RightTrigger) { - flags |= XINPUT_RB; - } - - // Face buttons (bits 12-15) - if gamepad.is_pressed(Button::South) { - flags |= XINPUT_A; - } - if gamepad.is_pressed(Button::East) { - flags |= XINPUT_B; - } - if gamepad.is_pressed(Button::West) { - flags |= XINPUT_X; - } - if gamepad.is_pressed(Button::North) { - flags |= XINPUT_Y; - } - - // Analog triggers (0-255) - let get_trigger_value = |button: Button, axis: Axis| -> u8 { - if let Some(data) = gamepad.button_data(button) { - let val = data.value(); - if val > 0.01 { - return (val.clamp(0.0, 1.0) * 255.0) as u8; - } - } - let axis_val = gamepad.value(axis); - if axis_val.abs() > 0.01 { - let normalized = if axis_val < -0.5 { - (axis_val + 1.0) / 2.0 - } else { - axis_val - }; - let result = (normalized.clamp(0.0, 1.0) * 255.0) as u8; - if result > 0 { - return result; - } - } - if gamepad.is_pressed(button) { - return 255u8; - } - 0u8 - }; - - let lt = get_trigger_value(Button::LeftTrigger2, Axis::LeftZ); - let rt = get_trigger_value(Button::RightTrigger2, Axis::RightZ); - - // Analog sticks - let lx_val = gamepad.value(Axis::LeftStickX); - let ly_val = gamepad.value(Axis::LeftStickY); - let rx_val = gamepad.value(Axis::RightStickX); - let ry_val = gamepad.value(Axis::RightStickY); - - // Apply RADIAL deadzone - let apply_radial_deadzone = |x: f32, y: f32| -> (f32, f32) { - let magnitude = (x * x + y * y).sqrt(); - if magnitude < STICK_DEADZONE { - (0.0, 0.0) - } else { - let scale = (magnitude - STICK_DEADZONE) - / (1.0 - STICK_DEADZONE) - / magnitude; - (x * scale, y * scale) - } - }; - - let (lx, ly) = apply_radial_deadzone(lx_val, ly_val); - let (rx, ry) = apply_radial_deadzone(rx_val, ry_val); - - let lsx = (lx * 32767.0).clamp(-32768.0, 32767.0) as i16; - let lsy = (ly * 32767.0).clamp(-32768.0, 32767.0) as i16; - let rsx = (rx * 32767.0).clamp(-32768.0, 32767.0) as i16; - let rsy = (ry * 32767.0).clamp(-32768.0, 32767.0) as i16; - - (flags, lt, rt, lsx, lsy, rsx, rsy) - }; - - // Log button changes - if button_flags != last_button_flags { - debug!( - "Button state changed: 0x{:04X} -> 0x{:04X}", - last_button_flags, button_flags - ); - last_button_flags = button_flags; - } - - // Log stick movement occasionally - if left_stick_x != 0 - || left_stick_y != 0 - || right_stick_x != 0 - || right_stick_y != 0 - { - trace!( - "Sticks: L({}, {}) R({}, {}) Triggers: L={} R={}", - left_stick_x, - left_stick_y, - right_stick_x, - right_stick_y, - left_trigger, - right_trigger - ); - } - - let event = InputEvent::Gamepad { - controller_id, - button_flags, - left_trigger, - right_trigger, - left_stick_x, - left_stick_y, - right_stick_x, - right_stick_y, - flags: 1, // 1 = controller connected - timestamp_us: get_timestamp_us(), - }; - - // Send event - if let Err(e) = tx.try_send(event) { - trace!("Controller event channel full: {:?}", e); - } - } - } - } - - // Poll sleep - 1ms for 1000Hz polling rate (low latency) - std::thread::sleep(Duration::from_millis(1)); - } - - info!( - "Controller input thread stopped (processed {} events)", - event_count - ); - }); - } - - /// Queue a rumble effect for a controller - /// The effect will be applied on the next polling cycle - pub fn queue_rumble( - &self, - controller_id: u8, - left_motor: u8, - right_motor: u8, - duration_ms: u16, - ) { - let effect = RumbleEffect::new(left_motor, right_motor, duration_ms); - - debug!( - "Queuing rumble for controller {}: left={}, right={}, duration={}ms", - controller_id, left_motor, right_motor, duration_ms - ); - - self.rumble_queue.lock().insert(controller_id, effect); - } - - /// Stop rumble on a specific controller - pub fn stop_rumble(&self, controller_id: u8) { - self.queue_rumble(controller_id, 0, 0, 0); - } - - /// Stop rumble on all controllers - pub fn stop_all_rumble(&self) { - let mut queue = self.rumble_queue.lock(); - let mut active = self.active_rumble.lock(); - - // Queue stop commands for all active controllers - for controller_id in active.keys() { - queue.insert(*controller_id, RumbleEffect::new(0, 0, 0)); - } - active.clear(); - - debug!("Stopped all controller rumble"); - } - - /// Apply pending rumble effects (called from gilrs context) - /// Note: gilrs rumble support is limited on some platforms - /// This method is designed to be extended with platform-specific backends - #[cfg(target_os = "windows")] - fn apply_rumble_effects(&self, gilrs: &mut gilrs::Gilrs) { - let mut queue = self.rumble_queue.lock(); - let mut active = self.active_rumble.lock(); - let now = std::time::Instant::now(); - - // Check for expired active effects - active.retain(|controller_id, expiry| { - if now >= *expiry { - debug!("Rumble expired for controller {}", controller_id); - false - } else { - true - } - }); - - // Apply new effects from queue - for (controller_id, effect) in queue.drain() { - if effect.is_stop() { - active.remove(&controller_id); - // Note: gilrs doesn't have a direct "stop rumble" API - // We'd need platform-specific code here - debug!("Stopping rumble for controller {}", controller_id); - } else { - // Calculate expiry time - let expiry = now + Duration::from_millis(effect.duration_ms as u64); - active.insert(controller_id, expiry); - - // Try to apply via gilrs force feedback (limited support) - // For full support, we'd need XInput directly on Windows - debug!( - "Applying rumble to controller {}: L={}, R={} for {}ms", - controller_id, effect.left_motor, effect.right_motor, effect.duration_ms - ); - - // gilrs force feedback is experimental and not widely supported - // For now, log the attempt - full implementation would use XInput - for (id, gamepad) in gilrs.gamepads() { - if usize::from(id) as u8 == controller_id { - if gamepad.is_ff_supported() { - info!( - "Controller {} supports force feedback via gilrs", - controller_id - ); - // gilrs FF would go here - but it's limited - } - break; - } - } - } - } - } - - /// Apply pending rumble effects (non-Windows fallback) - #[cfg(not(target_os = "windows"))] - fn apply_rumble_effects(&self, gilrs: &mut gilrs::Gilrs) { - let mut queue = self.rumble_queue.lock(); - let mut active = self.active_rumble.lock(); - let now = std::time::Instant::now(); - - // Check for expired active effects - active.retain(|controller_id, expiry| { - if now >= *expiry { - debug!("Rumble expired for controller {}", controller_id); - false - } else { - true - } - }); - - // Apply new effects from queue - for (controller_id, effect) in queue.drain() { - if effect.is_stop() { - active.remove(&controller_id); - } else { - let expiry = now + Duration::from_millis(effect.duration_ms as u64); - active.insert(controller_id, expiry); - - // Try gilrs force feedback on Linux (evdev based) - for (id, gamepad) in gilrs.gamepads() { - if usize::from(id) as u8 == controller_id { - if gamepad.is_ff_supported() { - info!("Controller {} supports force feedback", controller_id); - // Linux FF via evdev would go here - } - break; - } - } - } - } - } - - /// Check if any rumble is currently active - pub fn is_rumble_active(&self) -> bool { - !self.active_rumble.lock().is_empty() - } - - /// Stop the controller input loop - pub fn stop(&self) { - self.stop_all_rumble(); - self.running.store(false, Ordering::SeqCst); - } -} - -impl Default for ControllerManager { - fn default() -> Self { - Self::new() - } -} diff --git a/opennow-streamer/src/input/linux.rs b/opennow-streamer/src/input/linux.rs deleted file mode 100644 index 0b93fc7..0000000 --- a/opennow-streamer/src/input/linux.rs +++ /dev/null @@ -1,690 +0,0 @@ -//! Linux Raw Input API -//! -//! Provides hardware-level mouse input using evdev (direct device access). -//! Captures mouse deltas directly from input devices for responsive input without -//! desktop acceleration effects. Falls back to X11 XInput2 for Wayland compatibility. -//! -//! Events are coalesced (batched) every 2ms like the official GFN client. -//! -//! Key optimizations: -//! - Lock-free event accumulation using atomics -//! - Local cursor tracking for instant visual feedback -//! - Direct evdev access for lowest latency (requires input group membership) -//! - X11 XInput2 fallback for unprivileged access (requires x11-input feature) - -use log::{debug, error, info, warn}; -use parking_lot::Mutex; -use std::path::Path; -use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU64, Ordering}; -use tokio::sync::mpsc; - -use crate::input::{get_timestamp_us, session_elapsed_us, MOUSE_COALESCE_INTERVAL_US}; -use crate::webrtc::InputEvent; - -// evdev bindings -use evdev::{Device, InputEventKind, RelativeAxisType}; - -// X11 bindings for fallback (optional feature) -#[cfg(feature = "x11-input")] -use std::ffi::CString; -#[cfg(feature = "x11-input")] -use x11::xinput2 as xi2; -#[cfg(feature = "x11-input")] -use x11::xlib; - -// Static state -static RAW_INPUT_REGISTERED: AtomicBool = AtomicBool::new(false); -static RAW_INPUT_ACTIVE: AtomicBool = AtomicBool::new(false); -static ACCUMULATED_DX: AtomicI32 = AtomicI32::new(0); -static ACCUMULATED_DY: AtomicI32 = AtomicI32::new(0); -static STOP_REQUESTED: AtomicBool = AtomicBool::new(false); - -// Coalescing state - accumulates events for 2ms batches (like official GFN client) -static COALESCE_DX: AtomicI32 = AtomicI32::new(0); -static COALESCE_DY: AtomicI32 = AtomicI32::new(0); -static COALESCE_LAST_SEND_US: AtomicU64 = AtomicU64::new(0); -static COALESCED_EVENT_COUNT: AtomicU64 = AtomicU64::new(0); - -// Local cursor tracking for instant visual feedback (updated on every event) -static LOCAL_CURSOR_X: AtomicI32 = AtomicI32::new(960); -static LOCAL_CURSOR_Y: AtomicI32 = AtomicI32::new(540); -static LOCAL_CURSOR_WIDTH: AtomicI32 = AtomicI32::new(1920); -static LOCAL_CURSOR_HEIGHT: AtomicI32 = AtomicI32::new(1080); - -// Direct event sender for immediate mouse events -static EVENT_SENDER: Mutex>> = Mutex::new(None); - -// Input backend type -#[derive(Debug, Clone, Copy, PartialEq)] -enum InputBackend { - Evdev, - #[cfg(feature = "x11-input")] - X11, - None, -} - -static ACTIVE_BACKEND: Mutex = Mutex::new(InputBackend::None); - -/// Flush coalesced mouse events - sends accumulated deltas if any -#[inline] -fn flush_coalesced_events() { - let dx = COALESCE_DX.swap(0, Ordering::AcqRel); - let dy = COALESCE_DY.swap(0, Ordering::AcqRel); - - if dx != 0 || dy != 0 { - let timestamp_us = get_timestamp_us(); - let now_us = session_elapsed_us(); - COALESCE_LAST_SEND_US.store(now_us, Ordering::Release); - - let guard = EVENT_SENDER.lock(); - if let Some(ref sender) = *guard { - let _ = sender.try_send(InputEvent::MouseMove { - dx: dx as i16, - dy: dy as i16, - timestamp_us, - }); - } - } -} - -/// Process mouse delta from any backend -#[inline] -fn process_mouse_delta(dx: i32, dy: i32) { - if dx == 0 && dy == 0 { - return; - } - - // 1. Update local cursor IMMEDIATELY for instant visual feedback - let width = LOCAL_CURSOR_WIDTH.load(Ordering::Acquire); - let height = LOCAL_CURSOR_HEIGHT.load(Ordering::Acquire); - let old_x = LOCAL_CURSOR_X.load(Ordering::Acquire); - let old_y = LOCAL_CURSOR_Y.load(Ordering::Acquire); - LOCAL_CURSOR_X.store((old_x + dx).clamp(0, width), Ordering::Release); - LOCAL_CURSOR_Y.store((old_y + dy).clamp(0, height), Ordering::Release); - - // 2. Accumulate delta for coalescing - COALESCE_DX.fetch_add(dx, Ordering::Relaxed); - COALESCE_DY.fetch_add(dy, Ordering::Relaxed); - COALESCED_EVENT_COUNT.fetch_add(1, Ordering::Relaxed); - - // Also accumulate for legacy API - ACCUMULATED_DX.fetch_add(dx, Ordering::Relaxed); - ACCUMULATED_DY.fetch_add(dy, Ordering::Relaxed); - - // 3. Check if enough time has passed to send batch (2ms default) - let now_us = session_elapsed_us(); - let last_us = COALESCE_LAST_SEND_US.load(Ordering::Acquire); - - if now_us.saturating_sub(last_us) >= MOUSE_COALESCE_INTERVAL_US { - flush_coalesced_events(); - } -} - -/// Process scroll wheel event -fn process_scroll(delta: i32) { - if delta == 0 { - return; - } - - let timestamp_us = get_timestamp_us(); - let guard = EVENT_SENDER.lock(); - if let Some(ref sender) = *guard { - // Linux scroll is typically 1 unit per notch, Windows uses 120 - // Scale to match Windows WHEEL_DELTA - let _ = sender.try_send(InputEvent::MouseWheel { - delta: (delta * 120) as i16, - timestamp_us, - }); - } -} - -/// Find the primary mouse device in /dev/input/ -fn find_mouse_device() -> Option { - // Try common mouse device paths - let candidates = [ - "/dev/input/mice", // Combined mice device - "/dev/input/mouse0", // First mouse - "/dev/input/event0", // First event device (may be mouse) - ]; - - // First, try to find a proper mouse via evdev enumeration - if let Ok(entries) = std::fs::read_dir("/dev/input") { - for entry in entries.filter_map(|e| e.ok()) { - let path = entry.path(); - if let Some(name) = path.file_name().and_then(|n| n.to_str()) { - if name.starts_with("event") { - if let Ok(device) = Device::open(&path) { - // Check if this device has relative axes (mouse) - if device.supported_relative_axes().map_or(false, |axes| { - axes.contains(RelativeAxisType::REL_X) - && axes.contains(RelativeAxisType::REL_Y) - }) { - let device_name = device.name().unwrap_or("Unknown"); - // Skip virtual/tablet devices - let name_lower = device_name.to_lowercase(); - if !name_lower.contains("tablet") - && !name_lower.contains("touch") - && !name_lower.contains("wacom") - { - info!("Found mouse device: {} ({})", path.display(), device_name); - return Some(path.to_string_lossy().to_string()); - } - } - } - } - } - } - } - - // Fallback to known paths - for path in &candidates { - if Path::new(path).exists() { - info!("Using fallback mouse device: {}", path); - return Some(path.to_string()); - } - } - - None -} - -/// evdev input thread - direct device access for lowest latency -fn start_evdev_input(device_path: &str) -> Result<(), String> { - let mut device = Device::open(device_path) - .map_err(|e| format!("Failed to open evdev device {}: {}", device_path, e))?; - - let device_name = device.name().unwrap_or("Unknown").to_string(); - info!("evdev: Opened device '{}' at {}", device_name, device_path); - - // Grab the device for exclusive access (optional - may fail on some systems) - // This prevents other applications from receiving the events - if let Err(e) = device.grab() { - warn!( - "evdev: Could not grab device exclusively: {} (continuing anyway)", - e - ); - } - - // Mark as registered before spawning thread - RAW_INPUT_REGISTERED.store(true, Ordering::SeqCst); - RAW_INPUT_ACTIVE.store(true, Ordering::SeqCst); - *ACTIVE_BACKEND.lock() = InputBackend::Evdev; - - let device_path_owned = device_path.to_string(); - std::thread::spawn(move || { - info!("evdev input thread started for {}", device_path_owned); - - // Re-open in the thread to avoid Send issues - let mut device = match Device::open(&device_path_owned) { - Ok(d) => d, - Err(e) => { - error!("evdev: Failed to reopen device: {}", e); - RAW_INPUT_REGISTERED.store(false, Ordering::SeqCst); - RAW_INPUT_ACTIVE.store(false, Ordering::SeqCst); - return; - } - }; - - // Event loop - loop { - if STOP_REQUESTED.load(Ordering::SeqCst) { - break; - } - - // Fetch events (blocking with timeout would be better, but evdev crate - // doesn't support that directly - we use a small sleep instead) - match device.fetch_events() { - Ok(events) => { - if !RAW_INPUT_ACTIVE.load(Ordering::SeqCst) { - continue; - } - - for event in events { - match event.kind() { - InputEventKind::RelAxis(axis) => { - let value = event.value(); - match axis { - RelativeAxisType::REL_X => { - process_mouse_delta(value, 0); - } - RelativeAxisType::REL_Y => { - process_mouse_delta(0, value); - } - RelativeAxisType::REL_WHEEL - | RelativeAxisType::REL_WHEEL_HI_RES => { - process_scroll(value); - } - _ => {} - } - } - _ => {} - } - } - } - Err(e) => { - // EAGAIN is normal for non-blocking reads - if e.raw_os_error() != Some(libc::EAGAIN) { - debug!("evdev: Error reading events: {}", e); - } - // Small sleep to prevent busy-waiting - std::thread::sleep(std::time::Duration::from_micros(100)); - } - } - } - - // Cleanup - let _ = device.ungrab(); - RAW_INPUT_REGISTERED.store(false, Ordering::SeqCst); - RAW_INPUT_ACTIVE.store(false, Ordering::SeqCst); - info!("evdev input thread stopped"); - }); - - Ok(()) -} - -/// X11 XInput2 input thread - fallback for when evdev isn't available -#[cfg(feature = "x11-input")] -fn start_x11_input() -> Result<(), String> { - unsafe { - // Open X display - let display = xlib::XOpenDisplay(std::ptr::null()); - if display.is_null() { - return Err("Failed to open X11 display".to_string()); - } - - let root = xlib::XDefaultRootWindow(display); - - // Check for XInput2 extension - let mut xi_opcode = 0; - let mut event = 0; - let mut error = 0; - let xinput_name = CString::new("XInputExtension").unwrap(); - - if xlib::XQueryExtension( - display, - xinput_name.as_ptr(), - &mut xi_opcode, - &mut event, - &mut error, - ) == 0 - { - xlib::XCloseDisplay(display); - return Err("XInput2 extension not available".to_string()); - } - - // Query XInput2 version (need at least 2.0) - let mut major = 2; - let mut minor = 0; - if xi2::XIQueryVersion(display, &mut major, &mut minor) != xlib::Success as i32 { - xlib::XCloseDisplay(display); - return Err(format!("XInput2 version {}.{} not supported", major, minor)); - } - - info!("X11: Using XInput2 version {}.{}", major, minor); - - // Select raw motion events - let mut mask: [u8; 4] = [0; 4]; - xi2::XISetMask(&mut mask, xi2::XI_RawMotion); - xi2::XISetMask(&mut mask, xi2::XI_RawButtonPress); - xi2::XISetMask(&mut mask, xi2::XI_RawButtonRelease); - - let mut evmask = xi2::XIEventMask { - deviceid: xi2::XIAllMasterDevices, - mask_len: mask.len() as i32, - mask: mask.as_mut_ptr(), - }; - - if xi2::XISelectEvents(display, root, &mut evmask, 1) != xlib::Success as i32 { - xlib::XCloseDisplay(display); - return Err("Failed to select XInput2 events".to_string()); - } - - // Mark as registered - RAW_INPUT_REGISTERED.store(true, Ordering::SeqCst); - RAW_INPUT_ACTIVE.store(true, Ordering::SeqCst); - *ACTIVE_BACKEND.lock() = InputBackend::X11; - - // Spawn event thread - std::thread::spawn(move || { - info!("X11 XInput2 input thread started"); - - let xi_opcode = xi_opcode; // Move into closure - - loop { - if STOP_REQUESTED.load(Ordering::SeqCst) { - break; - } - - // Check for pending events - while xlib::XPending(display) > 0 { - let mut event: xlib::XEvent = std::mem::zeroed(); - xlib::XNextEvent(display, &mut event); - - // Check if this is a GenericEvent (XInput2 events) - if event.get_type() == xlib::GenericEvent { - let cookie = &mut event.generic_event_cookie; - - if xlib::XGetEventData(display, cookie) != 0 { - if cookie.extension == xi_opcode { - if RAW_INPUT_ACTIVE.load(Ordering::SeqCst) { - match cookie.evtype { - xi2::XI_RawMotion => { - let raw = cookie.data as *const xi2::XIRawEvent; - if !raw.is_null() { - let raw_event = &*raw; - - // Extract raw values (unaccelerated) - let valuators = raw_event.raw_values; - let mask = raw_event.valuators.mask; - let mask_len = raw_event.valuators.mask_len; - - let mut dx = 0.0f64; - let mut dy = 0.0f64; - let mut idx = 0; - - // Iterate through set bits in mask - for i in 0..(mask_len * 8) { - let byte_idx = (i / 8) as usize; - let bit_idx = i % 8; - - if byte_idx < mask_len as usize { - let mask_byte = *mask.add(byte_idx); - if (mask_byte & (1 << bit_idx)) != 0 { - let value = *valuators.add(idx); - match i { - 0 => dx = value, - 1 => dy = value, - _ => {} - } - idx += 1; - } - } - } - - if dx != 0.0 || dy != 0.0 { - process_mouse_delta(dx as i32, dy as i32); - } - } - } - _ => {} - } - } - } - xlib::XFreeEventData(display, cookie); - } - } - } - - // Small sleep when no events pending - std::thread::sleep(std::time::Duration::from_micros(500)); - } - - // Cleanup - xlib::XCloseDisplay(display); - RAW_INPUT_REGISTERED.store(false, Ordering::SeqCst); - RAW_INPUT_ACTIVE.store(false, Ordering::SeqCst); - info!("X11 XInput2 input thread stopped"); - }); - - Ok(()) - } -} - -/// Start raw input capture -/// Tries evdev first (lowest latency), falls back to X11 XInput2 -pub fn start_raw_input() -> Result<(), String> { - // If already registered AND active, just return success - if RAW_INPUT_REGISTERED.load(Ordering::SeqCst) { - if RAW_INPUT_ACTIVE.load(Ordering::SeqCst) { - info!("Raw input already active"); - return Ok(()); - } - // Re-activating existing registration - ACCUMULATED_DX.store(0, Ordering::SeqCst); - ACCUMULATED_DY.store(0, Ordering::SeqCst); - COALESCE_DX.store(0, Ordering::SeqCst); - COALESCE_DY.store(0, Ordering::SeqCst); - COALESCE_LAST_SEND_US.store(0, Ordering::SeqCst); - RAW_INPUT_ACTIVE.store(true, Ordering::SeqCst); - info!("Raw input resumed with clean state"); - return Ok(()); - } - - // Reset state - STOP_REQUESTED.store(false, Ordering::SeqCst); - ACCUMULATED_DX.store(0, Ordering::SeqCst); - ACCUMULATED_DY.store(0, Ordering::SeqCst); - COALESCE_DX.store(0, Ordering::SeqCst); - COALESCE_DY.store(0, Ordering::SeqCst); - COALESCE_LAST_SEND_US.store(0, Ordering::SeqCst); - COALESCED_EVENT_COUNT.store(0, Ordering::SeqCst); - - // Try evdev first (requires user to be in 'input' group or root) - if let Some(device_path) = find_mouse_device() { - match start_evdev_input(&device_path) { - Ok(()) => { - // Wait for thread to start - std::thread::sleep(std::time::Duration::from_millis(50)); - if RAW_INPUT_REGISTERED.load(Ordering::SeqCst) { - info!("Raw input started via evdev - lowest latency mode"); - return Ok(()); - } - } - Err(e) => { - warn!("evdev failed: {} - trying X11 fallback", e); - } - } - } else { - warn!("No mouse device found for evdev"); - #[cfg(feature = "x11-input")] - warn!("Trying X11 fallback..."); - } - - // Fall back to X11 XInput2 (requires x11-input feature) - #[cfg(feature = "x11-input")] - { - match start_x11_input() { - Ok(()) => { - std::thread::sleep(std::time::Duration::from_millis(50)); - if RAW_INPUT_REGISTERED.load(Ordering::SeqCst) { - info!("Raw input started via X11 XInput2"); - return Ok(()); - } - return Err("X11 input thread failed to start".to_string()); - } - Err(e) => { - // Check if running on Raspberry Pi for better error message - let is_pi = Path::new("/sys/firmware/devicetree/base/model").exists() - && std::fs::read_to_string("/sys/firmware/devicetree/base/model") - .map(|s| s.to_lowercase().contains("raspberry pi")) - .unwrap_or(false); - - if is_pi { - error!( - "Input setup failed on Raspberry Pi. Please add your user to the 'input' group:\n\ - sudo usermod -aG input $USER\n\ - Then log out and back in." - ); - } else { - error!( - "All input backends failed. evdev requires 'input' group membership. X11 error: {}", - e - ); - } - return Err(format!("Failed to start raw input: {}", e)); - } - } - } - - // No X11 fallback available - #[cfg(not(feature = "x11-input"))] - { - let is_pi = Path::new("/sys/firmware/devicetree/base/model").exists() - && std::fs::read_to_string("/sys/firmware/devicetree/base/model") - .map(|s| s.to_lowercase().contains("raspberry pi")) - .unwrap_or(false); - - if is_pi { - error!( - "Input setup failed on Raspberry Pi. Please add your user to the 'input' group:\n\ - sudo usermod -aG input $USER\n\ - Then log out and back in." - ); - } else { - error!( - "evdev input failed. Please add your user to the 'input' group:\n\ - sudo usermod -aG input $USER\n\ - Then log out and back in.\n\ - Note: X11 fallback not available (build without x11-input feature)" - ); - } - return Err( - "Failed to start raw input: evdev not available and X11 fallback disabled".to_string(), - ); - } -} - -/// Pause raw input capture -pub fn pause_raw_input() { - RAW_INPUT_ACTIVE.store(false, Ordering::SeqCst); - ACCUMULATED_DX.store(0, Ordering::SeqCst); - ACCUMULATED_DY.store(0, Ordering::SeqCst); - debug!("Raw input paused"); -} - -/// Resume raw input capture -pub fn resume_raw_input() { - if RAW_INPUT_REGISTERED.load(Ordering::SeqCst) { - ACCUMULATED_DX.store(0, Ordering::SeqCst); - ACCUMULATED_DY.store(0, Ordering::SeqCst); - RAW_INPUT_ACTIVE.store(true, Ordering::SeqCst); - debug!("Raw input resumed"); - } -} - -/// Stop raw input completely -pub fn stop_raw_input() { - // Signal thread to stop - STOP_REQUESTED.store(true, Ordering::SeqCst); - RAW_INPUT_ACTIVE.store(false, Ordering::SeqCst); - - // Clear the event sender - clear_raw_input_sender(); - - // Reset state - ACCUMULATED_DX.store(0, Ordering::SeqCst); - ACCUMULATED_DY.store(0, Ordering::SeqCst); - COALESCE_DX.store(0, Ordering::SeqCst); - COALESCE_DY.store(0, Ordering::SeqCst); - COALESCE_LAST_SEND_US.store(0, Ordering::SeqCst); - - // Reset local cursor to center - let width = LOCAL_CURSOR_WIDTH.load(Ordering::Acquire); - let height = LOCAL_CURSOR_HEIGHT.load(Ordering::Acquire); - LOCAL_CURSOR_X.store(width / 2, Ordering::SeqCst); - LOCAL_CURSOR_Y.store(height / 2, Ordering::SeqCst); - - // Wait for thread to stop - let start = std::time::Instant::now(); - while RAW_INPUT_REGISTERED.load(Ordering::SeqCst) { - if start.elapsed() > std::time::Duration::from_millis(1000) { - error!("Raw input thread did not exit in time, forcing reset"); - RAW_INPUT_REGISTERED.store(false, Ordering::SeqCst); - break; - } - std::thread::sleep(std::time::Duration::from_millis(10)); - } - - *ACTIVE_BACKEND.lock() = InputBackend::None; - std::thread::sleep(std::time::Duration::from_millis(50)); - info!("Raw input stopped and fully cleaned up"); -} - -/// Get accumulated mouse deltas and reset -pub fn get_raw_mouse_delta() -> (i32, i32) { - let dx = ACCUMULATED_DX.swap(0, Ordering::SeqCst); - let dy = ACCUMULATED_DY.swap(0, Ordering::SeqCst); - (dx, dy) -} - -/// Check if raw input is active -pub fn is_raw_input_active() -> bool { - RAW_INPUT_ACTIVE.load(Ordering::SeqCst) -} - -/// Update center position (no-op on Linux with evdev, kept for API compatibility) -pub fn update_raw_input_center() { - // Linux evdev provides pure relative motion, no recentering needed -} - -/// Set the event sender for direct mouse event delivery -pub fn set_raw_input_sender(sender: mpsc::Sender) { - let mut guard = EVENT_SENDER.lock(); - *guard = Some(sender); - info!("Raw input direct sender configured"); -} - -/// Clear the event sender -pub fn clear_raw_input_sender() { - let mut guard = EVENT_SENDER.lock(); - *guard = None; -} - -/// Set local cursor dimensions (call when stream starts or resolution changes) -pub fn set_local_cursor_dimensions(width: u32, height: u32) { - LOCAL_CURSOR_WIDTH.store(width as i32, Ordering::Release); - LOCAL_CURSOR_HEIGHT.store(height as i32, Ordering::Release); - // Center cursor when dimensions change - LOCAL_CURSOR_X.store(width as i32 / 2, Ordering::Release); - LOCAL_CURSOR_Y.store(height as i32 / 2, Ordering::Release); - info!("Local cursor dimensions set to {}x{}", width, height); -} - -/// Get local cursor position (for rendering) -pub fn get_local_cursor_position() -> (i32, i32) { - ( - LOCAL_CURSOR_X.load(Ordering::Acquire), - LOCAL_CURSOR_Y.load(Ordering::Acquire), - ) -} - -/// Get local cursor position normalized (0.0-1.0) -pub fn get_local_cursor_normalized() -> (f32, f32) { - let x = LOCAL_CURSOR_X.load(Ordering::Acquire) as f32; - let y = LOCAL_CURSOR_Y.load(Ordering::Acquire) as f32; - let w = LOCAL_CURSOR_WIDTH.load(Ordering::Acquire) as f32; - let h = LOCAL_CURSOR_HEIGHT.load(Ordering::Acquire) as f32; - (x / w.max(1.0), y / h.max(1.0)) -} - -/// Flush any pending coalesced mouse events -pub fn flush_pending_mouse_events() { - flush_coalesced_events(); -} - -/// Get count of coalesced events (for stats) -pub fn get_coalesced_event_count() -> u64 { - COALESCED_EVENT_COUNT.load(Ordering::Relaxed) -} - -/// Reset coalescing state (call when streaming stops) -pub fn reset_coalescing() { - COALESCE_DX.store(0, Ordering::Release); - COALESCE_DY.store(0, Ordering::Release); - COALESCE_LAST_SEND_US.store(0, Ordering::Release); - COALESCED_EVENT_COUNT.store(0, Ordering::Release); - // Center cursor based on actual dimensions - let width = LOCAL_CURSOR_WIDTH.load(Ordering::Acquire); - let height = LOCAL_CURSOR_HEIGHT.load(Ordering::Acquire); - LOCAL_CURSOR_X.store(width / 2, Ordering::Release); - LOCAL_CURSOR_Y.store(height / 2, Ordering::Release); -} - -/// Get the active input backend name (for debugging) -pub fn get_active_backend_name() -> &'static str { - match *ACTIVE_BACKEND.lock() { - InputBackend::Evdev => "evdev", - #[cfg(feature = "x11-input")] - InputBackend::X11 => "X11 XInput2", - InputBackend::None => "none", - } -} diff --git a/opennow-streamer/src/input/macos.rs b/opennow-streamer/src/input/macos.rs deleted file mode 100644 index 140705c..0000000 --- a/opennow-streamer/src/input/macos.rs +++ /dev/null @@ -1,509 +0,0 @@ -//! macOS Raw Input API -//! -//! Provides hardware-level mouse input using Core Graphics event taps. -//! Captures mouse deltas directly for responsive input without OS acceleration effects. -//! Events are coalesced (batched) every 2ms like the official GFN client. -//! -//! Key optimizations: -//! - Lock-free event accumulation using atomics -//! - Local cursor tracking for instant visual feedback - -use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU64, AtomicPtr, Ordering}; -use std::ffi::c_void; -use log::{info, error, debug, warn}; -use tokio::sync::mpsc; -use parking_lot::Mutex; - -use crate::webrtc::InputEvent; -use super::{get_timestamp_us, session_elapsed_us, MOUSE_COALESCE_INTERVAL_US}; - - - -// Core Graphics bindings -#[link(name = "CoreGraphics", kind = "framework")] -extern "C" { - fn CGEventTapCreate( - tap: CGEventTapLocation, - place: CGEventTapPlacement, - options: CGEventTapOptions, - events_of_interest: CGEventMask, - callback: CGEventTapCallBack, - user_info: *mut c_void, - ) -> CFMachPortRef; - - fn CGEventTapEnable(tap: CFMachPortRef, enable: bool); - fn CGEventGetIntegerValueField(event: CGEventRef, field: CGEventField) -> i64; - fn CGEventGetType(event: CGEventRef) -> CGEventType; - fn CGEventSourceSetLocalEventsSuppressionInterval(source: CGEventSourceRef, seconds: f64); -} - -#[link(name = "CoreFoundation", kind = "framework")] -extern "C" { - fn CFMachPortCreateRunLoopSource( - allocator: CFAllocatorRef, - port: CFMachPortRef, - order: CFIndex, - ) -> CFRunLoopSourceRef; - - fn CFRunLoopGetCurrent() -> CFRunLoopRef; - fn CFRunLoopAddSource(rl: CFRunLoopRef, source: CFRunLoopSourceRef, mode: CFStringRef); - fn CFRunLoopRun(); - fn CFRunLoopStop(rl: CFRunLoopRef); - fn CFRelease(cf: *const c_void); - - static kCFRunLoopCommonModes: CFStringRef; - static kCFAllocatorDefault: CFAllocatorRef; -} - -// Core Graphics types -type CFMachPortRef = *mut c_void; -type CFRunLoopSourceRef = *mut c_void; -type CFRunLoopRef = *mut c_void; -type CFAllocatorRef = *const c_void; -type CFStringRef = *const c_void; -type CFIndex = isize; -type CGEventRef = *mut c_void; -type CGEventSourceRef = *mut c_void; -type CGEventMask = u64; - -type CGEventTapCallBack = extern "C" fn( - proxy: *mut c_void, - event_type: CGEventType, - event: CGEventRef, - user_info: *mut c_void, -) -> CGEventRef; - -#[repr(u32)] -#[derive(Clone, Copy)] -enum CGEventTapLocation { - HIDEventTap = 0, - SessionEventTap = 1, - AnnotatedSessionEventTap = 2, -} - -#[repr(u32)] -#[derive(Clone, Copy)] -enum CGEventTapPlacement { - HeadInsertEventTap = 0, - TailAppendEventTap = 1, -} - -#[repr(u32)] -#[derive(Clone, Copy)] -enum CGEventTapOptions { - Default = 0, - ListenOnly = 1, -} - -#[repr(u32)] -#[derive(Clone, Copy, PartialEq, Debug)] -enum CGEventType { - Null = 0, - LeftMouseDown = 1, - LeftMouseUp = 2, - RightMouseDown = 3, - RightMouseUp = 4, - MouseMoved = 5, - LeftMouseDragged = 6, - RightMouseDragged = 7, - KeyDown = 10, - KeyUp = 11, - FlagsChanged = 12, - ScrollWheel = 22, - TabletPointer = 23, - TabletProximity = 24, - OtherMouseDown = 25, - OtherMouseUp = 26, - OtherMouseDragged = 27, - TapDisabledByTimeout = 0xFFFFFFFE, - TapDisabledByUserInput = 0xFFFFFFFF, -} - -#[repr(u32)] -#[derive(Clone, Copy)] -enum CGEventField { - MouseEventDeltaX = 4, - MouseEventDeltaY = 5, - ScrollWheelEventDeltaAxis1 = 11, - KeyboardEventKeycode = 9, -} - -// Event masks -const CGMOUSEDOWN_MASK: u64 = (1 << CGEventType::LeftMouseDown as u64) - | (1 << CGEventType::RightMouseDown as u64) - | (1 << CGEventType::OtherMouseDown as u64); -const CGMOUSEUP_MASK: u64 = (1 << CGEventType::LeftMouseUp as u64) - | (1 << CGEventType::RightMouseUp as u64) - | (1 << CGEventType::OtherMouseUp as u64); -const CGMOUSEMOVED_MASK: u64 = (1 << CGEventType::MouseMoved as u64) - | (1 << CGEventType::LeftMouseDragged as u64) - | (1 << CGEventType::RightMouseDragged as u64) - | (1 << CGEventType::OtherMouseDragged as u64); -const CGSCROLL_MASK: u64 = 1 << CGEventType::ScrollWheel as u64; - -// Static state -static RAW_INPUT_REGISTERED: AtomicBool = AtomicBool::new(false); -static RAW_INPUT_ACTIVE: AtomicBool = AtomicBool::new(false); -static ACCUMULATED_DX: AtomicI32 = AtomicI32::new(0); -static ACCUMULATED_DY: AtomicI32 = AtomicI32::new(0); - -// Coalescing state -static COALESCE_DX: AtomicI32 = AtomicI32::new(0); -static COALESCE_DY: AtomicI32 = AtomicI32::new(0); -static COALESCE_LAST_SEND_US: AtomicU64 = AtomicU64::new(0); -static COALESCED_EVENT_COUNT: AtomicU64 = AtomicU64::new(0); - -// Local cursor tracking -static LOCAL_CURSOR_X: AtomicI32 = AtomicI32::new(960); -static LOCAL_CURSOR_Y: AtomicI32 = AtomicI32::new(540); -static LOCAL_CURSOR_WIDTH: AtomicI32 = AtomicI32::new(1920); -static LOCAL_CURSOR_HEIGHT: AtomicI32 = AtomicI32::new(1080); - -// Event sender - use Mutex but minimize lock time -static EVENT_SENDER: Mutex>> = Mutex::new(None); - -// Run loop reference for stopping (use AtomicPtr for thread-safety with raw pointers) -static RUN_LOOP: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); -static EVENT_TAP: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); - - - -/// Flush coalesced mouse events -/// Uses blocking lock to ensure events are never dropped (matches Windows behavior) -#[inline] -fn flush_coalesced_events() { - let dx = COALESCE_DX.swap(0, Ordering::AcqRel); - let dy = COALESCE_DY.swap(0, Ordering::AcqRel); - - if dx != 0 || dy != 0 { - let timestamp_us = get_timestamp_us(); - let now_us = session_elapsed_us(); - COALESCE_LAST_SEND_US.store(now_us, Ordering::Release); - - // Log first few flushes to verify input flow - static FLUSH_LOG_COUNT: AtomicU64 = AtomicU64::new(0); - let count = FLUSH_LOG_COUNT.fetch_add(1, Ordering::Relaxed); - if count < 10 { - info!("Mouse flush #{}: dx={}, dy={}", count, dx, dy); - } - - // Use blocking lock to match Windows behavior - never drop events - let guard = EVENT_SENDER.lock(); - if let Some(ref sender) = *guard { - if sender.try_send(InputEvent::MouseMove { - dx: dx as i16, - dy: dy as i16, - timestamp_us, - }).is_err() { - // Channel full - this is a real backpressure situation - // Log it but don't re-queue (would cause more delays) - warn!("Input channel full - event dropped"); - } - } else if count < 5 { - warn!("EVENT_SENDER is None - raw input sender not configured!"); - } - } -} - -/// Core Graphics event tap callback -extern "C" fn event_tap_callback( - _proxy: *mut c_void, - event_type: CGEventType, - event: CGEventRef, - _user_info: *mut c_void, -) -> CGEventRef { - // Handle tap being disabled - if event_type == CGEventType::TapDisabledByTimeout - || event_type == CGEventType::TapDisabledByUserInput { - // Re-enable the tap - let tap = EVENT_TAP.load(Ordering::Acquire); - if !tap.is_null() { - unsafe { - CGEventTapEnable(tap, true); - } - } - warn!("Event tap was disabled, re-enabling"); - return event; - } - - if !RAW_INPUT_ACTIVE.load(Ordering::SeqCst) { - return event; - } - - unsafe { - let actual_type = CGEventGetType(event); - - match actual_type { - CGEventType::MouseMoved - | CGEventType::LeftMouseDragged - | CGEventType::RightMouseDragged - | CGEventType::OtherMouseDragged => { - // Get raw mouse delta (unaccelerated on modern macOS) - let dx = CGEventGetIntegerValueField(event, CGEventField::MouseEventDeltaX) as i32; - let dy = CGEventGetIntegerValueField(event, CGEventField::MouseEventDeltaY) as i32; - - if dx != 0 || dy != 0 { - // 1. Update local cursor immediately for visual feedback - let width = LOCAL_CURSOR_WIDTH.load(Ordering::Acquire); - let height = LOCAL_CURSOR_HEIGHT.load(Ordering::Acquire); - let old_x = LOCAL_CURSOR_X.load(Ordering::Acquire); - let old_y = LOCAL_CURSOR_Y.load(Ordering::Acquire); - LOCAL_CURSOR_X.store((old_x + dx).clamp(0, width), Ordering::Release); - LOCAL_CURSOR_Y.store((old_y + dy).clamp(0, height), Ordering::Release); - - // 2. Accumulate for coalescing - COALESCE_DX.fetch_add(dx, Ordering::Relaxed); - COALESCE_DY.fetch_add(dy, Ordering::Relaxed); - COALESCED_EVENT_COUNT.fetch_add(1, Ordering::Relaxed); - - // Also accumulate for legacy API - ACCUMULATED_DX.fetch_add(dx, Ordering::Relaxed); - ACCUMULATED_DY.fetch_add(dy, Ordering::Relaxed); - - // 3. Check if enough time to send batch - let now_us = session_elapsed_us(); - let last_us = COALESCE_LAST_SEND_US.load(Ordering::Acquire); - - if now_us.saturating_sub(last_us) >= MOUSE_COALESCE_INTERVAL_US { - flush_coalesced_events(); - } - } - } - CGEventType::ScrollWheel => { - let delta = CGEventGetIntegerValueField(event, CGEventField::ScrollWheelEventDeltaAxis1) as i16; - if delta != 0 { - let timestamp_us = get_timestamp_us(); - // Use try_lock to avoid blocking the event tap callback - if let Some(guard) = EVENT_SENDER.try_lock() { - if let Some(ref sender) = *guard { - // macOS scroll is inverted compared to Windows, and uses different scale - // Multiply by 120 to match Windows WHEEL_DELTA - let _ = sender.try_send(InputEvent::MouseWheel { - delta: delta * 120, - timestamp_us, - }); - } - } - // Note: scroll events dropped if lock contention, acceptable for wheel - } - } - _ => {} - } - } - - event -} - - - -/// Start raw input capture -pub fn start_raw_input() -> Result<(), String> { - if RAW_INPUT_REGISTERED.load(Ordering::SeqCst) { - RAW_INPUT_ACTIVE.store(true, Ordering::SeqCst); - info!("Raw input resumed"); - return Ok(()); - } - - // Spawn thread for event tap run loop - std::thread::spawn(|| { - unsafe { - // Create event tap for mouse events - let event_mask: CGEventMask = CGMOUSEMOVED_MASK | CGSCROLL_MASK; - - let tap = CGEventTapCreate( - CGEventTapLocation::HIDEventTap, // Capture at HID level for raw input - CGEventTapPlacement::HeadInsertEventTap, - CGEventTapOptions::ListenOnly, // Don't modify events - event_mask, - event_tap_callback, - std::ptr::null_mut(), - ); - - if tap.is_null() { - error!("Failed to create event tap. Make sure Accessibility permissions are granted in System Preferences > Security & Privacy > Privacy > Accessibility"); - return; - } - - EVENT_TAP.store(tap, Ordering::Release); - - // Create run loop source - let source = CFMachPortCreateRunLoopSource( - kCFAllocatorDefault, - tap, - 0, - ); - - if source.is_null() { - error!("Failed to create run loop source"); - CFRelease(tap); - EVENT_TAP.store(std::ptr::null_mut(), Ordering::Release); - return; - } - - // Get current run loop and add source - let run_loop = CFRunLoopGetCurrent(); - RUN_LOOP.store(run_loop, Ordering::Release); - - CFRunLoopAddSource(run_loop, source, kCFRunLoopCommonModes); - - // Enable the tap - CGEventTapEnable(tap, true); - - RAW_INPUT_REGISTERED.store(true, Ordering::SeqCst); - RAW_INPUT_ACTIVE.store(true, Ordering::SeqCst); - info!("Raw input started - capturing mouse events via CGEventTap"); - - // Flush timer DISABLED - causes lock contention latency - - // Run the loop (blocks until stopped) - CFRunLoopRun(); - - // Cleanup - CGEventTapEnable(tap, false); - CFRelease(source); - CFRelease(tap); - - RAW_INPUT_REGISTERED.store(false, Ordering::SeqCst); - RAW_INPUT_ACTIVE.store(false, Ordering::SeqCst); - EVENT_TAP.store(std::ptr::null_mut(), Ordering::Release); - RUN_LOOP.store(std::ptr::null_mut(), Ordering::Release); - info!("Raw input thread stopped"); - } - }); - - // Wait for initialization - std::thread::sleep(std::time::Duration::from_millis(100)); - - if RAW_INPUT_REGISTERED.load(Ordering::SeqCst) { - Ok(()) - } else { - Err("Failed to start raw input. Check Accessibility permissions.".to_string()) - } -} - -/// Pause raw input capture -pub fn pause_raw_input() { - RAW_INPUT_ACTIVE.store(false, Ordering::SeqCst); - ACCUMULATED_DX.store(0, Ordering::SeqCst); - ACCUMULATED_DY.store(0, Ordering::SeqCst); - debug!("Raw input paused"); -} - -/// Resume raw input capture -pub fn resume_raw_input() { - if RAW_INPUT_REGISTERED.load(Ordering::SeqCst) { - ACCUMULATED_DX.store(0, Ordering::SeqCst); - ACCUMULATED_DY.store(0, Ordering::SeqCst); - RAW_INPUT_ACTIVE.store(true, Ordering::SeqCst); - debug!("Raw input resumed"); - } -} - -/// Stop raw input completely -pub fn stop_raw_input() { - RAW_INPUT_ACTIVE.store(false, Ordering::SeqCst); - - // Stop the run loop - let run_loop = RUN_LOOP.swap(std::ptr::null_mut(), Ordering::AcqRel); - if !run_loop.is_null() { - unsafe { - CFRunLoopStop(run_loop); - } - } - - // Wait for the thread to actually exit (up to 500ms) - // This prevents race conditions when starting a new session immediately - let start = std::time::Instant::now(); - while RAW_INPUT_REGISTERED.load(Ordering::SeqCst) { - if start.elapsed() > std::time::Duration::from_millis(500) { - error!("Raw input thread did not exit in time, forcing reset"); - RAW_INPUT_REGISTERED.store(false, Ordering::SeqCst); - break; - } - std::thread::sleep(std::time::Duration::from_millis(10)); - } - - // Clear the event sender to avoid stale channel issues - clear_raw_input_sender(); - - info!("Raw input stopped"); -} - -/// Get accumulated mouse deltas and reset -pub fn get_raw_mouse_delta() -> (i32, i32) { - let dx = ACCUMULATED_DX.swap(0, Ordering::SeqCst); - let dy = ACCUMULATED_DY.swap(0, Ordering::SeqCst); - (dx, dy) -} - -/// Check if raw input is active -pub fn is_raw_input_active() -> bool { - RAW_INPUT_ACTIVE.load(Ordering::SeqCst) -} - -/// Update center position (no-op on macOS, kept for API compatibility) -pub fn update_raw_input_center() { - // macOS doesn't need cursor recentering with CGEventTap -} - -/// Set the event sender for direct mouse event delivery -pub fn set_raw_input_sender(sender: mpsc::Sender) { - let mut guard = EVENT_SENDER.lock(); - *guard = Some(sender); - info!("Raw input direct sender configured"); -} - -/// Clear the event sender -pub fn clear_raw_input_sender() { - let mut guard = EVENT_SENDER.lock(); - *guard = None; -} - -/// Set local cursor dimensions -pub fn set_local_cursor_dimensions(width: u32, height: u32) { - LOCAL_CURSOR_WIDTH.store(width as i32, Ordering::Release); - LOCAL_CURSOR_HEIGHT.store(height as i32, Ordering::Release); - LOCAL_CURSOR_X.store(width as i32 / 2, Ordering::Release); - LOCAL_CURSOR_Y.store(height as i32 / 2, Ordering::Release); - info!("Local cursor dimensions set to {}x{}", width, height); -} - -/// Get local cursor position -pub fn get_local_cursor_position() -> (i32, i32) { - ( - LOCAL_CURSOR_X.load(Ordering::Acquire), - LOCAL_CURSOR_Y.load(Ordering::Acquire), - ) -} - -/// Get local cursor position normalized (0.0-1.0) -pub fn get_local_cursor_normalized() -> (f32, f32) { - let x = LOCAL_CURSOR_X.load(Ordering::Acquire) as f32; - let y = LOCAL_CURSOR_Y.load(Ordering::Acquire) as f32; - let w = LOCAL_CURSOR_WIDTH.load(Ordering::Acquire) as f32; - let h = LOCAL_CURSOR_HEIGHT.load(Ordering::Acquire) as f32; - (x / w.max(1.0), y / h.max(1.0)) -} - -/// Flush pending coalesced mouse events -pub fn flush_pending_mouse_events() { - flush_coalesced_events(); -} - -/// Get count of coalesced events -pub fn get_coalesced_event_count() -> u64 { - COALESCED_EVENT_COUNT.load(Ordering::Relaxed) -} - -/// Reset coalescing state -pub fn reset_coalescing() { - COALESCE_DX.store(0, Ordering::Release); - COALESCE_DY.store(0, Ordering::Release); - COALESCE_LAST_SEND_US.store(0, Ordering::Release); - COALESCED_EVENT_COUNT.store(0, Ordering::Release); - // Center cursor based on actual dimensions, not hardcoded values - let width = LOCAL_CURSOR_WIDTH.load(Ordering::Acquire); - let height = LOCAL_CURSOR_HEIGHT.load(Ordering::Acquire); - LOCAL_CURSOR_X.store(width / 2, Ordering::Release); - LOCAL_CURSOR_Y.store(height / 2, Ordering::Release); -} diff --git a/opennow-streamer/src/input/mod.rs b/opennow-streamer/src/input/mod.rs deleted file mode 100644 index b389edb..0000000 --- a/opennow-streamer/src/input/mod.rs +++ /dev/null @@ -1,747 +0,0 @@ -//! Input Handling -//! -//! Cross-platform input capture for mouse and keyboard. -//! -//! Key optimizations for native-feeling input: -//! - Mouse event coalescing (batches events every 4-8ms like official client) -//! - Local cursor rendering (instant visual feedback independent of network) -//! - Queue depth management (prevents server-side buffering) - -#[cfg(target_os = "linux")] -mod linux; -#[cfg(target_os = "macos")] -mod macos; -#[cfg(target_os = "windows")] -mod windows; - -pub mod controller; -mod protocol; -pub mod wheel; - -pub use controller::{ControllerManager, RumbleEffect}; -pub use protocol::*; -pub use wheel::{FfbEffectType, G29FfbManager, WheelManager}; - -// Re-export raw input functions for Windows -#[cfg(target_os = "windows")] -pub use windows::{ - clear_raw_input_sender, flush_pending_mouse_events, get_coalesced_event_count, - get_local_cursor_normalized, get_local_cursor_position, get_raw_mouse_delta, - is_raw_input_active, pause_raw_input, reset_coalescing, resume_raw_input, - set_local_cursor_dimensions, set_raw_input_sender, start_raw_input, stop_raw_input, - update_raw_input_center, -}; - -// Re-export raw input functions for macOS -#[cfg(target_os = "macos")] -pub use macos::{ - clear_raw_input_sender, flush_pending_mouse_events, get_coalesced_event_count, - get_local_cursor_normalized, get_local_cursor_position, get_raw_mouse_delta, - is_raw_input_active, pause_raw_input, reset_coalescing, resume_raw_input, - set_local_cursor_dimensions, set_raw_input_sender, start_raw_input, stop_raw_input, - update_raw_input_center, -}; - -// Re-export raw input functions for Linux -#[cfg(target_os = "linux")] -pub use linux::{ - clear_raw_input_sender, flush_pending_mouse_events, get_coalesced_event_count, - get_local_cursor_normalized, get_local_cursor_position, get_raw_mouse_delta, - is_raw_input_active, pause_raw_input, reset_coalescing, resume_raw_input, - set_local_cursor_dimensions, set_raw_input_sender, start_raw_input, stop_raw_input, - update_raw_input_center, -}; - -use parking_lot::RwLock; -use std::time::{Instant, SystemTime, UNIX_EPOCH}; - -/// Session timing state - resettable for each streaming session -/// GFN server expects timestamps relative to session start for proper input timing -struct SessionTiming { - start: Instant, - unix_us: u64, -} - -impl SessionTiming { - fn new() -> Self { - let unix_us = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_micros() as u64) - .unwrap_or(0); - Self { - start: Instant::now(), - unix_us, - } - } -} - -static SESSION_TIMING: RwLock> = RwLock::new(None); - -/// Initialize session timing (call when streaming starts) -/// This MUST be called before each new streaming session to reset timestamps -pub fn init_session_timing() { - let timing = SessionTiming::new(); - log::info!( - "Session timing initialized at {} us (new session)", - timing.unix_us - ); - *SESSION_TIMING.write() = Some(timing); -} - -/// Reset session timing (call when streaming stops) -pub fn reset_session_timing() { - *SESSION_TIMING.write() = None; - log::info!("Session timing reset"); -} - -/// Get timestamp in microseconds -/// Uses a hybrid approach: absolute Unix time base + relative offset from session start -/// This provides both accurate server synchronization and consistent timing -#[inline] -pub fn get_timestamp_us() -> u64 { - let timing = SESSION_TIMING.read(); - if let Some(ref t) = *timing { - let elapsed_us = t.start.elapsed().as_micros() as u64; - t.unix_us.wrapping_add(elapsed_us) - } else { - // Fallback if not initialized (shouldn't happen during streaming) - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_micros() as u64) - .unwrap_or(0) - } -} - -/// Get elapsed time since session start (for coalescing decisions) -#[inline] -pub fn session_elapsed_us() -> u64 { - let timing = SESSION_TIMING.read(); - if let Some(ref t) = *timing { - t.start.elapsed().as_micros() as u64 - } else { - 0 - } -} - -use parking_lot::Mutex; -use std::collections::HashSet; -use std::sync::atomic::{AtomicBool, AtomicI32, Ordering}; -use tokio::sync::mpsc; -use winit::event::{ElementState, MouseButton}; - -use crate::webrtc::{InputEncoder, InputEvent, MAX_CLIPBOARD_PASTE_SIZE}; - -/// Mouse event coalescing interval in microseconds -/// Official client uses 4-16ms depending on browser, we use 2ms for lowest latency -pub const MOUSE_COALESCE_INTERVAL_US: u64 = 2_000; // 2ms = 500Hz effective rate - -/// Maximum input queue depth before throttling -/// Official client maintains 4-8 events ahead of consumption -pub const MAX_INPUT_QUEUE_DEPTH: usize = 8; - -/// Mouse event coalescer - batches high-frequency mouse events -/// Similar to official GFN client's getCoalescedEvents() handling -pub struct MouseCoalescer { - /// Accumulated delta X - accumulated_dx: AtomicI32, - /// Accumulated delta Y - accumulated_dy: AtomicI32, - /// Last send timestamp (microseconds since session start) - last_send_us: std::sync::atomic::AtomicU64, - /// Coalescing interval in microseconds - coalesce_interval_us: u64, - /// Count of coalesced events (for stats) - coalesced_count: std::sync::atomic::AtomicU64, -} - -impl MouseCoalescer { - pub fn new() -> Self { - Self::with_interval(MOUSE_COALESCE_INTERVAL_US) - } - - pub fn with_interval(interval_us: u64) -> Self { - use std::sync::atomic::AtomicU64; - Self { - accumulated_dx: AtomicI32::new(0), - accumulated_dy: AtomicI32::new(0), - last_send_us: AtomicU64::new(0), - coalesce_interval_us: interval_us, - coalesced_count: AtomicU64::new(0), - } - } - - /// Accumulate mouse delta, returns Some if enough time has passed to send - /// Returns (dx, dy, timestamp_us) if ready to send, None if still accumulating - #[inline] - pub fn accumulate(&self, dx: i32, dy: i32) -> Option<(i16, i16, u64)> { - // Accumulate the delta - self.accumulated_dx.fetch_add(dx, Ordering::Relaxed); - self.accumulated_dy.fetch_add(dy, Ordering::Relaxed); - self.coalesced_count.fetch_add(1, Ordering::Relaxed); - - let now_us = session_elapsed_us(); - let last_us = self.last_send_us.load(Ordering::Acquire); - - // Check if enough time has passed since last send - if now_us.saturating_sub(last_us) >= self.coalesce_interval_us { - self.flush_internal(now_us) - } else { - None - } - } - - /// Force flush accumulated events (call periodically or on button events) - pub fn flush(&self) -> Option<(i16, i16, u64)> { - let now_us = session_elapsed_us(); - self.flush_internal(now_us) - } - - #[inline] - fn flush_internal(&self, now_us: u64) -> Option<(i16, i16, u64)> { - // Atomically take the accumulated deltas - let dx = self.accumulated_dx.swap(0, Ordering::AcqRel); - let dy = self.accumulated_dy.swap(0, Ordering::AcqRel); - - // Only send if there's actual movement - if dx != 0 || dy != 0 { - self.last_send_us.store(now_us, Ordering::Release); - let timestamp_us = get_timestamp_us(); - Some((dx as i16, dy as i16, timestamp_us)) - } else { - None - } - } - - /// Get count of coalesced events (for stats) - pub fn coalesced_count(&self) -> u64 { - self.coalesced_count.load(Ordering::Relaxed) - } - - /// Reset the coalescer state - pub fn reset(&self) { - self.accumulated_dx.store(0, Ordering::Release); - self.accumulated_dy.store(0, Ordering::Release); - self.last_send_us.store(0, Ordering::Release); - self.coalesced_count.store(0, Ordering::Release); - } -} - -impl Default for MouseCoalescer { - fn default() -> Self { - Self::new() - } -} - -/// Local cursor position tracker for instant visual feedback -/// Updates immediately on raw input, independent of network latency -pub struct LocalCursor { - /// Current X position (screen coordinates) - x: AtomicI32, - /// Current Y position (screen coordinates) - y: AtomicI32, - /// Stream width for bounds - stream_width: AtomicI32, - /// Stream height for bounds - stream_height: AtomicI32, - /// Whether cursor is visible/active - active: AtomicBool, -} - -impl LocalCursor { - pub fn new() -> Self { - Self { - x: AtomicI32::new(0), - y: AtomicI32::new(0), - stream_width: AtomicI32::new(1920), - stream_height: AtomicI32::new(1080), - active: AtomicBool::new(false), - } - } - - /// Set stream dimensions (for cursor bounds) - pub fn set_dimensions(&self, width: u32, height: u32) { - self.stream_width.store(width as i32, Ordering::Release); - self.stream_height.store(height as i32, Ordering::Release); - } - - /// Apply relative movement to cursor position - #[inline] - pub fn apply_delta(&self, dx: i32, dy: i32) { - let width = self.stream_width.load(Ordering::Acquire); - let height = self.stream_height.load(Ordering::Acquire); - - // Update X with clamping - let old_x = self.x.load(Ordering::Acquire); - let new_x = (old_x + dx).clamp(0, width); - self.x.store(new_x, Ordering::Release); - - // Update Y with clamping - let old_y = self.y.load(Ordering::Acquire); - let new_y = (old_y + dy).clamp(0, height); - self.y.store(new_y, Ordering::Release); - } - - /// Get current cursor position (normalized 0.0-1.0) - pub fn position_normalized(&self) -> (f32, f32) { - let x = self.x.load(Ordering::Acquire) as f32; - let y = self.y.load(Ordering::Acquire) as f32; - let w = self.stream_width.load(Ordering::Acquire) as f32; - let h = self.stream_height.load(Ordering::Acquire) as f32; - (x / w.max(1.0), y / h.max(1.0)) - } - - /// Get current cursor position (screen coordinates) - pub fn position(&self) -> (i32, i32) { - ( - self.x.load(Ordering::Acquire), - self.y.load(Ordering::Acquire), - ) - } - - /// Set absolute cursor position - pub fn set_position(&self, x: i32, y: i32) { - let width = self.stream_width.load(Ordering::Acquire); - let height = self.stream_height.load(Ordering::Acquire); - self.x.store(x.clamp(0, width), Ordering::Release); - self.y.store(y.clamp(0, height), Ordering::Release); - } - - /// Center the cursor - pub fn center(&self) { - let width = self.stream_width.load(Ordering::Acquire); - let height = self.stream_height.load(Ordering::Acquire); - self.x.store(width / 2, Ordering::Release); - self.y.store(height / 2, Ordering::Release); - } - - pub fn set_active(&self, active: bool) { - self.active.store(active, Ordering::Release); - } - - pub fn is_active(&self) -> bool { - self.active.load(Ordering::Acquire) - } -} - -impl Default for LocalCursor { - fn default() -> Self { - Self::new() - } -} - -/// Cross-platform input handler with coalescing and local cursor support -pub struct InputHandler { - /// Input event sender - event_tx: Mutex>>, - - /// Input encoder - encoder: Mutex, - - /// Whether cursor is captured - cursor_captured: AtomicBool, - - /// Currently pressed keys (for releasing on focus loss) - pressed_keys: Mutex>, - - /// Mouse event coalescer for batching high-frequency events - mouse_coalescer: MouseCoalescer, - - /// Local cursor for instant visual feedback - local_cursor: LocalCursor, - - /// Input queue depth estimate (for throttling) - queue_depth: std::sync::atomic::AtomicU64, - - /// Accumulated mouse delta (legacy, for fallback) - accumulated_dx: AtomicI32, - accumulated_dy: AtomicI32, - - /// Last known cursor position - last_x: AtomicI32, - last_y: AtomicI32, -} - -impl InputHandler { - pub fn new() -> Self { - use std::sync::atomic::AtomicU64; - Self { - event_tx: Mutex::new(None), - encoder: Mutex::new(InputEncoder::new()), - cursor_captured: AtomicBool::new(false), - pressed_keys: Mutex::new(HashSet::new()), - mouse_coalescer: MouseCoalescer::new(), - local_cursor: LocalCursor::new(), - queue_depth: AtomicU64::new(0), - accumulated_dx: AtomicI32::new(0), - accumulated_dy: AtomicI32::new(0), - last_x: AtomicI32::new(0), - last_y: AtomicI32::new(0), - } - } - - /// Set the event sender channel (can be called on Arc) - pub fn set_event_sender(&self, tx: mpsc::Sender) { - *self.event_tx.lock() = Some(tx); - } - - /// Get local cursor for rendering - pub fn local_cursor(&self) -> &LocalCursor { - &self.local_cursor - } - - /// Get mouse coalescer stats - pub fn coalesced_event_count(&self) -> u64 { - self.mouse_coalescer.coalesced_count() - } - - /// Set stream dimensions for local cursor - pub fn set_stream_dimensions(&self, width: u32, height: u32) { - self.local_cursor.set_dimensions(width, height); - self.local_cursor.center(); - self.local_cursor.set_active(true); - } - - /// Update queue depth estimate (call from WebRTC layer) - pub fn update_queue_depth(&self, depth: u64) { - self.queue_depth.store(depth, Ordering::Release); - } - - /// Handle mouse button event - /// Flushes any accumulated mouse movement before button event for proper ordering - pub fn handle_mouse_button(&self, button: MouseButton, state: ElementState) { - // Flush accumulated mouse movement BEFORE button event - // This ensures proper event ordering (move -> click, not click -> move) - if let Some((dx, dy, timestamp_us)) = self.mouse_coalescer.flush() { - self.send_event(InputEvent::MouseMove { - dx, - dy, - timestamp_us, - }); - } - - // GFN uses 1-based button indices: 1=Left, 2=Middle, 3=Right - let btn = match button { - MouseButton::Left => 1, - MouseButton::Middle => 2, - MouseButton::Right => 3, - MouseButton::Back => 4, - MouseButton::Forward => 5, - MouseButton::Other(n) => (n + 1) as u8, - }; - - let timestamp_us = get_timestamp_us(); - let event = match state { - ElementState::Pressed => InputEvent::MouseButtonDown { - button: btn, - timestamp_us, - }, - ElementState::Released => InputEvent::MouseButtonUp { - button: btn, - timestamp_us, - }, - }; - - self.send_event(event); - } - - /// Handle cursor move (for non-captured mode) - pub fn handle_cursor_move(&self, x: f64, y: f64) { - if !self.cursor_captured.load(Ordering::Relaxed) { - return; - } - - let x = x as i32; - let y = y as i32; - - let last_x = self.last_x.swap(x, Ordering::Relaxed); - let last_y = self.last_y.swap(y, Ordering::Relaxed); - - if last_x != 0 || last_y != 0 { - let dx = x - last_x; - let dy = y - last_y; - - if dx != 0 || dy != 0 { - // Update local cursor for instant feedback - self.local_cursor.apply_delta(dx, dy); - - // Use coalescer for network events - if let Some((cdx, cdy, timestamp_us)) = self.mouse_coalescer.accumulate(dx, dy) { - self.send_event(InputEvent::MouseMove { - dx: cdx, - dy: cdy, - timestamp_us, - }); - } - } - } - } - - /// Handle raw mouse delta (for captured mode) - WITH COALESCING - /// This is the primary path for mouse input during streaming - pub fn handle_mouse_delta(&self, dx: i16, dy: i16) { - if dx == 0 && dy == 0 { - return; - } - - // Update local cursor immediately for instant visual feedback - self.local_cursor.apply_delta(dx as i32, dy as i32); - - // Check queue depth - throttle if queue is getting full - let depth = self.queue_depth.load(Ordering::Acquire); - if depth > MAX_INPUT_QUEUE_DEPTH as u64 { - // Queue is full, still accumulate but may decimate - self.mouse_coalescer.accumulate(dx as i32, dy as i32); - return; - } - - // Use coalescer for batching - sends every 4ms instead of every event - if let Some((cdx, cdy, timestamp_us)) = - self.mouse_coalescer.accumulate(dx as i32, dy as i32) - { - self.send_event(InputEvent::MouseMove { - dx: cdx, - dy: cdy, - timestamp_us, - }); - } - } - - /// Handle raw mouse delta WITHOUT coalescing (for immediate events) - /// Use this for single-shot movements or when you need immediate transmission - pub fn handle_mouse_delta_immediate(&self, dx: i16, dy: i16) { - if dx == 0 && dy == 0 { - return; - } - - // Update local cursor - self.local_cursor.apply_delta(dx as i32, dy as i32); - - // Send immediately without coalescing - self.send_event(InputEvent::MouseMove { - dx, - dy, - timestamp_us: get_timestamp_us(), - }); - } - - /// Flush any pending coalesced mouse events - /// Call this periodically (e.g., every frame) to ensure events don't get stuck - pub fn flush_mouse_events(&self) { - if let Some((dx, dy, timestamp_us)) = self.mouse_coalescer.flush() { - self.send_event(InputEvent::MouseMove { - dx, - dy, - timestamp_us, - }); - } - } - - /// Reset input state (call when streaming stops) - pub fn reset(&self) { - self.mouse_coalescer.reset(); - self.local_cursor.set_active(false); - self.queue_depth.store(0, Ordering::Release); - self.pressed_keys.lock().clear(); - } - - /// Handle keyboard event - /// keycode is the Windows Virtual Key code (VK code) - pub fn handle_key(&self, keycode: u16, pressed: bool, modifiers: u16) { - // Track key state to prevent duplicate events and enable proper release - let mut pressed_keys = self.pressed_keys.lock(); - - if pressed { - // Only send key down if not already pressed (prevents duplicates) - if !pressed_keys.insert(keycode) { - // Key was already pressed, skip to avoid duplicates - return; - } - } else { - // Only send key up if key was actually pressed - if !pressed_keys.remove(&keycode) { - // Key wasn't tracked as pressed, but send release anyway to be safe - } - } - drop(pressed_keys); - - let timestamp_us = get_timestamp_us(); - // GFN uses keycode (VK code), scancode is set to 0 - let event = if pressed { - InputEvent::KeyDown { - keycode, - scancode: 0, - modifiers, - timestamp_us, - } - } else { - InputEvent::KeyUp { - keycode, - scancode: 0, - modifiers, - timestamp_us, - } - }; - - self.send_event(event); - } - - /// Release all currently pressed keys (call when focus is lost) - pub fn release_all_keys(&self) { - let mut pressed_keys = self.pressed_keys.lock(); - let keys_to_release: Vec = pressed_keys.drain().collect(); - drop(pressed_keys); - - let timestamp_us = get_timestamp_us(); - for keycode in keys_to_release { - log::debug!("Releasing stuck key: 0x{:02X}", keycode); - let event = InputEvent::KeyUp { - keycode, - scancode: 0, - modifiers: 0, - timestamp_us, - }; - self.send_event(event); - } - } - - /// Handle mouse wheel - pub fn handle_wheel(&self, delta: i16) { - self.send_event(InputEvent::MouseWheel { - delta, - timestamp_us: get_timestamp_us(), - }); - } - - /// Set cursor capture state - pub fn set_cursor_captured(&self, captured: bool) { - self.cursor_captured.store(captured, Ordering::Relaxed); - - if captured { - // Reset last position - self.last_x.store(0, Ordering::Relaxed); - self.last_y.store(0, Ordering::Relaxed); - } - } - - /// Check if cursor is captured - pub fn is_cursor_captured(&self) -> bool { - self.cursor_captured.load(Ordering::Relaxed) - } - - /// Get and reset accumulated mouse delta - pub fn take_accumulated_delta(&self) -> (i32, i32) { - let dx = self.accumulated_dx.swap(0, Ordering::Relaxed); - let dy = self.accumulated_dy.swap(0, Ordering::Relaxed); - (dx, dy) - } - - /// Accumulate mouse delta - pub fn accumulate_delta(&self, dx: i32, dy: i32) { - self.accumulated_dx.fetch_add(dx, Ordering::Relaxed); - self.accumulated_dy.fetch_add(dy, Ordering::Relaxed); - } - - /// Send input event - uses blocking send to ensure events aren't dropped - fn send_event(&self, event: InputEvent) { - if let Some(ref tx) = *self.event_tx.lock() { - // Use blocking_send would require async context - // For now, use try_send with larger buffer - critical events are tracked - if tx.try_send(event).is_err() { - log::warn!("Input channel full - event may be dropped"); - } - } - } - - /// Encode and send input directly (for WebRTC data channel) - pub fn encode_and_send(&self, event: &InputEvent) -> Vec { - let mut encoder = self.encoder.lock(); - encoder.encode(event) - } - - /// Handle clipboard paste (Ctrl+V) - /// Reads text from clipboard and sends it as key events - /// Returns the number of characters sent, or 0 if clipboard is empty/unavailable - pub fn handle_clipboard_paste(&self) -> usize { - // Try to read clipboard text - let clipboard_text = match arboard::Clipboard::new() { - Ok(mut clipboard) => match clipboard.get_text() { - Ok(text) => text, - Err(e) => { - log::debug!("Failed to read clipboard: {:?}", e); - return 0; - } - }, - Err(e) => { - log::debug!("Failed to access clipboard: {:?}", e); - return 0; - } - }; - - if clipboard_text.is_empty() { - return 0; - } - - // Truncate to max size (64KB like official GFN client) - let text = if clipboard_text.len() > MAX_CLIPBOARD_PASTE_SIZE { - log::warn!( - "Clipboard text truncated from {} to {} bytes", - clipboard_text.len(), - MAX_CLIPBOARD_PASTE_SIZE - ); - &clipboard_text[..MAX_CLIPBOARD_PASTE_SIZE] - } else { - &clipboard_text - }; - - log::info!("Pasting {} characters from clipboard", text.chars().count()); - - let char_count = text.chars().count(); - - // Send ClipboardPaste event - it will be expanded into key events in the WebRTC layer - self.send_event(InputEvent::ClipboardPaste { - text: text.to_string(), - }); - - char_count - } - - /// Check if a key event is Ctrl+V (paste shortcut) - /// Returns true if the key should trigger clipboard paste - pub fn is_paste_shortcut(keycode: u16, modifiers: u16) -> bool { - // VK_V = 0x56, Ctrl modifier = 0x02 - keycode == 0x56 && (modifiers & 0x02) != 0 - } -} - -impl Default for InputHandler { - fn default() -> Self { - Self::new() - } -} - -/// Convert winit scancode to GFN scancode -pub fn convert_scancode(scancode: u32) -> u16 { - // Winit uses platform-specific scancodes - // For now, pass through directly - scancode as u16 -} - -/// Get current modifier state -pub fn get_modifiers(modifiers: &winit::keyboard::ModifiersState) -> u16 { - let mut result = 0u16; - - if modifiers.shift_key() { - result |= 0x01; - } - if modifiers.control_key() { - result |= 0x02; - } - if modifiers.alt_key() { - result |= 0x04; - } - if modifiers.super_key() { - result |= 0x08; - } - - result -} diff --git a/opennow-streamer/src/input/protocol.rs b/opennow-streamer/src/input/protocol.rs deleted file mode 100644 index 7f025a1..0000000 --- a/opennow-streamer/src/input/protocol.rs +++ /dev/null @@ -1,103 +0,0 @@ -//! Input Protocol Constants -//! -//! GFN input protocol definitions. - -/// Input event types -pub mod event_types { - pub const HEARTBEAT: u32 = 2; - pub const KEY_UP: u32 = 3; - pub const KEY_DOWN: u32 = 4; - pub const MOUSE_ABS: u32 = 5; - pub const MOUSE_REL: u32 = 7; - pub const MOUSE_BUTTON_DOWN: u32 = 8; - pub const MOUSE_BUTTON_UP: u32 = 9; - pub const MOUSE_WHEEL: u32 = 10; -} - -/// Mouse button indices -pub mod mouse_buttons { - pub const LEFT: u8 = 0; - pub const RIGHT: u8 = 1; - pub const MIDDLE: u8 = 2; - pub const BACK: u8 = 3; - pub const FORWARD: u8 = 4; -} - -/// Keyboard modifier flags -pub mod modifiers { - pub const SHIFT: u16 = 0x01; - pub const CTRL: u16 = 0x02; - pub const ALT: u16 = 0x04; - pub const META: u16 = 0x08; - pub const CAPS_LOCK: u16 = 0x10; - pub const NUM_LOCK: u16 = 0x20; -} - -/// Common scancodes (USB HID) -pub mod scancodes { - pub const A: u16 = 0x04; - pub const B: u16 = 0x05; - pub const C: u16 = 0x06; - pub const D: u16 = 0x07; - pub const E: u16 = 0x08; - pub const F: u16 = 0x09; - pub const G: u16 = 0x0A; - pub const H: u16 = 0x0B; - pub const I: u16 = 0x0C; - pub const J: u16 = 0x0D; - pub const K: u16 = 0x0E; - pub const L: u16 = 0x0F; - pub const M: u16 = 0x10; - pub const N: u16 = 0x11; - pub const O: u16 = 0x12; - pub const P: u16 = 0x13; - pub const Q: u16 = 0x14; - pub const R: u16 = 0x15; - pub const S: u16 = 0x16; - pub const T: u16 = 0x17; - pub const U: u16 = 0x18; - pub const V: u16 = 0x19; - pub const W: u16 = 0x1A; - pub const X: u16 = 0x1B; - pub const Y: u16 = 0x1C; - pub const Z: u16 = 0x1D; - - pub const NUM_1: u16 = 0x1E; - pub const NUM_2: u16 = 0x1F; - pub const NUM_3: u16 = 0x20; - pub const NUM_4: u16 = 0x21; - pub const NUM_5: u16 = 0x22; - pub const NUM_6: u16 = 0x23; - pub const NUM_7: u16 = 0x24; - pub const NUM_8: u16 = 0x25; - pub const NUM_9: u16 = 0x26; - pub const NUM_0: u16 = 0x27; - - pub const ENTER: u16 = 0x28; - pub const ESCAPE: u16 = 0x29; - pub const BACKSPACE: u16 = 0x2A; - pub const TAB: u16 = 0x2B; - pub const SPACE: u16 = 0x2C; - - pub const F1: u16 = 0x3A; - pub const F2: u16 = 0x3B; - pub const F3: u16 = 0x3C; - pub const F4: u16 = 0x3D; - pub const F5: u16 = 0x3E; - pub const F6: u16 = 0x3F; - pub const F7: u16 = 0x40; - pub const F8: u16 = 0x41; - pub const F9: u16 = 0x42; - pub const F10: u16 = 0x43; - pub const F11: u16 = 0x44; - pub const F12: u16 = 0x45; - - pub const LEFT_CTRL: u16 = 0xE0; - pub const LEFT_SHIFT: u16 = 0xE1; - pub const LEFT_ALT: u16 = 0xE2; - pub const LEFT_META: u16 = 0xE3; - pub const RIGHT_CTRL: u16 = 0xE4; - pub const RIGHT_SHIFT: u16 = 0xE5; - pub const RIGHT_ALT: u16 = 0xE6; - pub const RIGHT_META: u16 = 0xE7; -} diff --git a/opennow-streamer/src/input/wheel.rs b/opennow-streamer/src/input/wheel.rs deleted file mode 100644 index 21387d7..0000000 --- a/opennow-streamer/src/input/wheel.rs +++ /dev/null @@ -1,774 +0,0 @@ -//! Racing Wheel Input Handler -//! -//! Supports racing wheels via Windows.Gaming.Input RacingWheel API. -//! Provides proper axis separation for wheel, throttle, brake, clutch, and handbrake. -//! Includes force feedback support for immersive racing experiences. -//! -//! For GFN compatibility, wheel input is mapped to the gamepad Type 12 format: -//! - Wheel rotation → Left Stick X (-32768 to 32767) -//! - Throttle → Right Trigger (0-255) -//! - Brake → Left Trigger (0-255) -//! - Clutch → Left Stick Y (mapped, or button) -//! - Handbrake → Button flag -//! - Wheel buttons → Standard button flags - -use log::{debug, error, info, trace, warn}; -use parking_lot::Mutex; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; -use std::time::Duration; -use tokio::sync::mpsc; - -use super::get_timestamp_us; -use crate::webrtc::InputEvent; - -/// Force feedback effect types -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum FfbEffectType { - /// Constant force in one direction - Constant = 0, - /// Spring effect (centering force) - Spring = 1, - /// Damper effect (resistance to motion) - Damper = 2, - /// Friction effect - Friction = 3, -} - -impl From for FfbEffectType { - fn from(value: u8) -> Self { - match value { - 0 => FfbEffectType::Constant, - 1 => FfbEffectType::Spring, - 2 => FfbEffectType::Damper, - 3 => FfbEffectType::Friction, - _ => FfbEffectType::Constant, - } - } -} - -#[cfg(target_os = "windows")] -mod windows_impl { - use super::*; - use std::collections::HashMap; - use windows::Foundation::TimeSpan; - use windows::Gaming::Input::ForceFeedback::{ - ConstantForceEffect, ForceFeedbackLoadEffectResult, ForceFeedbackMotor, - }; - use windows::Gaming::Input::RacingWheel; - use windows_numerics::Vector3; - - /// Racing wheel state - #[derive(Debug, Clone, Default)] - pub struct WheelState { - /// Wheel rotation (-1.0 to 1.0, negative = left, positive = right) - pub wheel: f64, - /// Throttle pedal (0.0 to 1.0) - pub throttle: f64, - /// Brake pedal (0.0 to 1.0) - pub brake: f64, - /// Clutch pedal (0.0 to 1.0) - pub clutch: f64, - /// Handbrake (0.0 to 1.0) - pub handbrake: f64, - /// Button state (RacingWheelButtons flags) - pub buttons: u32, - /// Pattern shifter gear (-1 = reverse, 0 = neutral, 1-10 = gears) - pub gear: i32, - } - - /// Force feedback motor state for a wheel - struct FfbState { - motor: ForceFeedbackMotor, - constant_effect: Option, - effect_loaded: bool, - } - - /// Racing wheel manager using Windows.Gaming.Input - pub struct WheelManagerImpl { - running: Arc, - event_tx: Mutex>>, - wheels: Mutex>, - /// Force feedback state per wheel (indexed by wheel index) - ffb_states: Mutex>, - } - - impl WheelManagerImpl { - pub fn new() -> Self { - Self { - running: Arc::new(AtomicBool::new(false)), - event_tx: Mutex::new(None), - wheels: Mutex::new(Vec::new()), - ffb_states: Mutex::new(HashMap::new()), - } - } - - /// Set the input event sender - pub fn set_event_sender(&self, tx: mpsc::Sender) { - *self.event_tx.lock() = Some(tx); - } - - /// Detect connected racing wheels - pub fn detect_wheels(&self) -> usize { - match RacingWheel::RacingWheels() { - Ok(wheels_view) => { - let count = wheels_view.Size().unwrap_or(0) as usize; - let mut wheels = self.wheels.lock(); - wheels.clear(); - - for i in 0..count { - if let Ok(wheel) = wheels_view.GetAt(i as u32) { - info!("Racing wheel {} detected", i); - wheels.push(wheel); - } - } - - if count > 0 { - info!("Found {} racing wheel(s)", count); - } - count - } - Err(e) => { - debug!("No racing wheels found: {:?}", e); - 0 - } - } - } - - /// Start the wheel input polling loop - pub fn start(&self) { - if self.running.load(Ordering::SeqCst) { - return; - } - - // Detect wheels first - let wheel_count = self.detect_wheels(); - if wheel_count == 0 { - info!("No racing wheels detected - wheel input disabled"); - return; - } - - self.running.store(true, Ordering::SeqCst); - let running = self.running.clone(); - - let tx_opt = self.event_tx.lock().clone(); - if tx_opt.is_none() { - warn!("WheelManager started without event sender!"); - return; - } - let tx = tx_opt.unwrap(); - - // Clone wheels for the thread - let wheels: Vec = self.wheels.lock().clone(); - - std::thread::spawn(move || { - info!( - "Racing wheel input thread starting with {} wheel(s)...", - wheels.len() - ); - - let mut last_states: Vec = vec![WheelState::default(); wheels.len()]; - let mut event_count: u64 = 0; - - while running.load(Ordering::Relaxed) { - for (idx, wheel) in wheels.iter().enumerate() { - // Read current wheel state - if let Ok(reading) = wheel.GetCurrentReading() { - let state = WheelState { - wheel: reading.Wheel, - throttle: reading.Throttle, - brake: reading.Brake, - clutch: reading.Clutch, - handbrake: reading.Handbrake, - buttons: reading.Buttons.0 as u32, - gear: 0, // Pattern shifter handled separately - }; - - // Check if state changed (with small deadzone for analog values) - let last = &last_states[idx]; - let changed = (state.wheel - last.wheel).abs() > 0.001 - || (state.throttle - last.throttle).abs() > 0.01 - || (state.brake - last.brake).abs() > 0.01 - || (state.clutch - last.clutch).abs() > 0.01 - || (state.handbrake - last.handbrake).abs() > 0.01 - || state.buttons != last.buttons; - - if changed { - event_count += 1; - - // Log first few events - if event_count <= 5 { - debug!( - "Wheel {}: rotation={:.2}, throttle={:.2}, brake={:.2}, buttons=0x{:08X}", - idx, state.wheel, state.throttle, state.brake, state.buttons - ); - } - - // Map wheel state to gamepad format for GFN compatibility - // This allows racing games to work without dedicated wheel protocol - let event = Self::map_to_gamepad_event(idx as u8, &state); - - if let Err(e) = tx.try_send(event) { - trace!("Wheel event channel full: {:?}", e); - } - - last_states[idx] = state; - } - } - } - - // Poll at 1000Hz for low latency - std::thread::sleep(Duration::from_millis(1)); - } - - info!( - "Racing wheel input thread stopped (processed {} events)", - event_count - ); - }); - } - - /// Map wheel state to gamepad InputEvent for GFN compatibility - fn map_to_gamepad_event(wheel_idx: u8, state: &WheelState) -> InputEvent { - // Map wheel rotation to left stick X - // Wheel: -1.0 (full left) to 1.0 (full right) -> -32768 to 32767 - let left_stick_x = (state.wheel * 32767.0).clamp(-32768.0, 32767.0) as i16; - - // Map throttle to right trigger (0-255) - let right_trigger = (state.throttle * 255.0).clamp(0.0, 255.0) as u8; - - // Map brake to left trigger (0-255) - let left_trigger = (state.brake * 255.0).clamp(0.0, 255.0) as u8; - - // Map clutch to left stick Y (some games use this) - // Clutch: 0.0 (released) to 1.0 (pressed) -> 0 to 32767 - let left_stick_y = (state.clutch * 32767.0).clamp(0.0, 32767.0) as i16; - - // Map handbrake to right stick Y - let right_stick_y = (state.handbrake * 32767.0).clamp(0.0, 32767.0) as i16; - - // Map wheel buttons to XInput button flags - let button_flags = Self::map_wheel_buttons(state.buttons); - - InputEvent::Gamepad { - controller_id: wheel_idx, - button_flags, - left_trigger, - right_trigger, - left_stick_x, - left_stick_y, - right_stick_x: 0, - right_stick_y, - flags: 1, // Connected flag - timestamp_us: get_timestamp_us(), - } - } - - /// Map RacingWheelButtons to XInput button flags - fn map_wheel_buttons(wheel_buttons: u32) -> u16 { - let mut flags: u16 = 0; - - // RacingWheelButtons enum values (from Windows.Gaming.Input): - // None = 0 - // PreviousGear = 1 - // NextGear = 2 - // DPadUp = 4 - // DPadDown = 8 - // DPadLeft = 16 - // DPadRight = 32 - // Button1-16 = 64 onwards - - // D-Pad mapping (direct match to XInput) - if wheel_buttons & 4 != 0 { - flags |= 0x0001; - } // DPadUp - if wheel_buttons & 8 != 0 { - flags |= 0x0002; - } // DPadDown - if wheel_buttons & 16 != 0 { - flags |= 0x0004; - } // DPadLeft - if wheel_buttons & 32 != 0 { - flags |= 0x0008; - } // DPadRight - - // Gear shift buttons to bumpers - if wheel_buttons & 1 != 0 { - flags |= 0x0100; - } // PreviousGear -> LB - if wheel_buttons & 2 != 0 { - flags |= 0x0200; - } // NextGear -> RB - - // Wheel-specific buttons to face buttons - // Button1 (usually main action) -> A - if wheel_buttons & 64 != 0 { - flags |= 0x1000; - } - // Button2 -> B - if wheel_buttons & 128 != 0 { - flags |= 0x2000; - } - // Button3 -> X - if wheel_buttons & 256 != 0 { - flags |= 0x4000; - } - // Button4 -> Y - if wheel_buttons & 512 != 0 { - flags |= 0x8000; - } - - // Button5-6 to Start/Back - if wheel_buttons & 1024 != 0 { - flags |= 0x0010; - } // Start - if wheel_buttons & 2048 != 0 { - flags |= 0x0020; - } // Back - - flags - } - - /// Initialize force feedback for a wheel - /// Must be called after detect_wheels() to set up FFB motors - pub fn init_force_feedback(&self, wheel_idx: usize) -> bool { - let wheels = self.wheels.lock(); - if wheel_idx >= wheels.len() { - warn!("Cannot init FFB: wheel index {} out of range", wheel_idx); - return false; - } - - let wheel = &wheels[wheel_idx]; - - // Check if wheel has force feedback motor - match wheel.WheelMotor() { - Ok(motor) => { - info!("Wheel {} has force feedback motor", wheel_idx); - - // Check supported axes - if let Ok(axes) = motor.SupportedAxes() { - info!("FFB supported axes: {:?}", axes); - } - - // Create constant force effect - match ConstantForceEffect::new() { - Ok(effect) => { - info!("Created ConstantForceEffect for wheel {}", wheel_idx); - - let ffb_state = FfbState { - motor, - constant_effect: Some(effect), - effect_loaded: false, - }; - - self.ffb_states.lock().insert(wheel_idx, ffb_state); - true - } - Err(e) => { - error!("Failed to create ConstantForceEffect: {:?}", e); - false - } - } - } - Err(e) => { - info!( - "Wheel {} does not support force feedback: {:?}", - wheel_idx, e - ); - false - } - } - } - - /// Apply force feedback effect to a wheel - /// magnitude: -1.0 (full left) to 1.0 (full right) - /// duration_ms: effect duration in milliseconds - pub fn apply_force_feedback( - &self, - wheel_idx: usize, - effect_type: super::FfbEffectType, - magnitude: f64, - duration_ms: u16, - ) { - let mut ffb_states = self.ffb_states.lock(); - - let Some(ffb_state) = ffb_states.get_mut(&wheel_idx) else { - // Try to initialize FFB if not already done - drop(ffb_states); - if self.init_force_feedback(wheel_idx) { - // Retry after initialization - let mut ffb_states = self.ffb_states.lock(); - if let Some(ffb_state) = ffb_states.get_mut(&wheel_idx) { - self.apply_ffb_internal(ffb_state, effect_type, magnitude, duration_ms); - } - } - return; - }; - - self.apply_ffb_internal(ffb_state, effect_type, magnitude, duration_ms); - } - - /// Internal helper to apply FFB effect - fn apply_ffb_internal( - &self, - ffb_state: &mut FfbState, - effect_type: super::FfbEffectType, - magnitude: f64, - duration_ms: u16, - ) { - // Currently only support constant force effect - if effect_type != super::FfbEffectType::Constant { - debug!( - "Effect type {:?} not yet implemented, using constant force", - effect_type - ); - } - - let Some(ref effect) = ffb_state.constant_effect else { - warn!("No constant effect available"); - return; - }; - - // Clamp magnitude to valid range - let mag = magnitude.clamp(-1.0, 1.0); - - // Direction vector: X axis for steering wheel - // Positive X = force to the right, Negative X = force to the left - let direction = Vector3 { - X: mag as f32, - Y: 0.0, - Z: 0.0, - }; - - // Duration in 100-nanosecond units (TimeSpan) - let duration = TimeSpan { - Duration: (duration_ms as i64) * 10_000, // ms to 100ns - }; - - // Set effect parameters - if let Err(e) = effect.SetParameters(direction, duration) { - error!("Failed to set FFB parameters: {:?}", e); - return; - } - - // Load effect if not already loaded - if !ffb_state.effect_loaded { - // LoadEffectAsync returns an async operation - // We'll wait briefly for it to complete - match ffb_state.motor.LoadEffectAsync(effect) { - Ok(async_op) => { - // Wait a short time for the async operation to complete - std::thread::sleep(std::time::Duration::from_millis(10)); - - // Try to get results - if still pending, we'll retry next time - match async_op.GetResults() { - Ok(result) => match result { - ForceFeedbackLoadEffectResult::Succeeded => { - info!("FFB effect loaded successfully"); - ffb_state.effect_loaded = true; - } - ForceFeedbackLoadEffectResult::EffectStorageFull => { - warn!("FFB effect storage full"); - } - ForceFeedbackLoadEffectResult::EffectNotSupported => { - warn!("FFB effect not supported by device"); - } - _ => { - warn!( - "FFB effect load returned unexpected result: {:?}", - result - ); - } - }, - Err(e) => { - // May fail if still pending - will retry next time - debug!("FFB load pending or failed: {:?}", e); - } - } - } - Err(e) => { - error!("Failed to start FFB effect load: {:?}", e); - return; - } - } - } - - // Start the effect - if ffb_state.effect_loaded { - if let Err(e) = effect.Start() { - error!("Failed to start FFB effect: {:?}", e); - } - } - } - - /// Stop all force feedback effects on a wheel - pub fn stop_force_feedback(&self, wheel_idx: usize) { - let ffb_states = self.ffb_states.lock(); - - if let Some(ffb_state) = ffb_states.get(&wheel_idx) { - if let Some(ref effect) = ffb_state.constant_effect { - if let Err(e) = effect.Stop() { - debug!("Failed to stop FFB effect: {:?}", e); - } - } - - // Also stop all effects on the motor - if let Err(e) = ffb_state.motor.StopAllEffects() { - debug!("Failed to stop all FFB effects: {:?}", e); - } - } - } - - /// Stop all force feedback on all wheels - pub fn stop_all_force_feedback(&self) { - let ffb_states = self.ffb_states.lock(); - - for (idx, ffb_state) in ffb_states.iter() { - if let Some(ref effect) = ffb_state.constant_effect { - let _ = effect.Stop(); - } - let _ = ffb_state.motor.StopAllEffects(); - debug!("Stopped FFB on wheel {}", idx); - } - } - - /// Check if a wheel supports force feedback - pub fn has_force_feedback(&self, wheel_idx: usize) -> bool { - self.ffb_states.lock().contains_key(&wheel_idx) - } - - /// Stop the wheel input loop - pub fn stop(&self) { - // Stop all force feedback before stopping - self.stop_all_force_feedback(); - self.running.store(false, Ordering::SeqCst); - } - - /// Check if any wheels are connected - pub fn has_wheels(&self) -> bool { - !self.wheels.lock().is_empty() - } - - /// Get the number of connected wheels - pub fn wheel_count(&self) -> usize { - self.wheels.lock().len() - } - } - - impl Default for WheelManagerImpl { - fn default() -> Self { - Self::new() - } - } -} - -#[cfg(not(target_os = "windows"))] -mod fallback_impl { - use super::*; - - /// Fallback wheel manager for non-Windows platforms - /// Racing wheels are handled by gilrs as generic gamepads - pub struct WheelManagerImpl { - _running: Arc, - } - - impl WheelManagerImpl { - pub fn new() -> Self { - Self { - _running: Arc::new(AtomicBool::new(false)), - } - } - - pub fn set_event_sender(&self, _tx: mpsc::Sender) { - // No-op on non-Windows - } - - pub fn detect_wheels(&self) -> usize { - info!("Racing wheel detection not available on this platform - using gilrs fallback"); - 0 - } - - pub fn start(&self) { - info!("Racing wheel support uses gilrs fallback on this platform"); - } - - pub fn stop(&self) {} - - pub fn has_wheels(&self) -> bool { - false - } - - pub fn wheel_count(&self) -> usize { - 0 - } - - // Force feedback stubs for non-Windows platforms - pub fn init_force_feedback(&self, _wheel_idx: usize) -> bool { - info!("Force feedback not available on this platform"); - false - } - - pub fn apply_force_feedback( - &self, - _wheel_idx: usize, - _effect_type: super::FfbEffectType, - _magnitude: f64, - _duration_ms: u16, - ) { - // No-op on non-Windows - } - - pub fn stop_force_feedback(&self, _wheel_idx: usize) { - // No-op on non-Windows - } - - pub fn stop_all_force_feedback(&self) { - // No-op on non-Windows - } - - pub fn has_force_feedback(&self, _wheel_idx: usize) -> bool { - false - } - } - - impl Default for WheelManagerImpl { - fn default() -> Self { - Self::new() - } - } -} - -/// G29 force feedback support using the g29 crate (HID-based) -/// This works when the G29 is in PS3 mode and provides direct FFB control -mod g29_ffb { - use g29::interface::G29Interface; - use log::{debug, info}; - use parking_lot::Mutex; - use std::panic::{catch_unwind, AssertUnwindSafe}; - use std::sync::atomic::{AtomicBool, Ordering}; - - /// G29 Force Feedback Manager - /// Uses the g29 crate for HID-based force feedback control - pub struct G29FfbManager { - g29: Mutex>, - initialized: AtomicBool, - connected: AtomicBool, - } - - impl G29FfbManager { - pub fn new() -> Self { - Self { - g29: Mutex::new(None), - initialized: AtomicBool::new(false), - connected: AtomicBool::new(false), - } - } - - /// Try to initialize G29 connection - /// Returns true if G29 was found and initialized - pub fn init(&self) -> bool { - if self.initialized.load(Ordering::Relaxed) { - return self.connected.load(Ordering::Relaxed); - } - - self.initialized.store(true, Ordering::Relaxed); - - info!("Attempting to connect to Logitech G29 via HID..."); - info!("Note: G29 must be in PS3 mode (switch on wheel) for HID FFB to work"); - - // G29Interface::new() panics on failure, so we catch it - let result = catch_unwind(AssertUnwindSafe(|| G29Interface::new())); - - match result { - Ok(g29_device) => { - info!("Logitech G29 connected via HID!"); - - // Note: reset() takes 10 seconds and calibrates the wheel - // We skip it here to avoid blocking - games typically do their own calibration - - *self.g29.lock() = Some(g29_device); - self.connected.store(true, Ordering::Relaxed); - true - } - Err(_) => { - info!("G29 not found via HID (may be in PS4 mode or not connected)"); - self.connected.store(false, Ordering::Relaxed); - false - } - } - } - - /// Check if G29 is connected - pub fn is_connected(&self) -> bool { - self.connected.load(Ordering::Relaxed) - } - - /// Apply constant force feedback - /// magnitude: -1.0 (full left) to 1.0 (full right) - pub fn apply_constant_force(&self, magnitude: f64) { - let g29_lock = self.g29.lock(); - if let Some(ref g29_device) = *g29_lock { - // g29 crate uses 0.0-1.0 range - // Clamp to valid range and convert - let mag = magnitude.clamp(-1.0, 1.0); - - // The g29 crate's force_feedback_constant takes strength 0-1 - // For now we use absolute value - direction handling may need improvement - let strength = mag.abs() as f32; - - // Catch any panics from the device communication - let result = catch_unwind(AssertUnwindSafe(|| { - g29_device.force_feedback_constant(strength); - })); - - if result.is_err() { - debug!("Failed to apply G29 FFB"); - } - } - } - - /// Set autocenter strength - /// strength: 0.0 (off) to 1.0 (full) - /// rate: 0.0 (slow) to 1.0 (fast) - #[allow(dead_code)] - pub fn set_autocenter(&self, strength: f32, rate: f32) { - let g29_lock = self.g29.lock(); - if let Some(ref g29_device) = *g29_lock { - let result = catch_unwind(AssertUnwindSafe(|| { - g29_device.set_autocenter(strength.clamp(0.0, 1.0), rate.clamp(0.0, 1.0)); - })); - - if result.is_err() { - debug!("Failed to set G29 autocenter"); - } - } - } - - /// Stop all force feedback - pub fn stop(&self) { - let g29_lock = self.g29.lock(); - if let Some(ref g29_device) = *g29_lock { - // Turn off force feedback - let _ = catch_unwind(AssertUnwindSafe(|| { - g29_device.force_feedback_constant(0.0); - })); - } - } - } - - impl Default for G29FfbManager { - fn default() -> Self { - Self::new() - } - } -} - -// Re-export the appropriate implementation -#[cfg(target_os = "windows")] -pub use windows_impl::WheelManagerImpl as WheelManager; - -#[cfg(not(target_os = "windows"))] -pub use fallback_impl::WheelManagerImpl as WheelManager; - -// Export G29 FFB manager for direct use -pub use g29_ffb::G29FfbManager; diff --git a/opennow-streamer/src/input/windows.rs b/opennow-streamer/src/input/windows.rs deleted file mode 100644 index f1289ec..0000000 --- a/opennow-streamer/src/input/windows.rs +++ /dev/null @@ -1,665 +0,0 @@ -//! Windows Raw Input API -//! -//! Provides hardware-level mouse input without OS acceleration. -//! Uses WM_INPUT messages to get raw mouse deltas directly from hardware. -//! Events are coalesced (batched) every 4ms like the official GFN client -//! to prevent server-side buffering while maintaining responsiveness. - -use log::{debug, error, info}; -use parking_lot::Mutex; -use std::ffi::c_void; -use std::mem::size_of; -use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU64, Ordering}; -use tokio::sync::mpsc; - -use super::{get_timestamp_us, session_elapsed_us, MOUSE_COALESCE_INTERVAL_US}; -use crate::webrtc::InputEvent; - -// Static state -static RAW_INPUT_REGISTERED: AtomicBool = AtomicBool::new(false); -static RAW_INPUT_ACTIVE: AtomicBool = AtomicBool::new(false); -static ACCUMULATED_DX: AtomicI32 = AtomicI32::new(0); -static ACCUMULATED_DY: AtomicI32 = AtomicI32::new(0); -static MESSAGE_WINDOW: Mutex> = Mutex::new(None); -// Track if window class was registered (persists for process lifetime) -static WINDOW_CLASS_REGISTERED: AtomicBool = AtomicBool::new(false); - -// Coalescing state - accumulates events for 4ms batches (like official GFN client) -static COALESCE_DX: AtomicI32 = AtomicI32::new(0); -static COALESCE_DY: AtomicI32 = AtomicI32::new(0); -static COALESCE_LAST_SEND_US: AtomicU64 = AtomicU64::new(0); -static COALESCED_EVENT_COUNT: AtomicU64 = AtomicU64::new(0); - -// Local cursor tracking for instant visual feedback (updated on every event) -static LOCAL_CURSOR_X: AtomicI32 = AtomicI32::new(960); -static LOCAL_CURSOR_Y: AtomicI32 = AtomicI32::new(540); -static LOCAL_CURSOR_WIDTH: AtomicI32 = AtomicI32::new(1920); -static LOCAL_CURSOR_HEIGHT: AtomicI32 = AtomicI32::new(1080); - -// Direct event sender for immediate mouse events -// Using parking_lot::Mutex for fast, non-blocking access -static EVENT_SENDER: Mutex>> = Mutex::new(None); - -// Win32 types -type HWND = isize; -type WPARAM = usize; -type LPARAM = isize; -type LRESULT = isize; -type HINSTANCE = isize; -type ATOM = u16; - -// Window messages -const WM_INPUT: u32 = 0x00FF; -const WM_DESTROY: u32 = 0x0002; - -// Raw input constants -const RIDEV_REMOVE: u32 = 0x00000001; -const RID_INPUT: u32 = 0x10000003; -const RIM_TYPEMOUSE: u32 = 0; -const MOUSE_MOVE_RELATIVE: u16 = 0x00; - -// HID usage page and usage for mouse -const HID_USAGE_PAGE_GENERIC: u16 = 0x01; -const HID_USAGE_GENERIC_MOUSE: u16 = 0x02; - -// Center position for cursor recentering -static CENTER_X: AtomicI32 = AtomicI32::new(0); -static CENTER_Y: AtomicI32 = AtomicI32::new(0); - -#[repr(C)] -#[derive(Clone, Copy)] -struct RAWINPUTDEVICE { - usage_page: u16, - usage: u16, - flags: u32, - hwnd_target: HWND, -} - -#[repr(C)] -#[derive(Clone, Copy)] -struct RAWINPUTHEADER { - dw_type: u32, - dw_size: u32, - h_device: *mut c_void, - w_param: WPARAM, -} - -#[repr(C)] -#[derive(Clone, Copy)] -struct RAWMOUSE { - flags: u16, - button_flags: u16, - button_data: u16, - raw_buttons: u32, - last_x: i32, - last_y: i32, - extra_information: u32, -} - -#[repr(C)] -#[derive(Clone, Copy)] -union RAWINPUT_DATA { - mouse: RAWMOUSE, - keyboard: [u8; 24], - hid: [u8; 40], -} - -#[repr(C)] -#[derive(Clone, Copy)] -struct RAWINPUT { - header: RAWINPUTHEADER, - data: RAWINPUT_DATA, -} - -#[repr(C)] -struct WNDCLASSEXW { - cb_size: u32, - style: u32, - lpfn_wnd_proc: Option LRESULT>, - cb_cls_extra: i32, - cb_wnd_extra: i32, - h_instance: HINSTANCE, - h_icon: *mut c_void, - h_cursor: *mut c_void, - hbr_background: *mut c_void, - lpsz_menu_name: *const u16, - lpsz_class_name: *const u16, - h_icon_sm: *mut c_void, -} - -#[repr(C)] -struct MSG { - hwnd: HWND, - message: u32, - w_param: WPARAM, - l_param: LPARAM, - time: u32, - pt_x: i32, - pt_y: i32, -} - -#[repr(C)] -struct POINT { - x: i32, - y: i32, -} - -#[repr(C)] -struct RECT { - left: i32, - top: i32, - right: i32, - bottom: i32, -} - -#[link(name = "user32")] -extern "system" { - fn RegisterRawInputDevices(devices: *const RAWINPUTDEVICE, num_devices: u32, size: u32) -> i32; - fn GetRawInputData( - raw_input: *mut c_void, - command: u32, - data: *mut c_void, - size: *mut u32, - header_size: u32, - ) -> u32; - fn DefWindowProcW(hwnd: HWND, msg: u32, wparam: WPARAM, lparam: LPARAM) -> LRESULT; - fn RegisterClassExW(wc: *const WNDCLASSEXW) -> ATOM; - fn CreateWindowExW( - ex_style: u32, - class_name: *const u16, - window_name: *const u16, - style: u32, - x: i32, - y: i32, - width: i32, - height: i32, - parent: HWND, - menu: *mut c_void, - instance: HINSTANCE, - param: *mut c_void, - ) -> HWND; - fn DestroyWindow(hwnd: HWND) -> i32; - fn GetMessageW(msg: *mut MSG, hwnd: HWND, filter_min: u32, filter_max: u32) -> i32; - fn TranslateMessage(msg: *const MSG) -> i32; - fn DispatchMessageW(msg: *const MSG) -> LRESULT; - fn PostMessageW(hwnd: HWND, msg: u32, wparam: WPARAM, lparam: LPARAM) -> i32; - fn GetModuleHandleW(module_name: *const u16) -> HINSTANCE; - fn PostQuitMessage(exit_code: i32); - fn SetCursorPos(x: i32, y: i32) -> i32; - fn GetForegroundWindow() -> isize; - fn GetClientRect(hwnd: isize, rect: *mut RECT) -> i32; - fn ClientToScreen(hwnd: isize, point: *mut POINT) -> i32; -} - -/// Convert a Rust string to a null-terminated wide string -fn to_wide(s: &str) -> Vec { - s.encode_utf16().chain(std::iter::once(0)).collect() -} - -/// Update the center position based on current window -fn update_center() -> bool { - unsafe { - let hwnd = GetForegroundWindow(); - if hwnd == 0 { - return false; - } - let mut rect = RECT { - left: 0, - top: 0, - right: 0, - bottom: 0, - }; - if GetClientRect(hwnd, &mut rect) == 0 { - return false; - } - let mut center = POINT { - x: rect.right / 2, - y: rect.bottom / 2, - }; - if ClientToScreen(hwnd, &mut center) == 0 { - return false; - } - CENTER_X.store(center.x, Ordering::SeqCst); - CENTER_Y.store(center.y, Ordering::SeqCst); - true - } -} - -/// Recenter the cursor to prevent it from hitting screen edges -#[inline] -fn recenter_cursor() { - let cx = CENTER_X.load(Ordering::SeqCst); - let cy = CENTER_Y.load(Ordering::SeqCst); - if cx != 0 && cy != 0 { - unsafe { - SetCursorPos(cx, cy); - } - } -} - -/// Register for raw mouse input -fn register_raw_mouse(hwnd: HWND) -> bool { - let device = RAWINPUTDEVICE { - usage_page: HID_USAGE_PAGE_GENERIC, - usage: HID_USAGE_GENERIC_MOUSE, - flags: 0, // Only receive input when window is focused - hwnd_target: hwnd, - }; - - unsafe { RegisterRawInputDevices(&device, 1, size_of::() as u32) != 0 } -} - -/// Unregister raw mouse input -fn unregister_raw_mouse() -> bool { - let device = RAWINPUTDEVICE { - usage_page: HID_USAGE_PAGE_GENERIC, - usage: HID_USAGE_GENERIC_MOUSE, - flags: RIDEV_REMOVE, - hwnd_target: 0, - }; - - unsafe { RegisterRawInputDevices(&device, 1, size_of::() as u32) != 0 } -} - -/// Process a WM_INPUT message and extract mouse delta -fn process_raw_input(lparam: LPARAM) -> Option<(i32, i32)> { - unsafe { - // Use a properly aligned buffer for RAWINPUT struct - #[repr(C, align(8))] - struct AlignedBuffer { - data: [u8; 64], - } - - let mut buffer = AlignedBuffer { data: [0; 64] }; - let mut size: u32 = buffer.data.len() as u32; - - let result = GetRawInputData( - lparam as *mut c_void, - RID_INPUT, - buffer.data.as_mut_ptr() as *mut c_void, - &mut size, - size_of::() as u32, - ); - - if result == u32::MAX || result == 0 { - return None; - } - - // Parse the raw input - let raw = &*(buffer.data.as_ptr() as *const RAWINPUT); - - // Check if it's mouse input - if raw.header.dw_type != RIM_TYPEMOUSE { - return None; - } - - let mouse = &raw.data.mouse; - - // Only process relative mouse movement - if mouse.flags == MOUSE_MOVE_RELATIVE { - if mouse.last_x != 0 || mouse.last_y != 0 { - return Some((mouse.last_x, mouse.last_y)); - } - } - - None - } -} - -/// Flush coalesced mouse events - sends accumulated deltas if any -#[inline] -fn flush_coalesced_events() { - let dx = COALESCE_DX.swap(0, Ordering::AcqRel); - let dy = COALESCE_DY.swap(0, Ordering::AcqRel); - - if dx != 0 || dy != 0 { - let timestamp_us = get_timestamp_us(); - let now_us = session_elapsed_us(); - COALESCE_LAST_SEND_US.store(now_us, Ordering::Release); - - let guard = EVENT_SENDER.lock(); - if let Some(ref sender) = *guard { - let _ = sender.try_send(InputEvent::MouseMove { - dx: dx as i16, - dy: dy as i16, - timestamp_us, - }); - } - } -} - -/// Window procedure for the message-only window -/// Implements event coalescing: accumulates mouse deltas and sends every 4ms -/// This matches official GFN client behavior and prevents server-side buffering -unsafe extern "system" fn raw_input_wnd_proc( - hwnd: HWND, - msg: u32, - wparam: WPARAM, - lparam: LPARAM, -) -> LRESULT { - match msg { - WM_INPUT => { - if RAW_INPUT_ACTIVE.load(Ordering::SeqCst) { - if let Some((dx, dy)) = process_raw_input(lparam) { - // 1. Update local cursor IMMEDIATELY for instant visual feedback - // This happens on every event regardless of coalescing - let width = LOCAL_CURSOR_WIDTH.load(Ordering::Acquire); - let height = LOCAL_CURSOR_HEIGHT.load(Ordering::Acquire); - let old_x = LOCAL_CURSOR_X.load(Ordering::Acquire); - let old_y = LOCAL_CURSOR_Y.load(Ordering::Acquire); - LOCAL_CURSOR_X.store((old_x + dx).clamp(0, width), Ordering::Release); - LOCAL_CURSOR_Y.store((old_y + dy).clamp(0, height), Ordering::Release); - - // 2. Accumulate delta for coalescing - COALESCE_DX.fetch_add(dx, Ordering::Relaxed); - COALESCE_DY.fetch_add(dy, Ordering::Relaxed); - COALESCED_EVENT_COUNT.fetch_add(1, Ordering::Relaxed); - - // 3. Check if enough time has passed to send batch (4ms default) - let now_us = session_elapsed_us(); - let last_us = COALESCE_LAST_SEND_US.load(Ordering::Acquire); - - if now_us.saturating_sub(last_us) >= MOUSE_COALESCE_INTERVAL_US { - flush_coalesced_events(); - } - } - } - 0 - } - WM_DESTROY => { - PostQuitMessage(0); - 0 - } - _ => DefWindowProcW(hwnd, msg, wparam, lparam), - } -} - -/// Start raw input capture -pub fn start_raw_input() -> Result<(), String> { - // No cursor recentering - cursor is hidden during streaming - // Recentering causes jitter and feedback loops - - // If already registered AND active, just return success - if RAW_INPUT_REGISTERED.load(Ordering::SeqCst) { - if RAW_INPUT_ACTIVE.load(Ordering::SeqCst) { - info!("Raw input already active"); - return Ok(()); - } - // Re-activating existing registration - // Reset accumulated state before reactivating to ensure clean state - ACCUMULATED_DX.store(0, Ordering::SeqCst); - ACCUMULATED_DY.store(0, Ordering::SeqCst); - COALESCE_DX.store(0, Ordering::SeqCst); - COALESCE_DY.store(0, Ordering::SeqCst); - COALESCE_LAST_SEND_US.store(0, Ordering::SeqCst); - - RAW_INPUT_ACTIVE.store(true, Ordering::SeqCst); - info!("Raw input resumed with clean state"); - return Ok(()); - } - - // Reset all state before starting fresh - ACCUMULATED_DX.store(0, Ordering::SeqCst); - ACCUMULATED_DY.store(0, Ordering::SeqCst); - COALESCE_DX.store(0, Ordering::SeqCst); - COALESCE_DY.store(0, Ordering::SeqCst); - COALESCE_LAST_SEND_US.store(0, Ordering::SeqCst); - COALESCED_EVENT_COUNT.store(0, Ordering::SeqCst); - - // Spawn a thread to handle the message loop - std::thread::spawn(|| { - unsafe { - let class_name = to_wide("OpenNOW_RawInput_Streamer"); - let h_instance = GetModuleHandleW(std::ptr::null()); - - // Register window class only once per process lifetime - // Windows keeps class registrations until process exit - if !WINDOW_CLASS_REGISTERED.load(Ordering::SeqCst) { - let wc = WNDCLASSEXW { - cb_size: std::mem::size_of::() as u32, - style: 0, - lpfn_wnd_proc: Some(raw_input_wnd_proc), - cb_cls_extra: 0, - cb_wnd_extra: 0, - h_instance, - h_icon: std::ptr::null_mut(), - h_cursor: std::ptr::null_mut(), - hbr_background: std::ptr::null_mut(), - lpsz_menu_name: std::ptr::null(), - lpsz_class_name: class_name.as_ptr(), - h_icon_sm: std::ptr::null_mut(), - }; - - if RegisterClassExW(&wc) == 0 { - // Check if class already exists (error 1410 = CLASS_ALREADY_EXISTS) - let err = std::io::Error::last_os_error(); - if err.raw_os_error() == Some(1410) { - // Class already registered, that's fine - info!("Raw input window class already registered"); - WINDOW_CLASS_REGISTERED.store(true, Ordering::SeqCst); - } else { - error!("Failed to register raw input window class: {}", err); - return; - } - } else { - WINDOW_CLASS_REGISTERED.store(true, Ordering::SeqCst); - info!("Raw input window class registered"); - } - } - - // Create message-only window (HWND_MESSAGE = -3) - let hwnd = CreateWindowExW( - 0, - class_name.as_ptr(), - std::ptr::null(), - 0, - 0, - 0, - 0, - 0, - -3isize, // HWND_MESSAGE - std::ptr::null_mut(), - h_instance, - std::ptr::null_mut(), - ); - - if hwnd == 0 { - let err = std::io::Error::last_os_error(); - error!("Failed to create raw input window: {}", err); - return; - } - - *MESSAGE_WINDOW.lock() = Some(hwnd); - - // Register for raw mouse input - if !register_raw_mouse(hwnd) { - error!("Failed to register raw mouse input"); - DestroyWindow(hwnd); - return; - } - - RAW_INPUT_REGISTERED.store(true, Ordering::SeqCst); - RAW_INPUT_ACTIVE.store(true, Ordering::SeqCst); - info!("Raw input started - receiving hardware mouse deltas (no acceleration)"); - - // Message loop - let mut msg: MSG = std::mem::zeroed(); - while GetMessageW(&mut msg, 0, 0, 0) > 0 { - TranslateMessage(&msg); - DispatchMessageW(&msg); - } - - // Cleanup - destroy window but keep class registered - DestroyWindow(hwnd); - RAW_INPUT_REGISTERED.store(false, Ordering::SeqCst); - RAW_INPUT_ACTIVE.store(false, Ordering::SeqCst); - *MESSAGE_WINDOW.lock() = None; - info!("Raw input thread stopped"); - } - }); - - // Wait for the thread to start - std::thread::sleep(std::time::Duration::from_millis(50)); - - if RAW_INPUT_REGISTERED.load(Ordering::SeqCst) { - Ok(()) - } else { - Err("Failed to start raw input".to_string()) - } -} - -/// Pause raw input capture -pub fn pause_raw_input() { - RAW_INPUT_ACTIVE.store(false, Ordering::SeqCst); - ACCUMULATED_DX.store(0, Ordering::SeqCst); - ACCUMULATED_DY.store(0, Ordering::SeqCst); - debug!("Raw input paused"); -} - -/// Resume raw input capture -pub fn resume_raw_input() { - if RAW_INPUT_REGISTERED.load(Ordering::SeqCst) { - // Update center when resuming (window may have moved) - update_center(); - ACCUMULATED_DX.store(0, Ordering::SeqCst); - ACCUMULATED_DY.store(0, Ordering::SeqCst); - RAW_INPUT_ACTIVE.store(true, Ordering::SeqCst); - debug!("Raw input resumed"); - } -} - -/// Stop raw input completely -pub fn stop_raw_input() { - // First, deactivate to stop processing new events immediately - RAW_INPUT_ACTIVE.store(false, Ordering::SeqCst); - - // Clear the event sender FIRST to prevent any in-flight events from being sent - // to a potentially stale channel - clear_raw_input_sender(); - - // Reset all accumulated state to prevent stale deltas in next session - ACCUMULATED_DX.store(0, Ordering::SeqCst); - ACCUMULATED_DY.store(0, Ordering::SeqCst); - COALESCE_DX.store(0, Ordering::SeqCst); - COALESCE_DY.store(0, Ordering::SeqCst); - COALESCE_LAST_SEND_US.store(0, Ordering::SeqCst); - - // Reset local cursor to center for next session - let width = LOCAL_CURSOR_WIDTH.load(Ordering::Acquire); - let height = LOCAL_CURSOR_HEIGHT.load(Ordering::Acquire); - LOCAL_CURSOR_X.store(width / 2, Ordering::SeqCst); - LOCAL_CURSOR_Y.store(height / 2, Ordering::SeqCst); - - unregister_raw_mouse(); - - let guard = MESSAGE_WINDOW.lock(); - if let Some(hwnd) = *guard { - unsafe { - PostMessageW(hwnd, WM_DESTROY, 0, 0); - } - } - drop(guard); - - // Wait for the thread to actually exit (up to 1000ms with better synchronization) - // This prevents race conditions when starting a new session immediately - let start = std::time::Instant::now(); - while RAW_INPUT_REGISTERED.load(Ordering::SeqCst) { - if start.elapsed() > std::time::Duration::from_millis(1000) { - error!("Raw input thread did not exit in time, forcing reset"); - RAW_INPUT_REGISTERED.store(false, Ordering::SeqCst); - *MESSAGE_WINDOW.lock() = None; - break; - } - std::thread::sleep(std::time::Duration::from_millis(10)); - } - - // Additional safety: ensure we're fully stopped before returning - // Sleep briefly to let any pending window messages drain - std::thread::sleep(std::time::Duration::from_millis(50)); - - info!("Raw input stopped and fully cleaned up"); -} - -/// Get accumulated mouse deltas and reset -pub fn get_raw_mouse_delta() -> (i32, i32) { - let dx = ACCUMULATED_DX.swap(0, Ordering::SeqCst); - let dy = ACCUMULATED_DY.swap(0, Ordering::SeqCst); - (dx, dy) -} - -/// Check if raw input is active -pub fn is_raw_input_active() -> bool { - RAW_INPUT_ACTIVE.load(Ordering::SeqCst) -} - -/// Update center position (call when window moves/resizes) -pub fn update_raw_input_center() { - update_center(); -} - -/// Set the event sender for direct mouse event delivery -/// This allows raw input to send events directly to the streaming loop -/// for minimal latency instead of polling accumulated deltas -pub fn set_raw_input_sender(sender: mpsc::Sender) { - let mut guard = EVENT_SENDER.lock(); - *guard = Some(sender); - info!("Raw input direct sender configured"); -} - -/// Clear the event sender -pub fn clear_raw_input_sender() { - let mut guard = EVENT_SENDER.lock(); - *guard = None; -} - -/// Set local cursor dimensions (call when stream starts or resolution changes) -pub fn set_local_cursor_dimensions(width: u32, height: u32) { - LOCAL_CURSOR_WIDTH.store(width as i32, Ordering::Release); - LOCAL_CURSOR_HEIGHT.store(height as i32, Ordering::Release); - // Center cursor when dimensions change - LOCAL_CURSOR_X.store(width as i32 / 2, Ordering::Release); - LOCAL_CURSOR_Y.store(height as i32 / 2, Ordering::Release); - info!("Local cursor dimensions set to {}x{}", width, height); -} - -/// Get local cursor position (for rendering) -/// Returns (x, y) in stream coordinates -pub fn get_local_cursor_position() -> (i32, i32) { - ( - LOCAL_CURSOR_X.load(Ordering::Acquire), - LOCAL_CURSOR_Y.load(Ordering::Acquire), - ) -} - -/// Get local cursor position normalized (0.0-1.0) -pub fn get_local_cursor_normalized() -> (f32, f32) { - let x = LOCAL_CURSOR_X.load(Ordering::Acquire) as f32; - let y = LOCAL_CURSOR_Y.load(Ordering::Acquire) as f32; - let w = LOCAL_CURSOR_WIDTH.load(Ordering::Acquire) as f32; - let h = LOCAL_CURSOR_HEIGHT.load(Ordering::Acquire) as f32; - (x / w.max(1.0), y / h.max(1.0)) -} - -/// Flush any pending coalesced mouse events -/// Call this before button events to ensure proper ordering -pub fn flush_pending_mouse_events() { - flush_coalesced_events(); -} - -/// Get count of coalesced events (for stats) -pub fn get_coalesced_event_count() -> u64 { - COALESCED_EVENT_COUNT.load(Ordering::Relaxed) -} - -/// Reset coalescing state (call when streaming stops) -pub fn reset_coalescing() { - COALESCE_DX.store(0, Ordering::Release); - COALESCE_DY.store(0, Ordering::Release); - COALESCE_LAST_SEND_US.store(0, Ordering::Release); - COALESCED_EVENT_COUNT.store(0, Ordering::Release); - // Center cursor based on actual dimensions, not hardcoded values - let width = LOCAL_CURSOR_WIDTH.load(Ordering::Acquire); - let height = LOCAL_CURSOR_HEIGHT.load(Ordering::Acquire); - LOCAL_CURSOR_X.store(width / 2, Ordering::Release); - LOCAL_CURSOR_Y.store(height / 2, Ordering::Release); -} diff --git a/opennow-streamer/src/lib.rs b/opennow-streamer/src/lib.rs deleted file mode 100644 index 99b8c13..0000000 --- a/opennow-streamer/src/lib.rs +++ /dev/null @@ -1,16 +0,0 @@ -//! OpenNow Streamer Library -//! -//! Core components for the native GeForce NOW streaming client. - -#![recursion_limit = "256"] - -pub mod app; -pub mod api; -pub mod auth; -pub mod gui; -pub mod input; -pub mod media; -pub mod webrtc; -pub mod utils; - -pub use app::{App, AppState}; diff --git a/opennow-streamer/src/main.rs b/opennow-streamer/src/main.rs deleted file mode 100644 index 103ad18..0000000 --- a/opennow-streamer/src/main.rs +++ /dev/null @@ -1,665 +0,0 @@ -//! OpenNow Streamer - Native GeForce NOW Client -//! -//! A high-performance, cross-platform streaming client for GFN. - -#![recursion_limit = "256"] - -mod api; -mod app; -mod auth; -mod gui; -mod input; -mod media; -mod profiling; -mod utils; -mod webrtc; - -// Re-export profiling functions for use throughout the codebase -#[allow(unused_imports)] -pub use profiling::frame_mark; - -use anyhow::Result; -use log::info; -use parking_lot::Mutex; -use std::sync::Arc; -use winit::application::ApplicationHandler; -use winit::event::{DeviceEvent, DeviceId, ElementState, KeyEvent, Modifiers, WindowEvent}; -use winit::event_loop::{ActiveEventLoop, ControlFlow, EventLoop}; -use winit::keyboard::{Key, KeyCode, NamedKey, PhysicalKey}; -use winit::platform::scancode::PhysicalKeyExtScancode; -use winit::window::WindowId; - -use app::{App, AppState, UiAction}; -use gui::Renderer; - -/// Application handler for winit 0.30+ -struct OpenNowApp { - /// Tokio runtime handle - runtime: tokio::runtime::Handle, - /// Application state (shared) - app: Arc>, - /// Renderer (created after window is available) - renderer: Option, - /// Current modifier state - modifiers: Modifiers, - /// Track if we were streaming (for cursor lock state changes) - was_streaming: bool, -} - -/// Convert winit KeyCode to Windows Virtual Key code -fn keycode_to_vk(key: PhysicalKey) -> u16 { - match key { - PhysicalKey::Code(code) => match code { - // Letters - KeyCode::KeyA => 0x41, - KeyCode::KeyB => 0x42, - KeyCode::KeyC => 0x43, - KeyCode::KeyD => 0x44, - KeyCode::KeyE => 0x45, - KeyCode::KeyF => 0x46, - KeyCode::KeyG => 0x47, - KeyCode::KeyH => 0x48, - KeyCode::KeyI => 0x49, - KeyCode::KeyJ => 0x4A, - KeyCode::KeyK => 0x4B, - KeyCode::KeyL => 0x4C, - KeyCode::KeyM => 0x4D, - KeyCode::KeyN => 0x4E, - KeyCode::KeyO => 0x4F, - KeyCode::KeyP => 0x50, - KeyCode::KeyQ => 0x51, - KeyCode::KeyR => 0x52, - KeyCode::KeyS => 0x53, - KeyCode::KeyT => 0x54, - KeyCode::KeyU => 0x55, - KeyCode::KeyV => 0x56, - KeyCode::KeyW => 0x57, - KeyCode::KeyX => 0x58, - KeyCode::KeyY => 0x59, - KeyCode::KeyZ => 0x5A, - // Numbers - KeyCode::Digit1 => 0x31, - KeyCode::Digit2 => 0x32, - KeyCode::Digit3 => 0x33, - KeyCode::Digit4 => 0x34, - KeyCode::Digit5 => 0x35, - KeyCode::Digit6 => 0x36, - KeyCode::Digit7 => 0x37, - KeyCode::Digit8 => 0x38, - KeyCode::Digit9 => 0x39, - KeyCode::Digit0 => 0x30, - // Function keys - KeyCode::F1 => 0x70, - KeyCode::F2 => 0x71, - KeyCode::F3 => 0x72, - KeyCode::F4 => 0x73, - KeyCode::F5 => 0x74, - KeyCode::F6 => 0x75, - KeyCode::F7 => 0x76, - KeyCode::F8 => 0x77, - KeyCode::F9 => 0x78, - KeyCode::F10 => 0x79, - KeyCode::F11 => 0x7A, - KeyCode::F12 => 0x7B, - // Special keys - KeyCode::Escape => 0x1B, - KeyCode::Tab => 0x09, - KeyCode::CapsLock => 0x14, - KeyCode::ShiftLeft => 0xA0, - KeyCode::ShiftRight => 0xA1, - KeyCode::ControlLeft => 0xA2, - KeyCode::ControlRight => 0xA3, - KeyCode::AltLeft => 0xA4, - KeyCode::AltRight => 0xA5, - KeyCode::SuperLeft => 0x5B, - KeyCode::SuperRight => 0x5C, - KeyCode::Space => 0x20, - KeyCode::Enter => 0x0D, - KeyCode::Backspace => 0x08, - KeyCode::Delete => 0x2E, - KeyCode::Insert => 0x2D, - KeyCode::Home => 0x24, - KeyCode::End => 0x23, - KeyCode::PageUp => 0x21, - KeyCode::PageDown => 0x22, - // Arrow keys - KeyCode::ArrowUp => 0x26, - KeyCode::ArrowDown => 0x28, - KeyCode::ArrowLeft => 0x25, - KeyCode::ArrowRight => 0x27, - // Numpad - KeyCode::Numpad0 => 0x60, - KeyCode::Numpad1 => 0x61, - KeyCode::Numpad2 => 0x62, - KeyCode::Numpad3 => 0x63, - KeyCode::Numpad4 => 0x64, - KeyCode::Numpad5 => 0x65, - KeyCode::Numpad6 => 0x66, - KeyCode::Numpad7 => 0x67, - KeyCode::Numpad8 => 0x68, - KeyCode::Numpad9 => 0x69, - KeyCode::NumpadAdd => 0x6B, - KeyCode::NumpadSubtract => 0x6D, - KeyCode::NumpadMultiply => 0x6A, - KeyCode::NumpadDivide => 0x6F, - KeyCode::NumpadDecimal => 0x6E, - KeyCode::NumpadEnter => 0x0D, - KeyCode::NumLock => 0x90, - // Punctuation - KeyCode::Minus => 0xBD, - KeyCode::Equal => 0xBB, - KeyCode::BracketLeft => 0xDB, - KeyCode::BracketRight => 0xDD, - KeyCode::Backslash => 0xDC, - KeyCode::Semicolon => 0xBA, - KeyCode::Quote => 0xDE, - KeyCode::Backquote => 0xC0, - KeyCode::Comma => 0xBC, - KeyCode::Period => 0xBE, - KeyCode::Slash => 0xBF, - KeyCode::ScrollLock => 0x91, - KeyCode::Pause => 0x13, - KeyCode::PrintScreen => 0x2C, - _ => 0, - }, - PhysicalKey::Unidentified(_) => 0, - } -} - -impl OpenNowApp { - fn new(runtime: tokio::runtime::Handle) -> Self { - let app = Arc::new(Mutex::new(App::new(runtime.clone()))); - Self { - runtime, - app, - renderer: None, - modifiers: Modifiers::default(), - was_streaming: false, - } - } - - /// Get GFN modifier flags from current modifier state - fn get_modifier_flags(&self) -> u16 { - let state = self.modifiers.state(); - let mut flags = 0u16; - if state.shift_key() { - flags |= 0x01; - } // GFN_MOD_SHIFT - if state.control_key() { - flags |= 0x02; - } // GFN_MOD_CTRL - if state.alt_key() { - flags |= 0x04; - } // GFN_MOD_ALT - if state.super_key() { - flags |= 0x08; - } // GFN_MOD_META - flags - } -} - -impl ApplicationHandler for OpenNowApp { - fn resumed(&mut self, event_loop: &ActiveEventLoop) { - // Create renderer when window is available - if self.renderer.is_none() { - info!("Creating renderer..."); - match pollster::block_on(Renderer::new(event_loop)) { - Ok(renderer) => { - info!("Renderer initialized"); - self.renderer = Some(renderer); - } - Err(e) => { - log::error!("Failed to create renderer: {}", e); - event_loop.exit(); - } - } - } - } - - fn window_event( - &mut self, - event_loop: &ActiveEventLoop, - _window_id: WindowId, - event: WindowEvent, - ) { - let Some(renderer) = self.renderer.as_mut() else { - return; - }; - - // Let egui handle events first - let response = renderer.handle_event(&event); - - // Request redraw based on app state: - // - When streaming: always honor egui repaint (low latency needed) - // - When in session setup: always repaint (need to show progress updates) - // - When not streaming: only repaint on actual user interaction events - // (egui's request_repaint_after handles timed repaints via ControlFlow) - let app_state = self.app.lock().state; - let should_repaint = match app_state { - AppState::Streaming | AppState::Session => response.repaint, - _ => { - // Only repaint on actual input events, not egui's internal repaint requests - matches!( - event, - WindowEvent::MouseInput { .. } - | WindowEvent::MouseWheel { .. } - | WindowEvent::KeyboardInput { .. } - | WindowEvent::CursorMoved { .. } - | WindowEvent::Resized(_) - | WindowEvent::Focused(_) - ) - } - }; - - if should_repaint { - renderer.window().request_redraw(); - } - - match event { - WindowEvent::CloseRequested => { - info!("Window close requested"); - event_loop.exit(); - } - WindowEvent::Resized(size) => { - renderer.resize(size); - // Save window size to settings (only when not fullscreen) - if !renderer.is_fullscreen() && size.width > 0 && size.height > 0 { - let mut app = self.app.lock(); - app.handle_action(UiAction::UpdateWindowSize(size.width, size.height)); - } - } - // Ctrl+Shift+Q to stop streaming (instead of ESC to avoid accidental stops) - WindowEvent::KeyboardInput { - event: - KeyEvent { - physical_key: PhysicalKey::Code(KeyCode::KeyQ), - state: ElementState::Pressed, - .. - }, - .. - } if self.modifiers.state().control_key() && self.modifiers.state().shift_key() => { - let mut app = self.app.lock(); - if app.state == AppState::Streaming { - info!("Ctrl+Shift+Q pressed - terminating session"); - app.terminate_current_session(); - } - } - WindowEvent::KeyboardInput { - event: - KeyEvent { - logical_key: Key::Named(NamedKey::F11), - state: ElementState::Pressed, - .. - }, - .. - } => { - renderer.toggle_fullscreen(); - // Lock cursor when entering fullscreen during streaming - let app = self.app.lock(); - if app.state == AppState::Streaming { - if renderer.is_fullscreen() { - renderer.lock_cursor(); - } else { - renderer.unlock_cursor(); - } - } - } - WindowEvent::KeyboardInput { - event: - KeyEvent { - logical_key: Key::Named(NamedKey::F3), - state: ElementState::Pressed, - .. - }, - .. - } => { - let mut app = self.app.lock(); - app.toggle_stats(); - } - // Ctrl+Shift+F10 to toggle anti-AFK mode - WindowEvent::KeyboardInput { - event: - KeyEvent { - physical_key: PhysicalKey::Code(KeyCode::F10), - state: ElementState::Pressed, - .. - }, - .. - } if self.modifiers.state().control_key() && self.modifiers.state().shift_key() => { - let mut app = self.app.lock(); - if app.state == AppState::Streaming { - app.toggle_anti_afk(); - } - } - // F8 to toggle mouse lock during streaming (for windowed mode) - WindowEvent::KeyboardInput { - event: - KeyEvent { - logical_key: Key::Named(NamedKey::F8), - state: ElementState::Pressed, - .. - }, - .. - } => { - let mut app = self.app.lock(); - if app.state == AppState::Streaming { - // Toggle cursor capture state - app.cursor_captured = !app.cursor_captured; - - if app.cursor_captured { - renderer.lock_cursor(); - // Resume raw input when locking - #[cfg(any(target_os = "windows", target_os = "macos"))] - input::resume_raw_input(); - info!("F8: Mouse locked"); - } else { - renderer.unlock_cursor(); - // Pause raw input when unlocking - #[cfg(any(target_os = "windows", target_os = "macos"))] - input::pause_raw_input(); - info!("F8: Mouse unlocked"); - } - } - } - // Ctrl+V to paste clipboard text into remote session - WindowEvent::KeyboardInput { - event: - KeyEvent { - physical_key: PhysicalKey::Code(KeyCode::KeyV), - state: ElementState::Pressed, - .. - }, - .. - } if self.modifiers.state().control_key() && !self.modifiers.state().shift_key() => { - let app = self.app.lock(); - if app.state == AppState::Streaming && app.settings.clipboard_paste_enabled { - if let Some(ref input_handler) = app.input_handler { - info!("Ctrl+V pressed - pasting clipboard to remote session"); - let char_count = input_handler.handle_clipboard_paste(); - if char_count > 0 { - info!("Pasted {} characters to remote session", char_count); - } - } - } - } - WindowEvent::ModifiersChanged(new_modifiers) => { - self.modifiers = new_modifiers; - } - WindowEvent::KeyboardInput { event, .. } => { - // Forward keyboard input to InputHandler when streaming - let app = self.app.lock(); - if app.state == AppState::Streaming && app.cursor_captured { - // Skip key repeat events (they cause sticky keys) - if event.repeat { - return; - } - - if let Some(ref input_handler) = app.input_handler { - // Convert to Windows VK code (GFN expects VK codes, not scancodes) - let vk_code = keycode_to_vk(event.physical_key); - let pressed = event.state == ElementState::Pressed; - - // Don't include modifier flags when the key itself is a modifier - let is_modifier_key = matches!( - event.physical_key, - PhysicalKey::Code(KeyCode::ShiftLeft) - | PhysicalKey::Code(KeyCode::ShiftRight) - | PhysicalKey::Code(KeyCode::ControlLeft) - | PhysicalKey::Code(KeyCode::ControlRight) - | PhysicalKey::Code(KeyCode::AltLeft) - | PhysicalKey::Code(KeyCode::AltRight) - | PhysicalKey::Code(KeyCode::SuperLeft) - | PhysicalKey::Code(KeyCode::SuperRight) - ); - let modifiers = if is_modifier_key { - 0 - } else { - self.get_modifier_flags() - }; - - // Only send if we have a valid VK code - if vk_code != 0 { - input_handler.handle_key(vk_code, pressed, modifiers); - } - } - } - } - WindowEvent::Focused(focused) => { - let mut app = self.app.lock(); - if app.state == AppState::Streaming { - if !focused { - // Lost focus - release all keys to prevent sticky keys - if let Some(ref input_handler) = app.input_handler { - log::info!("Window lost focus - releasing all keys"); - input_handler.release_all_keys(); - } - // Pause raw input while unfocused - #[cfg(any(target_os = "windows", target_os = "macos"))] - input::pause_raw_input(); - } else { - // Regained focus - re-lock cursor if it was captured - if app.cursor_captured { - log::info!("Window regained focus - re-locking cursor"); - renderer.lock_cursor(); - // Resume raw input - #[cfg(any(target_os = "windows", target_os = "macos"))] - input::resume_raw_input(); - - // Request keyframe to recover video stream after focus loss - // This prevents freeze caused by corrupted NAL data during unfocused state - let runtime = self.runtime.clone(); - runtime.spawn(async { - log::info!("Requesting keyframe after focus regain"); - webrtc::request_keyframe().await; - }); - } - } - } - } - WindowEvent::MouseWheel { delta, .. } => { - let app = self.app.lock(); - if app.state == AppState::Streaming { - if let Some(ref input_handler) = app.input_handler { - let wheel_delta = match delta { - winit::event::MouseScrollDelta::LineDelta(_, y) => (y * 120.0) as i16, - winit::event::MouseScrollDelta::PixelDelta(pos) => pos.y as i16, - }; - input_handler.handle_wheel(wheel_delta); - } - } - } - WindowEvent::RedrawRequested => { - // Mark frame for Tracy profiler (if enabled) - profiling::frame_mark(); - - let mut app_guard = self.app.lock(); - let is_streaming = app_guard.state == AppState::Streaming; - - // Check for streaming state change to lock/unlock cursor and start/stop raw input - if is_streaming && !self.was_streaming { - // Just started streaming - lock cursor, start raw input, disable vsync - renderer.lock_cursor(); - renderer.set_vsync(false); // Immediate mode for lowest latency - self.was_streaming = true; - - // Start Raw Input for unaccelerated mouse movement (Windows/macOS) - #[cfg(any(target_os = "windows", target_os = "macos"))] - { - match input::start_raw_input() { - Ok(()) => info!("Raw input enabled - mouse acceleration disabled"), - Err(e) => log::warn!( - "Failed to start raw input: {} - using winit fallback", - e - ), - } - } - } else if !is_streaming && self.was_streaming { - // Just stopped streaming - unlock cursor, stop raw input, enable vsync - renderer.unlock_cursor(); - renderer.set_vsync(true); // VSync for low CPU usage in UI - self.was_streaming = false; - - // Stop raw input - #[cfg(any(target_os = "windows", target_os = "macos"))] - { - input::stop_raw_input(); - } - } - - app_guard.update(); - - match renderer.render(&app_guard) { - Ok((actions, repaint_after)) => { - // Apply UI actions to app state - for action in actions { - app_guard.handle_action(action); - } - - // Schedule next repaint based on egui's request - // This enables idle throttling (e.g., 10 FPS when not interacting) - if !is_streaming { - if let Some(delay) = repaint_after { - if !delay.is_zero() { - // Schedule a repaint after the delay - let wake_time = std::time::Instant::now() + delay; - event_loop.set_control_flow( - winit::event_loop::ControlFlow::WaitUntil(wake_time), - ); - } - } - } - } - Err(e) => { - log::error!("Render error: {}", e); - } - } - - drop(app_guard); - - // Don't request redraw here - let about_to_wait handle frame pacing - // This ensures render rate matches decode rate (e.g., 120fps) - // Previously this caused double the frame rate (240+ fps) because - // both RedrawRequested and about_to_wait were requesting redraws - } - WindowEvent::MouseInput { state, button, .. } => { - let app = self.app.lock(); - if app.state == AppState::Streaming { - if let Some(ref input_handler) = app.input_handler { - input_handler.handle_mouse_button(button, state); - } - } - } - WindowEvent::CursorMoved { position, .. } => { - let app = self.app.lock(); - if app.state == AppState::Streaming { - if let Some(ref input_handler) = app.input_handler { - input_handler.handle_cursor_move(position.x, position.y); - } - } - } - _ => {} - } - } - - fn device_event( - &mut self, - _event_loop: &ActiveEventLoop, - _device_id: DeviceId, - event: DeviceEvent, - ) { - // Only use winit's MouseMotion as fallback when raw input is not active - #[cfg(any(target_os = "windows", target_os = "macos"))] - if input::is_raw_input_active() { - return; // Raw input handles mouse movement - } - - if let DeviceEvent::MouseMotion { delta } = event { - let app = self.app.lock(); - if app.state == AppState::Streaming && app.cursor_captured { - if let Some(ref input_handler) = app.input_handler { - input_handler.handle_mouse_delta(delta.0 as i16, delta.1 as i16); - } - } - } - } - - fn about_to_wait(&mut self, _event_loop: &ActiveEventLoop) { - let Some(ref renderer) = self.renderer else { - return; - }; - - let app_guard = self.app.lock(); - let app_state = app_guard.state; - // Check if there's a new frame from the decoder before requesting redraw - // This prevents rendering faster than decode rate, saving GPU cycles - let has_new_frame = app_guard - .shared_frame - .as_ref() - .map(|sf| sf.has_new_frame()) - .unwrap_or(false); - drop(app_guard); - - // Dynamically switch control flow based on app state - // Poll during streaming for lowest latency, Wait for menus to save CPU - match app_state { - AppState::Streaming => { - _event_loop.set_control_flow(ControlFlow::Poll); - // Only request redraw when decoder has produced a new frame - // This synchronizes render rate to decode rate, avoiding wasted GPU cycles - if has_new_frame { - renderer.window().request_redraw(); - } - } - AppState::Session => { - // During session setup, poll at a reasonable rate (30 FPS) to show progress - // This ensures status updates are visible without wasting CPU - let wake_time = std::time::Instant::now() + std::time::Duration::from_millis(33); - _event_loop.set_control_flow(ControlFlow::WaitUntil(wake_time)); - renderer.window().request_redraw(); - } - _ => { - _event_loop.set_control_flow(ControlFlow::Wait); - // When not streaming, rely entirely on event-driven redraws - // ControlFlow::Wait will block until an event arrives - // This reduces CPU usage from 100% to <5% when idle - } - } - } -} - -fn main() -> Result<()> { - // Initialize profiling (Tracy) if enabled - // Build with: cargo build --release --features tracy - // Returns true if it initialized logging (we should skip env_logger) - let profiling_initialized_logging = profiling::init(); - - // Initialize logging (only if profiling didn't already set it up) - if !profiling_initialized_logging { - env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); - } - - info!("OpenNow Streamer v{}", env!("CARGO_PKG_VERSION")); - info!("Platform: {}", std::env::consts::OS); - - #[cfg(feature = "tracy")] - info!("Tracy profiler ENABLED - connect with Tracy Profiler application"); - - // Create tokio runtime for async operations - let runtime = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build()?; - - // Create event loop - let event_loop = EventLoop::new()?; - // Use Wait by default for low CPU usage in menus - // Dynamically switch to Poll during active streaming for lowest latency - event_loop.set_control_flow(ControlFlow::Wait); - - // Create application handler - let mut app = OpenNowApp::new(runtime.handle().clone()); - - // Run event loop with application handler - event_loop.run_app(&mut app)?; - - Ok(()) -} diff --git a/opennow-streamer/src/media/audio.rs b/opennow-streamer/src/media/audio.rs deleted file mode 100644 index 2a8d37c..0000000 --- a/opennow-streamer/src/media/audio.rs +++ /dev/null @@ -1,1246 +0,0 @@ -//! Audio Decoder and Player -//! -//! Decode Opus audio and play through cpal. -//! - macOS: Uses FFmpeg for Opus decoding -//! - Linux/Windows: Uses GStreamer for Opus decoding -//! Optimized for low-latency streaming with jitter buffer. -//! Supports dynamic device switching and sample rate conversion. - -use anyhow::{anyhow, Context, Result}; -use log::{debug, error, info, warn}; -use parking_lot::Mutex; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::sync::mpsc; -use std::sync::Arc; -use std::thread; - -// ============================================================================ -// macOS: FFmpeg-based Opus decoder -// ============================================================================ - -#[cfg(target_os = "macos")] -extern crate ffmpeg_next as ffmpeg; - -#[cfg(target_os = "macos")] -use ffmpeg::codec::{context::Context as CodecContext, decoder}; -#[cfg(target_os = "macos")] -use ffmpeg::Packet; - -/// Audio decoder - platform-specific implementation -/// Non-blocking: decoded samples are sent to a channel -pub struct AudioDecoder { - cmd_tx: mpsc::Sender, - /// For async decoding - samples come out here - sample_rx: Option>>, - sample_rate: u32, - channels: u32, -} - -enum AudioCommand { - /// Decode audio and send result to channel - DecodeAsync(Vec), - Stop, -} - -// ============================================================================ -// macOS implementation using FFmpeg -// ============================================================================ - -#[cfg(target_os = "macos")] -impl AudioDecoder { - /// Create a new Opus audio decoder using FFmpeg (macOS) - /// Returns decoder and a receiver for decoded samples (for async operation) - pub fn new(sample_rate: u32, channels: u32) -> Result { - info!( - "Creating Opus audio decoder (FFmpeg): {}Hz, {} channels", - sample_rate, channels - ); - - // Initialize FFmpeg (may already be initialized by video decoder) - let _ = ffmpeg::init(); - - // Create channels for thread communication - let (cmd_tx, cmd_rx) = mpsc::channel::(); - // Async channel for decoded samples - large buffer to prevent blocking - let (sample_tx, sample_rx) = tokio::sync::mpsc::channel::>(512); - - // Spawn decoder thread (FFmpeg types are not Send) - let sample_rate_clone = sample_rate; - let channels_clone = channels; - - thread::spawn(move || { - // Find Opus decoder - let codec = match ffmpeg::codec::decoder::find(ffmpeg::codec::Id::OPUS) { - Some(c) => c, - None => { - error!("Opus decoder not found in FFmpeg"); - return; - } - }; - - let ctx = CodecContext::new_with_codec(codec); - - let mut decoder = match ctx.decoder().audio() { - Ok(d) => d, - Err(e) => { - error!("Failed to create Opus decoder: {:?}", e); - return; - } - }; - - info!("Opus audio decoder initialized (FFmpeg, async mode)"); - - while let Ok(cmd) = cmd_rx.recv() { - match cmd { - AudioCommand::DecodeAsync(data) => { - let samples = Self::decode_opus_packet( - &mut decoder, - &data, - sample_rate_clone, - channels_clone, - ); - if !samples.is_empty() { - // Non-blocking send - drop samples if channel is full - let _ = sample_tx.try_send(samples); - } - } - AudioCommand::Stop => break, - } - } - - debug!("Audio decoder thread stopped"); - }); - - Ok(Self { - cmd_tx, - sample_rx: Some(sample_rx), - sample_rate, - channels, - }) - } - - /// Take the sample receiver (for passing to audio player thread) - pub fn take_sample_receiver(&mut self) -> Option>> { - self.sample_rx.take() - } - - /// Decode an Opus packet from RTP payload - fn decode_opus_packet( - decoder: &mut decoder::Audio, - data: &[u8], - target_sample_rate: u32, - target_channels: u32, - ) -> Vec { - if data.is_empty() { - return Vec::new(); - } - - // Create packet from raw Opus data - let mut packet = Packet::new(data.len()); - if let Some(pkt_data) = packet.data_mut() { - pkt_data.copy_from_slice(data); - } else { - return Vec::new(); - } - - // Send packet to decoder - if let Err(e) = decoder.send_packet(&packet) { - match e { - ffmpeg::Error::Other { errno } if errno == libc::EAGAIN => {} - _ => debug!("Audio send packet error: {:?}", e), - } - } - - // Receive decoded audio frame - let mut frame = ffmpeg::frame::Audio::empty(); - match decoder.receive_frame(&mut frame) { - Ok(_) => { - // Convert frame to i16 samples - let samples = Self::frame_to_samples(&frame, target_sample_rate, target_channels); - samples - } - Err(ffmpeg::Error::Other { errno }) if errno == libc::EAGAIN => Vec::new(), - Err(e) => { - debug!("Audio receive frame error: {:?}", e); - Vec::new() - } - } - } - - /// Convert FFmpeg audio frame to i16 samples - fn frame_to_samples( - frame: &ffmpeg::frame::Audio, - _target_sample_rate: u32, - target_channels: u32, - ) -> Vec { - use ffmpeg::format::Sample; - - let nb_samples = frame.samples(); - let channels = frame.channels() as usize; - - if nb_samples == 0 || channels == 0 { - return Vec::new(); - } - - let format = frame.format(); - let mut output = Vec::with_capacity(nb_samples * target_channels as usize); - - // Handle different sample formats - match format { - Sample::I16(planar) => { - if planar == ffmpeg::format::sample::Type::Planar { - // Planar format - interleave channels - for i in 0..nb_samples { - for ch in 0..channels.min(target_channels as usize) { - let plane = frame.plane::(ch); - if i < plane.len() { - output.push(plane[i]); - } - } - // Fill remaining channels with zeros if needed - for _ in channels..target_channels as usize { - output.push(0); - } - } - } else { - // Packed format - already interleaved - let data = frame.plane::(0); - output.extend_from_slice(&data[..nb_samples * channels]); - } - } - Sample::F32(planar) => { - // Convert f32 to i16 - if planar == ffmpeg::format::sample::Type::Planar { - for i in 0..nb_samples { - for ch in 0..channels.min(target_channels as usize) { - let plane = frame.plane::(ch); - if i < plane.len() { - let sample = (plane[i] * 32767.0).clamp(-32768.0, 32767.0) as i16; - output.push(sample); - } - } - for _ in channels..target_channels as usize { - output.push(0); - } - } - } else { - let data = frame.plane::(0); - for sample in &data[..nb_samples * channels] { - let s = (*sample * 32767.0).clamp(-32768.0, 32767.0) as i16; - output.push(s); - } - } - } - _ => { - // For other formats, try to get as bytes and convert - debug!("Unsupported audio format: {:?}, returning silence", format); - output.resize(nb_samples * target_channels as usize, 0); - } - } - - output - } - - /// Decode an Opus packet asynchronously (non-blocking, fire-and-forget) - /// Decoded samples are sent to the sample_rx channel - pub fn decode_async(&self, data: &[u8]) { - let _ = self.cmd_tx.send(AudioCommand::DecodeAsync(data.to_vec())); - } - - /// Get sample rate - pub fn sample_rate(&self) -> u32 { - self.sample_rate - } - - /// Get channel count - pub fn channels(&self) -> u32 { - self.channels - } -} - -#[cfg(target_os = "macos")] -impl Drop for AudioDecoder { - fn drop(&mut self) { - let _ = self.cmd_tx.send(AudioCommand::Stop); - } -} - -// ============================================================================ -// Linux/Windows x64 implementation using GStreamer -// ============================================================================ - -#[cfg(any(target_os = "linux", all(windows, target_arch = "x86_64")))] -impl AudioDecoder { - /// Create a new Opus audio decoder using GStreamer (Linux/Windows x64) - /// Returns decoder and a receiver for decoded samples (for async operation) - pub fn new(sample_rate: u32, channels: u32) -> Result { - use gstreamer as gst; - use gstreamer::prelude::*; - use gstreamer_app as gst_app; - - info!( - "Creating Opus audio decoder (GStreamer): {}Hz, {} channels", - sample_rate, channels - ); - - // Initialize GStreamer (uses bundled runtime on Windows) - super::init_gstreamer()?; - - // Create channels for thread communication - let (cmd_tx, cmd_rx) = mpsc::channel::(); - // Async channel for decoded samples - large buffer to prevent blocking - let (sample_tx, sample_rx) = tokio::sync::mpsc::channel::>(512); - - let sample_rate_clone = sample_rate; - let channels_clone = channels; - - thread::spawn(move || { - // Build GStreamer pipeline for Opus decoding - // Use opusparse to properly frame raw Opus packets from WebRTC - // The pipeline: appsrc -> opusparse -> opusdec -> audioconvert -> audioresample -> appsink - let pipeline_str = format!( - "appsrc name=src format=time do-timestamp=true ! \ - opusparse ! \ - opusdec plc=true ! \ - audioconvert ! \ - audioresample ! \ - audio/x-raw,format=S16LE,rate={},channels={} ! \ - appsink name=sink emit-signals=true sync=false", - sample_rate_clone, channels_clone - ); - - let pipeline = match gst::parse::launch(&pipeline_str) { - Ok(p) => p.downcast::().unwrap(), - Err(e) => { - error!("Failed to create GStreamer audio pipeline: {}", e); - error!("Make sure gstopus.dll and gstaudioconvert.dll plugins are present"); - return; - } - }; - - let appsrc = pipeline - .by_name("src") - .unwrap() - .downcast::() - .unwrap(); - - let appsink = pipeline - .by_name("sink") - .unwrap() - .downcast::() - .unwrap(); - - // Configure appsrc for raw Opus packets - // channel-mapping-family=0 means RTP mapping (stereo) - let caps = gst::Caps::builder("audio/x-opus") - .field("rate", sample_rate_clone as i32) - .field("channels", channels_clone as i32) - .field("channel-mapping-family", 0i32) - .build(); - appsrc.set_caps(Some(&caps)); - appsrc.set_format(gst::Format::Time); - - // Enable live mode for low latency - appsrc.set_is_live(true); - appsrc.set_max_bytes(64 * 1024); // 64KB max buffer - - // Set up appsink callback - let sample_tx_clone = sample_tx.clone(); - use std::sync::atomic::{AtomicU64, Ordering}; - static DECODED_SAMPLE_COUNT: AtomicU64 = AtomicU64::new(0); - appsink.set_callbacks( - gst_app::AppSinkCallbacks::builder() - .new_sample(move |sink| { - if let Ok(sample) = sink.pull_sample() { - if let Some(buffer) = sample.buffer() { - if let Ok(map) = buffer.map_readable() { - // Convert bytes to i16 samples - let bytes = map.as_slice(); - let samples: Vec = bytes - .chunks_exact(2) - .map(|chunk| i16::from_le_bytes([chunk[0], chunk[1]])) - .collect(); - - if !samples.is_empty() { - let count = DECODED_SAMPLE_COUNT - .fetch_add(samples.len() as u64, Ordering::Relaxed); - if count == 0 { - log::info!( - "First audio samples decoded: {} samples", - samples.len() - ); - } - let _ = sample_tx_clone.try_send(samples); - } - } - } - } - Ok(gst::FlowSuccess::Ok) - }) - .build(), - ); - - // Start pipeline - if let Err(e) = pipeline.set_state(gst::State::Playing) { - error!("Failed to start GStreamer audio pipeline: {:?}", e); - return; - } - - // Check for pipeline errors on bus - let bus = pipeline.bus().unwrap(); - std::thread::spawn(move || { - for msg in bus.iter_timed(gst::ClockTime::NONE) { - use gst::MessageView; - match msg.view() { - MessageView::Error(err) => { - error!("GStreamer audio error: {} ({:?})", err.error(), err.debug()); - } - MessageView::Warning(warn) => { - warn!( - "GStreamer audio warning: {} ({:?})", - warn.error(), - warn.debug() - ); - } - MessageView::Eos(..) => { - debug!("GStreamer audio EOS"); - break; - } - _ => {} - } - } - }); - - info!("Opus audio decoder initialized (GStreamer, async mode)"); - - let mut packets_pushed = 0u64; - while let Ok(cmd) = cmd_rx.recv() { - match cmd { - AudioCommand::DecodeAsync(data) => { - if !data.is_empty() { - let data_len = data.len(); - // Push Opus packet to GStreamer pipeline - let buffer = gst::Buffer::from_slice(data); - match appsrc.push_buffer(buffer) { - Ok(_) => { - packets_pushed += 1; - if packets_pushed == 1 { - info!("First Opus packet pushed to GStreamer pipeline: {} bytes", data_len); - } else if packets_pushed % 1000 == 0 { - debug!("Audio packets pushed: {}", packets_pushed); - } - } - Err(e) => { - warn!("Failed to push audio buffer: {:?}", e); - } - } - } - } - AudioCommand::Stop => break, - } - } - - // Cleanup - let _ = appsrc.end_of_stream(); - let _ = pipeline.set_state(gst::State::Null); - debug!("Audio decoder thread stopped"); - }); - - Ok(Self { - cmd_tx, - sample_rx: Some(sample_rx), - sample_rate, - channels, - }) - } - - /// Take the sample receiver (for passing to audio player thread) - pub fn take_sample_receiver(&mut self) -> Option>> { - self.sample_rx.take() - } - - /// Decode an Opus packet asynchronously (non-blocking, fire-and-forget) - /// Decoded samples are sent to the sample_rx channel - pub fn decode_async(&self, data: &[u8]) { - let _ = self.cmd_tx.send(AudioCommand::DecodeAsync(data.to_vec())); - } - - /// Get sample rate - pub fn sample_rate(&self) -> u32 { - self.sample_rate - } - - /// Get channel count - pub fn channels(&self) -> u32 { - self.channels - } -} - -#[cfg(any(target_os = "linux", all(windows, target_arch = "x86_64")))] -impl Drop for AudioDecoder { - fn drop(&mut self) { - let _ = self.cmd_tx.send(AudioCommand::Stop); - } -} - -// ============================================================================ -// Windows ARM64 - No audio decoding (GStreamer not available) -// ============================================================================ - -#[cfg(all(windows, target_arch = "aarch64"))] -impl AudioDecoder { - /// Create a stub audio decoder for Windows ARM64 - /// Note: GStreamer ARM64 binaries are not available, so audio is disabled - pub fn new(sample_rate: u32, channels: u32) -> Result { - warn!( - "Audio decoding not available on Windows ARM64 (GStreamer not available). \ - Audio will be silent. Sample rate: {}Hz, channels: {}", - sample_rate, channels - ); - - let (cmd_tx, _cmd_rx) = mpsc::channel::(); - let (_sample_tx, sample_rx) = tokio::sync::mpsc::channel::>(1); - - Ok(Self { - cmd_tx, - sample_rx: Some(sample_rx), - sample_rate, - channels, - }) - } - - /// Take the sample receiver (for passing to audio player thread) - pub fn take_sample_receiver(&mut self) -> Option>> { - self.sample_rx.take() - } - - /// Decode an Opus packet asynchronously - stub that does nothing on ARM64 - pub fn decode_async(&self, _data: &[u8]) { - // No-op: audio not supported on Windows ARM64 - } - - /// Get sample rate - pub fn sample_rate(&self) -> u32 { - self.sample_rate - } - - /// Get channel count - pub fn channels(&self) -> u32 { - self.channels - } -} - -#[cfg(all(windows, target_arch = "aarch64"))] -impl Drop for AudioDecoder { - fn drop(&mut self) { - let _ = self.cmd_tx.send(AudioCommand::Stop); - } -} - -/// Audio player using cpal with optimized lock-free-ish ring buffer -/// Supports sample rate conversion, channel upmixing, and dynamic device switching -pub struct AudioPlayer { - /// Input sample rate (from decoder, typically 48000Hz) - input_sample_rate: u32, - /// Output sample rate (device native rate) - output_sample_rate: u32, - /// Input channel count (from decoder, typically 2 for stereo) - input_channels: u32, - /// Output channel count (device channels, may be 8 for 7.1 headsets) - output_channels: u32, - buffer: Arc, - stream: Arc>>, - /// Flag to indicate stream needs recreation (device change) - needs_restart: Arc, - /// Current device name for change detection - current_device_name: Arc>, - /// Resampler state for rate conversion and channel upmixing - resampler: Arc>, -} - -/// High-quality audio resampler using Catmull-Rom spline interpolation -/// This provides much better quality than linear interpolation, especially for 2x upsampling -/// Also handles channel upmixing (e.g., stereo to 7.1 surround) -struct AudioResampler { - input_rate: u32, - output_rate: u32, - input_channels: u32, - output_channels: u32, - /// Fractional sample position for interpolation - phase: f64, - /// History buffer for 4-point interpolation (per input channel) - /// Stores [s_minus1, s0, s1, s2] for each channel - history: Vec<[i16; 4]>, -} - -/// Lock-free ring buffer for audio samples -/// Uses atomic indices for read/write positions to minimize lock contention -pub struct AudioRingBuffer { - samples: Mutex>, - read_pos: AtomicUsize, - write_pos: AtomicUsize, - capacity: usize, -} - -impl AudioRingBuffer { - fn new(capacity: usize) -> Self { - Self { - samples: Mutex::new(vec![0i16; capacity]), - read_pos: AtomicUsize::new(0), - write_pos: AtomicUsize::new(0), - capacity, - } - } - - fn available(&self) -> usize { - let write = self.write_pos.load(Ordering::Acquire); - let read = self.read_pos.load(Ordering::Acquire); - if write >= read { - write - read - } else { - self.capacity - read + write - } - } - - fn free_space(&self) -> usize { - self.capacity - 1 - self.available() - } - - /// Write samples to buffer (called from decoder thread) - fn write(&self, data: &[i16]) { - let mut samples = self.samples.lock(); - let mut write_pos = self.write_pos.load(Ordering::Acquire); - let read_pos = self.read_pos.load(Ordering::Acquire); - - for &sample in data { - let next_pos = (write_pos + 1) % self.capacity; - // Don't overwrite unread data - if next_pos != read_pos { - samples[write_pos] = sample; - write_pos = next_pos; - } else { - // Buffer full - drop remaining samples - break; - } - } - - self.write_pos.store(write_pos, Ordering::Release); - } - - /// Read samples from buffer (called from audio callback - must be fast!) - fn read(&self, out: &mut [i16]) { - let samples = self.samples.lock(); - let write_pos = self.write_pos.load(Ordering::Acquire); - let mut read_pos = self.read_pos.load(Ordering::Acquire); - - for sample in out.iter_mut() { - if read_pos == write_pos { - *sample = 0; // Underrun - output silence - } else { - *sample = samples[read_pos]; - read_pos = (read_pos + 1) % self.capacity; - } - } - - self.read_pos.store(read_pos, Ordering::Release); - } -} - -impl AudioResampler { - fn new(input_rate: u32, output_rate: u32, input_channels: u32, output_channels: u32) -> Self { - info!( - "Audio resampler: {}Hz {}ch -> {}Hz {}ch", - input_rate, input_channels, output_rate, output_channels - ); - Self { - input_rate, - output_rate, - input_channels, - output_channels, - phase: 0.0, - // Initialize history with zeros for each input channel - history: vec![[0i16; 4]; input_channels as usize], - } - } - - /// Resample audio using Catmull-Rom spline interpolation (4-point) - /// This provides much better quality than linear interpolation - /// The Catmull-Rom spline passes through all control points and provides - /// smooth C1 continuous curves, ideal for audio resampling - /// Also handles channel upmixing (stereo -> multi-channel) - fn resample(&mut self, input: &[i16]) -> Vec { - let in_ch = self.input_channels as usize; - let out_ch = self.output_channels as usize; - let input_frames = input.len() / in_ch; - - if input_frames == 0 { - return Vec::new(); - } - - // Calculate output frame count based on sample rate ratio - let ratio = self.input_rate as f64 / self.output_rate as f64; - let output_frames = if self.input_rate == self.output_rate { - input_frames - } else { - ((input_frames as f64) / ratio).ceil() as usize - }; - - let mut output = Vec::with_capacity(output_frames * out_ch); - - // Ensure history is properly sized for input channels - if self.history.len() != in_ch { - self.history = vec![[0i16; 4]; in_ch]; - } - - for _ in 0..output_frames { - let input_idx = self.phase as usize; - let frac = self.phase - input_idx as f64; - - // First, get the resampled stereo frame - let mut stereo_frame = [0i16; 2]; - - for ch in 0..in_ch.min(2) { - // Get 4 samples for Catmull-Rom interpolation: s[-1], s[0], s[1], s[2] - let get_sample = |frame_idx: isize| -> i16 { - if frame_idx < 0 { - // Use history for samples before current buffer - let hist_idx = (4 + frame_idx) as usize; - self.history[ch][hist_idx.min(3)] - } else if (frame_idx as usize) < input_frames { - input[frame_idx as usize * in_ch + ch] - } else { - // Clamp to last sample - if input_frames > 0 { - input[(input_frames - 1) * in_ch + ch] - } else { - self.history[ch][3] // Use last known sample - } - } - }; - - let s0 = get_sample(input_idx as isize - 1) as f64; - let s1 = get_sample(input_idx as isize) as f64; - let s2 = get_sample(input_idx as isize + 1) as f64; - let s3 = get_sample(input_idx as isize + 2) as f64; - - // Catmull-Rom spline interpolation formula - // This provides C1 continuity (smooth first derivative) - let t = frac; - let t2 = t * t; - let t3 = t2 * t; - - let interpolated = 0.5 - * ((2.0 * s1) - + (-s0 + s2) * t - + (2.0 * s0 - 5.0 * s1 + 4.0 * s2 - s3) * t2 - + (-s0 + 3.0 * s1 - 3.0 * s2 + s3) * t3); - - stereo_frame[ch] = interpolated.clamp(-32768.0, 32767.0) as i16; - } - - // Now upmix stereo to output channel count - // Standard channel mapping for common configurations: - // 2ch: FL, FR - // 6ch (5.1): FL, FR, FC, LFE, BL, BR - // 8ch (7.1): FL, FR, FC, LFE, BL, BR, SL, SR - match out_ch { - 1 => { - // Mono: mix L+R - let mono = ((stereo_frame[0] as i32 + stereo_frame[1] as i32) / 2) as i16; - output.push(mono); - } - 2 => { - // Stereo: swap L/R channels (GFN sends them inverted) - output.push(stereo_frame[1]); // Right -> Left - output.push(stereo_frame[0]); // Left -> Right - } - _ => { - // Multi-channel (5.1, 7.1, etc.) - // Standard layout: FL, FR, FC, LFE, BL, BR, [SL, SR for 7.1+] - // Swap L/R channels (GFN sends them inverted) - let left = stereo_frame[1]; - let right = stereo_frame[0]; - - for ch_idx in 0..out_ch { - let sample = match ch_idx { - 0 => left, // Front Left - 1 => right, // Front Right - 2 => { - // Center - mix of L+R at reduced level - ((left as i32 + right as i32) / 3) as i16 - } - 3 => 0, // LFE - no bass routing for now - 4 => { - // Back/Rear Left - copy of front left at reduced level - (left as i32 * 2 / 3) as i16 - } - 5 => { - // Back/Rear Right - copy of front right at reduced level - (right as i32 * 2 / 3) as i16 - } - 6 => { - // Side Left (7.1) - copy of front left at reduced level - (left as i32 / 2) as i16 - } - 7 => { - // Side Right (7.1) - copy of front right at reduced level - (right as i32 / 2) as i16 - } - _ => 0, // Any additional channels: silence - }; - output.push(sample); - } - } - } - - self.phase += ratio; - } - - // Update history with last 4 samples from this buffer for next call - if input_frames >= 4 { - for ch in 0..in_ch { - for i in 0..4 { - self.history[ch][i] = input[(input_frames - 4 + i) * in_ch + ch]; - } - } - } else if input_frames > 0 { - // Shift history and add new samples - for ch in 0..in_ch { - let shift = 4 - input_frames; - for i in 0..shift { - self.history[ch][i] = self.history[ch][i + input_frames]; - } - for i in 0..input_frames { - self.history[ch][shift + i] = input[i * in_ch + ch]; - } - } - } - - // Keep fractional phase, reset integer part - self.phase = self.phase.fract(); - - output - } - - /// Update output rate and channels (for device change) - fn set_output_config(&mut self, output_rate: u32, output_channels: u32) { - if self.output_rate != output_rate || self.output_channels != output_channels { - self.output_rate = output_rate; - self.output_channels = output_channels; - self.phase = 0.0; - // Reset history on config change - for hist in &mut self.history { - *hist = [0i16; 4]; - } - info!( - "Resampler updated: {}Hz {}ch -> {}Hz {}ch", - self.input_rate, self.input_channels, output_rate, output_channels - ); - } - } -} - -impl AudioPlayer { - /// Create a new audio player - pub fn new(sample_rate: u32, channels: u32) -> Result { - use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; - use cpal::SampleFormat; - - info!( - "Creating audio player: {}Hz, {} channels", - sample_rate, channels - ); - - let host = cpal::default_host(); - - let device = host - .default_output_device() - .context("No audio output device found")?; - - info!("Using audio device: {}", device.name().unwrap_or_default()); - - // Query supported configurations - let supported_configs: Vec<_> = device - .supported_output_configs() - .map(|configs| configs.collect()) - .unwrap_or_default(); - - if supported_configs.is_empty() { - return Err(anyhow!("No supported audio configurations found")); - } - - // Log available configurations for debugging - for cfg in &supported_configs { - debug!( - "Supported config: {:?} channels, {:?}-{:?} Hz, format {:?}", - cfg.channels(), - cfg.min_sample_rate().0, - cfg.max_sample_rate().0, - cfg.sample_format() - ); - } - - // Find best matching configuration - // Prefer: f32 format (most compatible), matching channels, matching sample rate - let target_rate = cpal::SampleRate(sample_rate); - let target_channels = channels as u16; - - // Try to find a config that supports our sample rate and channel count - let mut best_config = None; - let mut best_score = 0i32; - - for cfg in &supported_configs { - let mut score = 0i32; - - // Prefer f32 format (most widely supported) - if cfg.sample_format() == SampleFormat::F32 { - score += 100; - } else if cfg.sample_format() == SampleFormat::I16 { - score += 50; - } - - // Prefer matching channel count - if cfg.channels() == target_channels { - score += 50; - } else if cfg.channels() >= target_channels { - score += 25; - } - - // Check if sample rate is in range - if target_rate >= cfg.min_sample_rate() && target_rate <= cfg.max_sample_rate() { - score += 100; - } else if cfg.max_sample_rate().0 >= 44100 { - score += 25; // At least supports reasonable rates - } - - if score > best_score { - best_score = score; - best_config = Some(cfg.clone()); - } - } - - let supported_range = - best_config.ok_or_else(|| anyhow!("No suitable audio configuration found"))?; - - // Determine actual sample rate to use - let actual_rate = if target_rate >= supported_range.min_sample_rate() - && target_rate <= supported_range.max_sample_rate() - { - target_rate - } else if cpal::SampleRate(48000) >= supported_range.min_sample_rate() - && cpal::SampleRate(48000) <= supported_range.max_sample_rate() - { - cpal::SampleRate(48000) - } else if cpal::SampleRate(44100) >= supported_range.min_sample_rate() - && cpal::SampleRate(44100) <= supported_range.max_sample_rate() - { - cpal::SampleRate(44100) - } else { - supported_range.max_sample_rate() - }; - - let actual_channels = supported_range.channels(); - let sample_format = supported_range.sample_format(); - - info!( - "Using audio config: {}Hz, {} channels, format {:?}", - actual_rate.0, actual_channels, sample_format - ); - - // Buffer for ~150ms of audio (handles network jitter) - // 48000Hz * 2ch * 0.15s = 14400 samples - // Larger buffer prevents underruns from network jitter - let buffer_size = (actual_rate.0 as usize) * (actual_channels as usize) * 150 / 1000; - let buffer = Arc::new(AudioRingBuffer::new(buffer_size)); - - info!( - "Audio buffer size: {} samples (~{}ms)", - buffer_size, - buffer_size * 1000 / (actual_rate.0 as usize * actual_channels as usize) - ); - - let config = supported_range.with_sample_rate(actual_rate).into(); - - let buffer_clone = buffer.clone(); - - // Build stream based on sample format - // The callback reads from the ring buffer - optimized for low latency - let stream = match sample_format { - SampleFormat::F32 => { - let buffer_f32 = buffer_clone.clone(); - device - .build_output_stream( - &config, - move |data: &mut [f32], _| { - // Read i16 samples in bulk and convert to f32 - let mut i16_buf = vec![0i16; data.len()]; - buffer_f32.read(&mut i16_buf); - for (out, &sample) in data.iter_mut().zip(i16_buf.iter()) { - *out = sample as f32 / 32768.0; - } - }, - |err| { - error!("Audio stream error: {}", err); - }, - None, - ) - .context("Failed to create f32 audio stream")? - } - SampleFormat::I16 => { - let buffer_i16 = buffer_clone.clone(); - device - .build_output_stream( - &config, - move |data: &mut [i16], _| { - buffer_i16.read(data); - }, - |err| { - error!("Audio stream error: {}", err); - }, - None, - ) - .context("Failed to create i16 audio stream")? - } - _ => { - // Fallback: try f32 anyway - let buffer_fallback = buffer_clone.clone(); - device - .build_output_stream( - &config, - move |data: &mut [f32], _| { - let mut i16_buf = vec![0i16; data.len()]; - buffer_fallback.read(&mut i16_buf); - for (out, &sample) in data.iter_mut().zip(i16_buf.iter()) { - *out = sample as f32 / 32768.0; - } - }, - |err| { - error!("Audio stream error: {}", err); - }, - None, - ) - .context("Failed to create audio stream with fallback format")? - } - }; - - stream.play().context("Failed to start audio playback")?; - - let device_name = device.name().unwrap_or_default(); - info!("Audio player started successfully on '{}'", device_name); - - // Create resampler for input_rate -> output_rate conversion and channel upmixing - // Input: decoder's sample_rate (48000) and channels (2 for stereo Opus) - // Output: device's actual_rate and actual_channels (may be 8 for 7.1 headsets) - let resampler = - AudioResampler::new(sample_rate, actual_rate.0, channels, actual_channels as u32); - - if sample_rate != actual_rate.0 || channels != actual_channels as u32 { - info!( - "Audio conversion enabled: {}Hz {}ch -> {}Hz {}ch", - sample_rate, channels, actual_rate.0, actual_channels - ); - } - - Ok(Self { - input_sample_rate: sample_rate, - output_sample_rate: actual_rate.0, - input_channels: channels, - output_channels: actual_channels as u32, - buffer, - stream: Arc::new(Mutex::new(Some(stream))), - needs_restart: Arc::new(AtomicBool::new(false)), - current_device_name: Arc::new(Mutex::new(device_name)), - resampler: Arc::new(Mutex::new(resampler)), - }) - } - - /// Push audio samples to the player (with automatic resampling and channel upmixing) - pub fn push_samples(&self, samples: &[i16]) { - // Check if device changed and we need to restart - self.check_device_change(); - - // Resample and upmix (48000Hz stereo -> device rate and channels) - let resampled = { - let mut resampler = self.resampler.lock(); - resampler.resample(samples) - }; - - self.buffer.write(&resampled); - } - - /// Get buffer fill level - pub fn buffer_available(&self) -> usize { - self.buffer.available() - } - - /// Get output sample rate (device rate) - pub fn sample_rate(&self) -> u32 { - self.output_sample_rate - } - - /// Get output channel count (device channels) - pub fn channels(&self) -> u32 { - self.output_channels - } - - /// Check if the default audio device changed and restart stream if needed - fn check_device_change(&self) { - use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; - - let host = cpal::default_host(); - let current_device = match host.default_output_device() { - Some(d) => d, - None => return, - }; - - let new_name = current_device.name().unwrap_or_default(); - let current_name = self.current_device_name.lock().clone(); - - if new_name != current_name && !new_name.is_empty() { - warn!("Audio device changed: '{}' -> '{}'", current_name, new_name); - - // Update device name - *self.current_device_name.lock() = new_name.clone(); - - // Recreate the audio stream on the new device - if let Err(e) = self.recreate_stream(¤t_device) { - error!("Failed to switch audio device: {}", e); - } else { - info!("Audio switched to '{}'", new_name); - } - } - } - - /// Recreate the audio stream on a new device - fn recreate_stream(&self, device: &cpal::Device) -> Result<()> { - use cpal::traits::{DeviceTrait, StreamTrait}; - use cpal::SampleFormat; - - // Stop old stream - *self.stream.lock() = None; - - // Query supported configurations - let supported_configs: Vec<_> = device - .supported_output_configs() - .map(|configs| configs.collect()) - .unwrap_or_default(); - - if supported_configs.is_empty() { - return Err(anyhow!("No supported audio configurations on new device")); - } - - // Find best config (prefer F32, stereo-compatible) - let mut best_config = None; - let mut best_score = 0i32; - - for cfg in &supported_configs { - let mut score = 0i32; - if cfg.sample_format() == SampleFormat::F32 { - score += 100; - } - // Prefer stereo, but accept any channel count (we'll upmix) - if cfg.channels() == 2 { - score += 50; - } else if cfg.channels() >= 2 { - score += 25; - } - if cfg.max_sample_rate().0 >= 44100 { - score += 25; - } - - if score > best_score { - best_score = score; - best_config = Some(cfg.clone()); - } - } - - let supported_range = - best_config.ok_or_else(|| anyhow!("No suitable audio config on new device"))?; - - // Pick sample rate - let actual_rate = if cpal::SampleRate(48000) >= supported_range.min_sample_rate() - && cpal::SampleRate(48000) <= supported_range.max_sample_rate() - { - cpal::SampleRate(48000) - } else if cpal::SampleRate(44100) >= supported_range.min_sample_rate() - && cpal::SampleRate(44100) <= supported_range.max_sample_rate() - { - cpal::SampleRate(44100) - } else { - supported_range.max_sample_rate() - }; - - let actual_channels = supported_range.channels(); - let sample_format = supported_range.sample_format(); - let config = supported_range.with_sample_rate(actual_rate).into(); - let buffer = self.buffer.clone(); - - info!( - "New device config: {}Hz, {} channels, {:?}", - actual_rate.0, actual_channels, sample_format - ); - - // Update resampler for new output rate and channels - self.resampler - .lock() - .set_output_config(actual_rate.0, actual_channels as u32); - - // Build new stream - let stream = match sample_format { - SampleFormat::F32 => { - let buf = buffer.clone(); - device - .build_output_stream( - &config, - move |data: &mut [f32], _| { - let mut i16_buf = vec![0i16; data.len()]; - buf.read(&mut i16_buf); - for (out, &sample) in data.iter_mut().zip(i16_buf.iter()) { - *out = sample as f32 / 32768.0; - } - }, - |err| error!("Audio stream error: {}", err), - None, - ) - .context("Failed to create audio stream")? - } - SampleFormat::I16 => { - let buf = buffer.clone(); - device - .build_output_stream( - &config, - move |data: &mut [i16], _| { - buf.read(data); - }, - |err| error!("Audio stream error: {}", err), - None, - ) - .context("Failed to create audio stream")? - } - _ => { - let buf = buffer.clone(); - device - .build_output_stream( - &config, - move |data: &mut [f32], _| { - let mut i16_buf = vec![0i16; data.len()]; - buf.read(&mut i16_buf); - for (out, &sample) in data.iter_mut().zip(i16_buf.iter()) { - *out = sample as f32 / 32768.0; - } - }, - |err| error!("Audio stream error: {}", err), - None, - ) - .context("Failed to create audio stream")? - } - }; - - stream - .play() - .context("Failed to start audio on new device")?; - *self.stream.lock() = Some(stream); - - Ok(()) - } -} diff --git a/opennow-streamer/src/media/d3d11.rs b/opennow-streamer/src/media/d3d11.rs deleted file mode 100644 index 5cb6292..0000000 --- a/opennow-streamer/src/media/d3d11.rs +++ /dev/null @@ -1,406 +0,0 @@ -//! D3D11 Zero-Copy Video Support for Windows -//! -//! This module provides zero-copy video rendering on Windows by keeping -//! decoded frames on GPU as D3D11 textures and sharing them with wgpu's DX12 backend. -//! -//! Flow: -//! 1. FFmpeg D3D11VA decodes to ID3D11Texture2D (GPU VRAM) -//! 2. We extract the texture from FFmpeg frame -//! 3. Create a DXGI shared handle (NT handle for cross-API sharing) -//! 4. Import into wgpu's DX12 backend via the hal layer -//! -//! This eliminates the expensive GPU->CPU->GPU round-trip that kills latency. - -use anyhow::{anyhow, Result}; -use log::{debug, info, warn}; -use parking_lot::Mutex; - -use windows::core::Interface; -use windows::Win32::Foundation::HANDLE; -use windows::Win32::Graphics::Direct3D11::{ - ID3D11Asynchronous, ID3D11Device, ID3D11Query, ID3D11Texture2D, D3D11_CPU_ACCESS_READ, - D3D11_MAPPED_SUBRESOURCE, D3D11_MAP_READ, D3D11_QUERY_DESC, D3D11_QUERY_EVENT, - D3D11_TEXTURE2D_DESC, D3D11_USAGE_STAGING, -}; -use windows::Win32::Graphics::Dxgi::Common::{DXGI_FORMAT_NV12, DXGI_FORMAT_P010}; -use windows::Win32::Graphics::Dxgi::{IDXGIResource1, DXGI_SHARED_RESOURCE_READ}; - -/// Wrapper for a D3D11 texture from FFmpeg hardware decoder -/// Holds the texture alive and provides access for wgpu import -pub struct D3D11TextureWrapper { - /// The D3D11 texture (NV12 or P010 format) - texture: ID3D11Texture2D, - /// Texture array index (for texture arrays used by some decoders) - array_index: u32, - /// Shared NT handle for cross-API sharing (DX11 -> DX12) - shared_handle: Mutex>, - /// Texture dimensions - pub width: u32, - pub height: u32, - /// Whether this is a 10-bit HDR texture (P010 format) - pub is_10bit: bool, -} - -// Safety: D3D11 COM objects are thread-safe (they use internal ref counting) -unsafe impl Send for D3D11TextureWrapper {} -unsafe impl Sync for D3D11TextureWrapper {} - -impl std::fmt::Debug for D3D11TextureWrapper { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("D3D11TextureWrapper") - .field("width", &self.width) - .field("height", &self.height) - .field("array_index", &self.array_index) - .field("is_10bit", &self.is_10bit) - .field("has_shared_handle", &self.shared_handle.lock().is_some()) - .finish() - } -} - -impl D3D11TextureWrapper { - /// Create a new wrapper from an existing ID3D11Texture2D - /// - /// This is used by the native DXVA decoder to wrap its output textures. - /// The texture is cloned (AddRef'd) so the original can be safely dropped. - pub fn from_texture(texture: ID3D11Texture2D, array_index: u32) -> Self { - unsafe { - let mut desc = D3D11_TEXTURE2D_DESC::default(); - texture.GetDesc(&mut desc); - - let is_10bit = desc.Format == DXGI_FORMAT_P010; - - debug!( - "D3D11TextureWrapper::from_texture: {}x{}, array_index={}, format={:?}, is_10bit={}", - desc.Width, desc.Height, array_index, desc.Format, is_10bit - ); - - Self { - texture, - array_index, - shared_handle: Mutex::new(None), - width: desc.Width, - height: desc.Height, - is_10bit, - } - } - } - - /// Create a new wrapper from FFmpeg's D3D11VA frame data - /// - /// # Safety - /// The texture pointer must be valid and point to an ID3D11Texture2D - pub unsafe fn from_ffmpeg_frame( - texture_ptr: *mut std::ffi::c_void, - array_index: i32, - ) -> Option { - if texture_ptr.is_null() { - warn!("D3D11 texture pointer is null"); - return None; - } - - // Cast to ID3D11Texture2D - // FFmpeg stores the raw COM pointer in frame->data[0] - let texture: ID3D11Texture2D = std::mem::transmute_copy(&texture_ptr); - - // Get texture description - let mut desc = D3D11_TEXTURE2D_DESC::default(); - texture.GetDesc(&mut desc); - - // Detect format - NV12 (8-bit SDR) or P010 (10-bit HDR) - let is_10bit = desc.Format == DXGI_FORMAT_P010; - let format_name = if is_10bit { - "P010 (10-bit HDR)" - } else if desc.Format == DXGI_FORMAT_NV12 { - "NV12 (8-bit SDR)" - } else { - "Unknown" - }; - - debug!( - "D3D11 texture: {}x{}, format={} ({:?}), array_size={}, bind_flags={:?}", - desc.Width, desc.Height, format_name, desc.Format, desc.ArraySize, desc.BindFlags - ); - - // Verify it's NV12 or P010 format (expected from hardware decoders) - if desc.Format != DXGI_FORMAT_NV12 && desc.Format != DXGI_FORMAT_P010 { - warn!( - "D3D11 texture format is {:?}, expected NV12 or P010", - desc.Format - ); - // Still proceed - might work with other formats - } - - Some(Self { - texture, - array_index: array_index as u32, - shared_handle: Mutex::new(None), - width: desc.Width, - height: desc.Height, - is_10bit, - }) - } - - /// Get or create a shared NT handle for this texture - /// This handle can be used to import the texture into DX12 - pub fn get_shared_handle(&self) -> Result { - let mut guard = self.shared_handle.lock(); - if let Some(handle) = *guard { - return Ok(handle); - } - - unsafe { - // Query IDXGIResource1 interface for shared handle creation - let dxgi_resource: IDXGIResource1 = self - .texture - .cast() - .map_err(|e| anyhow!("Failed to cast to IDXGIResource1: {:?}", e))?; - - // Create shared NT handle - let handle = dxgi_resource - .CreateSharedHandle( - None, // No security attributes - DXGI_SHARED_RESOURCE_READ.0, - None, // No name - ) - .map_err(|e| anyhow!("Failed to create shared handle: {:?}", e))?; - - *guard = Some(handle); - Ok(handle) - } - } - - /// Get the raw D3D11 texture - pub fn texture(&self) -> &ID3D11Texture2D { - &self.texture - } - - /// Get the texture array index - pub fn array_index(&self) -> u32 { - self.array_index - } - - /// Check if this texture is part of a texture array - pub fn is_texture_array(&self) -> bool { - unsafe { - let mut desc = D3D11_TEXTURE2D_DESC::default(); - self.texture.GetDesc(&mut desc); - desc.ArraySize > 1 - } - } - - /// Lock the texture and copy Y and UV planes to CPU memory - /// This is the fallback path when zero-copy import fails - pub fn lock_and_get_planes(&self) -> Result { - unsafe { - // Get the device from the texture itself - let device = self - .texture - .GetDevice() - .map_err(|e| anyhow!("Failed to get D3D11 device from texture: {:?}", e))?; - - // Get the device context - let context = device - .GetImmediateContext() - .map_err(|e| anyhow!("Failed to get device context: {:?}", e))?; - - // Create a staging texture for CPU access - let mut desc = D3D11_TEXTURE2D_DESC::default(); - self.texture.GetDesc(&mut desc); - - let staging_desc = D3D11_TEXTURE2D_DESC { - Width: desc.Width, - Height: desc.Height, - MipLevels: 1, - ArraySize: 1, - Format: desc.Format, - SampleDesc: desc.SampleDesc, - Usage: D3D11_USAGE_STAGING, - BindFlags: Default::default(), - CPUAccessFlags: D3D11_CPU_ACCESS_READ.0 as u32, - MiscFlags: Default::default(), - }; - - let mut staging_texture: Option = None; - device - .CreateTexture2D(&staging_desc, None, Some(&mut staging_texture)) - .map_err(|e| anyhow!("Failed to create staging texture: {:?}", e))?; - let staging_texture = staging_texture.unwrap(); - - // Copy from source texture (specific array slice) to staging - context.CopySubresourceRegion( - &staging_texture, - 0, // Destination subresource - 0, - 0, - 0, // Destination x, y, z - &self.texture, - self.array_index, // Source subresource (array index) - None, // Copy entire resource - ); - - // CRITICAL: Wait for GPU copy to complete before mapping - // Flush() only submits commands but doesn't wait - we need a query to sync - let query_desc = D3D11_QUERY_DESC { - Query: D3D11_QUERY_EVENT, - MiscFlags: 0, - }; - let mut query: Option = None; - device - .CreateQuery(&query_desc, Some(&mut query)) - .map_err(|e| anyhow!("Failed to create query: {:?}", e))?; - let query = query.ok_or_else(|| anyhow!("Query creation returned None"))?; - - // Cast query to ID3D11Asynchronous for End/GetData calls - let async_query: ID3D11Asynchronous = query - .cast() - .map_err(|e| anyhow!("Failed to cast query to ID3D11Asynchronous: {:?}", e))?; - - // Insert the query after the copy command - context.End(&async_query); - - // Wait for GPU to complete (poll until done) - loop { - let result = context.GetData(&async_query, None, 0, 0); - if result.is_ok() { - break; // Query signaled - GPU work is done - } - std::thread::yield_now(); - } - - // Map the staging texture - let mut mapped = D3D11_MAPPED_SUBRESOURCE::default(); - context - .Map(&staging_texture, 0, D3D11_MAP_READ, 0, Some(&mut mapped)) - .map_err(|e| anyhow!("Failed to map staging texture: {:?}", e))?; - - // NV12 layout: Y plane (full height) followed by UV plane (half height) - let y_height = desc.Height; - let uv_height = desc.Height / 2; - let row_pitch = mapped.RowPitch; - - // Copy Y plane - let y_size = (row_pitch * y_height) as usize; - let y_data = std::slice::from_raw_parts(mapped.pData as *const u8, y_size); - let y_plane = y_data.to_vec(); - - // Copy UV plane (starts after Y plane) - let uv_offset = y_size; - let uv_size = (row_pitch * uv_height) as usize; - let uv_data = - std::slice::from_raw_parts((mapped.pData as *const u8).add(uv_offset), uv_size); - let uv_plane = uv_data.to_vec(); - - // Unmap - context.Unmap(&staging_texture, 0); - - Ok(LockedPlanes { - y_plane, - uv_plane, - y_stride: row_pitch, - uv_stride: row_pitch, - width: desc.Width, - height: desc.Height, - }) - } - } -} - -impl Drop for D3D11TextureWrapper { - fn drop(&mut self) { - // Close the shared handle if we created one - if let Some(handle) = self.shared_handle.lock().take() { - unsafe { - let _ = windows::Win32::Foundation::CloseHandle(handle); - } - } - // COM objects (texture) are automatically released when dropped - } -} - -/// Locked plane data from D3D11 texture -pub struct LockedPlanes { - pub y_plane: Vec, - pub uv_plane: Vec, - pub y_stride: u32, - pub uv_stride: u32, - pub width: u32, - pub height: u32, -} - -/// Manager for D3D11 zero-copy textures -/// Handles device creation and texture import into wgpu -pub struct D3D11ZeroCopyManager { - /// D3D11 device (shared with FFmpeg) - device: ID3D11Device, - /// Whether zero-copy is enabled - enabled: bool, -} - -impl D3D11ZeroCopyManager { - /// Create a new manager with the given D3D11 device - pub fn new(device: ID3D11Device) -> Self { - info!("D3D11 zero-copy manager created"); - Self { - device, - enabled: true, - } - } - - /// Get the D3D11 device - pub fn device(&self) -> &ID3D11Device { - &self.device - } - - /// Check if zero-copy is enabled - pub fn is_enabled(&self) -> bool { - self.enabled - } - - /// Disable zero-copy (fallback to CPU path) - pub fn disable(&mut self) { - warn!("D3D11 zero-copy disabled, falling back to CPU path"); - self.enabled = false; - } -} - -/// Extract D3D11 texture from FFmpeg frame data pointers -/// -/// FFmpeg D3D11VA frame layout: -/// - data[0] = ID3D11Texture2D* -/// - data[1] = texture array index (as intptr_t) -/// -/// # Safety -/// The data pointers must be from a valid D3D11VA decoded frame -pub unsafe fn extract_d3d11_texture_from_frame( - data0: *mut u8, - data1: *mut u8, -) -> Option { - if data0.is_null() { - return None; - } - - let texture_ptr = data0 as *mut std::ffi::c_void; - let array_index = data1 as isize as i32; - - D3D11TextureWrapper::from_ffmpeg_frame(texture_ptr, array_index) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_locked_planes_layout() { - // Test NV12 plane calculations - let width = 1920u32; - let height = 1080u32; - - // Y plane: full resolution - let y_size = width * height; - assert_eq!(y_size, 2073600); - - // UV plane: half height, same width (interleaved) - let uv_size = width * (height / 2); - assert_eq!(uv_size, 1036800); - } -} diff --git a/opennow-streamer/src/media/dxva_decoder.rs b/opennow-streamer/src/media/dxva_decoder.rs deleted file mode 100644 index a013043..0000000 --- a/opennow-streamer/src/media/dxva_decoder.rs +++ /dev/null @@ -1,1597 +0,0 @@ -//! Native D3D11 Video Decoder (DXVA2) -//! -//! This module implements hardware video decoding using the D3D11 Video API directly, -//! similar to NVIDIA's DXVADecoder in their GeForce NOW client. -//! -//! Benefits over FFmpeg D3D11VA: -//! - No MAX_SLICES limitation (FFmpeg has hardcoded limit of 32) -//! - Direct control over texture arrays (RTArray) -//! - Better compatibility with NVIDIA drivers -//! - Zero-copy output to D3D11 textures -//! -//! Note: This decoder only supports HEVC (H.265). H.264 support was removed as -//! GeForce NOW streams use HEVC for better compression and quality. - -use anyhow::{anyhow, Result}; -use log::{error, info}; - -use windows::core::Interface; -use windows::Win32::Foundation::HMODULE; -use windows::Win32::Graphics::Direct3D::*; -use windows::Win32::Graphics::Direct3D11::*; -use windows::Win32::Graphics::Dxgi::Common::*; - -/// Video codec types supported by the decoder -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum DxvaCodec { - HEVC, -} - -/// DXVA2 decoder profile GUIDs -mod profiles { - use windows::core::GUID; - - // HEVC/H.265 profiles - pub const D3D11_DECODER_PROFILE_HEVC_VLD_MAIN: GUID = - GUID::from_u128(0x5b11d51b_2f4c_4452_bcc3_09f2a1160cc0); - pub const D3D11_DECODER_PROFILE_HEVC_VLD_MAIN10: GUID = - GUID::from_u128(0x107af0e0_ef1a_4d19_aba8_67a163073d13); -} - -/// Decoder configuration -#[derive(Debug, Clone)] -pub struct DxvaDecoderConfig { - /// Video codec - pub codec: DxvaCodec, - /// Video width - pub width: u32, - /// Video height - pub height: u32, - /// Whether HDR (10-bit) is enabled - pub is_hdr: bool, - /// Number of surfaces in the decoder pool (RTArray size) - pub surface_count: u32, - /// Enable low latency mode (reduces buffering, prioritizes decode speed) - pub low_latency: bool, -} - -impl Default for DxvaDecoderConfig { - fn default() -> Self { - Self { - codec: DxvaCodec::HEVC, - width: 1920, - height: 1080, - is_hdr: false, - surface_count: 25, // Increased from 20 for high bitrate 4K streams - low_latency: true, // Default to low latency for streaming - } - } -} - -/// Reference picture entry in the DPB (Decoded Picture Buffer) -#[derive(Debug, Clone, Copy, Default)] -pub struct DpbEntry { - /// Surface index in the texture array - pub surface_index: u8, - /// Picture Order Count (full POC, not just LSB) - pub poc: i32, - /// Is this a reference frame - pub is_reference: bool, - /// Is this a long-term reference - pub is_long_term: bool, - /// Frame number (for debugging/ordering) - pub frame_num: u64, -} - -/// Native D3D11 Video Decoder -/// -/// Uses ID3D11VideoDecoder directly, bypassing FFmpeg's D3D11VA wrapper. -/// This gives us full control over texture arrays and avoids FFmpeg limitations. -pub struct DxvaDecoder { - /// D3D11 device - device: ID3D11Device, - /// D3D11 device context - #[allow(dead_code)] - context: ID3D11DeviceContext, - /// D3D11 Video device interface - video_device: ID3D11VideoDevice, - /// D3D11 Video context interface - #[allow(dead_code)] - video_context: ID3D11VideoContext, - /// Video decoder instance - decoder: Option, - /// Output texture array (RTArray) - output_textures: Option, - /// Decoder output views (one per surface in the array) - output_views: Vec, - /// Current configuration - config: DxvaDecoderConfig, - /// DXGI format for output (NV12 or P010) - output_format: DXGI_FORMAT, - /// Decoder profile GUID - profile_guid: windows::core::GUID, - /// Current surface index (round-robin) - current_surface: u32, - /// Maximum supported resolution - #[allow(dead_code)] - max_width: u32, - #[allow(dead_code)] - max_height: u32, - /// Decoded Picture Buffer (DPB) for reference frame management - dpb: Vec, - /// Maximum DPB size - dpb_max_size: usize, - /// ConfigBitstreamRaw value from decoder config - /// 1 = raw bitstream with start codes, 2 = raw bitstream without start codes - config_bitstream_raw: u32, - /// Frame counter for DPB ordering - frame_count: u64, - /// Previous POC LSB for POC MSB calculation - prev_poc_lsb: i32, - /// Previous POC MSB - prev_poc_msb: i32, - /// Max POC LSB (2^log2_max_pic_order_cnt_lsb) - max_poc_lsb: i32, -} - -// Safety: D3D11 COM objects are internally thread-safe -unsafe impl Send for DxvaDecoder {} -unsafe impl Sync for DxvaDecoder {} - -impl DxvaDecoder { - /// Create a new DXVA decoder - pub fn new(config: DxvaDecoderConfig) -> Result { - info!( - "Creating native DXVA decoder for {:?} {}x{} HDR={}", - config.codec, config.width, config.height, config.is_hdr - ); - - // Create D3D11 device with video support - let (device, context) = Self::create_d3d11_device()?; - - // Get video interfaces - let video_device: ID3D11VideoDevice = device - .cast() - .map_err(|e| anyhow!("Failed to get ID3D11VideoDevice: {:?}", e))?; - let video_context: ID3D11VideoContext = context - .cast() - .map_err(|e| anyhow!("Failed to get ID3D11VideoContext: {:?}", e))?; - - // Enable multithread protection - if let Ok(mt) = device.cast::() { - unsafe { - mt.SetMultithreadProtected(true); - } - info!("D3D11 multithread protection enabled"); - } - - // Determine output format and profile - let (output_format, profile_guid) = Self::get_format_and_profile(&config)?; - - info!( - "DXVA decoder using format {:?}, profile {:?}", - output_format, profile_guid - ); - - // DPB size - must be large enough to hold all reference frames - // For high bitrate 4K HEVC streams, we need more buffer space - // HEVC spec allows up to 16 reference pictures, but we use 18 to have margin - // This should be less than surface_count to ensure we always have free surfaces - let dpb_max_size = 18; - - let mut decoder = Self { - device, - context, - video_device, - video_context, - decoder: None, - output_textures: None, - output_views: Vec::new(), - config, - output_format, - profile_guid, - current_surface: 0, - max_width: 0, - max_height: 0, - dpb: Vec::with_capacity(dpb_max_size), - dpb_max_size, - config_bitstream_raw: 1, // Will be set during initialize_decoder - frame_count: 0, - prev_poc_lsb: 0, - prev_poc_msb: 0, - max_poc_lsb: 256, // Default, will be updated from SPS - }; - - // Check decoder capabilities - decoder.check_capabilities()?; - - // Initialize the decoder - decoder.initialize_decoder()?; - - Ok(decoder) - } - - /// Create D3D11 device with VIDEO_SUPPORT flag - fn create_d3d11_device() -> Result<(ID3D11Device, ID3D11DeviceContext)> { - unsafe { - let mut device: Option = None; - let mut context: Option = None; - let mut feature_level = D3D_FEATURE_LEVEL_11_0; - - // Flags for video decoding - let flags = D3D11_CREATE_DEVICE_VIDEO_SUPPORT | D3D11_CREATE_DEVICE_BGRA_SUPPORT; - - // Feature levels to try - let feature_levels = [ - D3D_FEATURE_LEVEL_12_1, - D3D_FEATURE_LEVEL_12_0, - D3D_FEATURE_LEVEL_11_1, - D3D_FEATURE_LEVEL_11_0, - ]; - - D3D11CreateDevice( - None, // Default adapter - D3D_DRIVER_TYPE_HARDWARE, - HMODULE::default(), - flags, - Some(&feature_levels), - D3D11_SDK_VERSION, - Some(&mut device), - Some(&mut feature_level), - Some(&mut context), - ) - .map_err(|e| anyhow!("Failed to create D3D11 device: {:?}", e))?; - - let device = device.ok_or_else(|| anyhow!("D3D11 device is null"))?; - let context = context.ok_or_else(|| anyhow!("D3D11 context is null"))?; - - info!( - "Created D3D11 device with feature level {:?} (0x{:x})", - feature_level, feature_level.0 - ); - - Ok((device, context)) - } - } - - /// Get output format and decoder profile based on config - fn get_format_and_profile( - config: &DxvaDecoderConfig, - ) -> Result<(DXGI_FORMAT, windows::core::GUID)> { - let format = if config.is_hdr { - DXGI_FORMAT_P010 // 10-bit HDR - } else { - DXGI_FORMAT_NV12 // 8-bit SDR - }; - - let profile = if config.is_hdr { - profiles::D3D11_DECODER_PROFILE_HEVC_VLD_MAIN10 - } else { - profiles::D3D11_DECODER_PROFILE_HEVC_VLD_MAIN - }; - - Ok((format, profile)) - } - - /// Check decoder capabilities and maximum resolution - fn check_capabilities(&mut self) -> Result<()> { - unsafe { - // Get number of decoder profiles - let profile_count = self.video_device.GetVideoDecoderProfileCount(); - info!("D3D11 Video device has {} decoder profiles", profile_count); - - // Check if our profile is supported - let mut profile_supported = false; - for i in 0..profile_count { - // New API returns Result - if let Ok(profile) = self.video_device.GetVideoDecoderProfile(i) { - if profile == self.profile_guid { - profile_supported = true; - info!("Found matching decoder profile at index {}", i); - break; - } - } - } - - if !profile_supported { - return Err(anyhow!( - "Decoder profile {:?} not supported", - self.profile_guid - )); - } - - // Check format support - new API returns Result - let format_supported = self - .video_device - .CheckVideoDecoderFormat(&self.profile_guid, self.output_format) - .map_err(|e| anyhow!("Failed to check decoder format: {:?}", e))?; - - if !format_supported.as_bool() { - return Err(anyhow!( - "Output format {:?} not supported for this profile", - self.output_format - )); - } - - info!("Output format {:?} is supported", self.output_format); - - // Get decoder config to check max resolution - let desc = D3D11_VIDEO_DECODER_DESC { - Guid: self.profile_guid, - SampleWidth: self.config.width, - SampleHeight: self.config.height, - OutputFormat: self.output_format, - }; - - let config_count = self - .video_device - .GetVideoDecoderConfigCount(&desc) - .map_err(|e| anyhow!("Failed to get decoder config count: {:?}", e))?; - - info!( - "Found {} decoder configurations for {}x{}", - config_count, self.config.width, self.config.height - ); - - if config_count == 0 { - return Err(anyhow!( - "No decoder configurations available for {}x{}", - self.config.width, - self.config.height - )); - } - - // Store max resolution (we'll refine this later if needed) - self.max_width = self.config.width; - self.max_height = self.config.height; - - Ok(()) - } - } - - /// Initialize the video decoder and output textures - fn initialize_decoder(&mut self) -> Result<()> { - unsafe { - info!( - "Initializing DXVA decoder {}x{} with {} surfaces", - self.config.width, self.config.height, self.config.surface_count - ); - - // Create decoder description - let decoder_desc = D3D11_VIDEO_DECODER_DESC { - Guid: self.profile_guid, - SampleWidth: self.config.width, - SampleHeight: self.config.height, - OutputFormat: self.output_format, - }; - - // Enumerate all decoder configurations and find one with ConfigBitstreamRaw=2 (short slices) - let config_count = self - .video_device - .GetVideoDecoderConfigCount(&decoder_desc) - .map_err(|e| anyhow!("Failed to get decoder config count: {:?}", e))?; - - info!("Enumerating {} decoder configurations:", config_count); - - let mut selected_config: Option = None; - let mut fallback_config: Option = None; - - for i in 0..config_count { - let mut config = D3D11_VIDEO_DECODER_CONFIG::default(); - if self - .video_device - .GetVideoDecoderConfig(&decoder_desc, i, &mut config) - .is_ok() - { - info!( - " Config {}: ConfigBitstreamRaw={}, ConfigMBcontrolRasterOrder={}, ConfigResidDiffHost={}, ConfigSpatialResid8={}, ConfigResid8Subtraction={}, ConfigSpatialHost8or9Clipping={}, ConfigSpatialResidInterleaved={}, ConfigIntraResidUnsigned={}, ConfigResidDiffAccelerator={}, ConfigHostInverseScan={}, ConfigSpecificIDCT={}, Config4GroupedCoefs={}", - i, - config.ConfigBitstreamRaw, - config.ConfigMBcontrolRasterOrder, - config.ConfigResidDiffHost, - config.ConfigSpatialResid8, - config.ConfigResid8Subtraction, - config.ConfigSpatialHost8or9Clipping, - config.ConfigSpatialResidInterleaved, - config.ConfigIntraResidUnsigned, - config.ConfigResidDiffAccelerator, - config.ConfigHostInverseScan, - config.ConfigSpecificIDCT, - config.Config4GroupedCoefs - ); - - // Prefer ConfigBitstreamRaw=1 (Annex-B format with start codes) - // This is more straightforward and widely compatible - // ConfigBitstreamRaw=2 (short slice format) requires more precise buffer formatting - if config.ConfigBitstreamRaw == 1 { - selected_config = Some(config); - info!(" -> Selected config {} (Annex-B with start codes)", i); - } else if fallback_config.is_none() { - fallback_config = Some(config); - } - } - } - - // Use selected config, or fallback to first available - let decoder_config = selected_config - .or(fallback_config) - .ok_or_else(|| anyhow!("No valid decoder configuration found"))?; - - // Store the ConfigBitstreamRaw value for bitstream formatting - self.config_bitstream_raw = decoder_config.ConfigBitstreamRaw; - - info!( - "Using decoder config: ConfigBitstreamRaw={}, ConfigMBcontrolRasterOrder={}", - decoder_config.ConfigBitstreamRaw, decoder_config.ConfigMBcontrolRasterOrder - ); - - // Create the video decoder - returns Result - let decoder = self - .video_device - .CreateVideoDecoder(&decoder_desc, &decoder_config) - .map_err(|e| anyhow!("Failed to create video decoder: {:?}", e))?; - - info!("Created ID3D11VideoDecoder successfully"); - - // Create output texture array (RTArray) - // This is the key difference from FFmpeg - we create a proper texture array - let texture_desc = D3D11_TEXTURE2D_DESC { - Width: self.config.width, - Height: self.config.height, - MipLevels: 1, - ArraySize: self.config.surface_count, - Format: self.output_format, - SampleDesc: DXGI_SAMPLE_DESC { - Count: 1, - Quality: 0, - }, - Usage: D3D11_USAGE_DEFAULT, - BindFlags: D3D11_BIND_DECODER.0 as u32, - CPUAccessFlags: 0, - MiscFlags: 0, - }; - - let mut output_texture: Option = None; - self.device - .CreateTexture2D(&texture_desc, None, Some(&mut output_texture)) - .map_err(|e| anyhow!("Failed to create output texture array: {:?}", e))?; - - let output_texture = output_texture.ok_or_else(|| anyhow!("Output texture is null"))?; - - info!( - "Created output texture array: {}x{} x {} slices, format {:?}", - self.config.width, - self.config.height, - self.config.surface_count, - self.output_format - ); - - // Create decoder output views for each surface in the array - let mut output_views = Vec::with_capacity(self.config.surface_count as usize); - - for i in 0..self.config.surface_count { - let view_desc = D3D11_VIDEO_DECODER_OUTPUT_VIEW_DESC { - DecodeProfile: self.profile_guid, - ViewDimension: D3D11_VDOV_DIMENSION_TEXTURE2D, - Anonymous: D3D11_VIDEO_DECODER_OUTPUT_VIEW_DESC_0 { - Texture2D: D3D11_TEX2D_VDOV { ArraySlice: i }, - }, - }; - - // New API: CreateVideoDecoderOutputView takes 3 params but last is Option<*mut Option> - let mut view: Option = None; - self.video_device - .CreateVideoDecoderOutputView(&output_texture, &view_desc, Some(&mut view)) - .map_err(|e| anyhow!("Failed to create output view {}: {:?}", i, e))?; - - let view = view.ok_or_else(|| anyhow!("Output view {} is null", i))?; - output_views.push(view); - } - - info!("Created {} decoder output views", output_views.len()); - - self.decoder = Some(decoder); - self.output_textures = Some(output_texture); - self.output_views = output_views; - self.current_surface = 0; - - Ok(()) - } - } - - /// Get the next available surface index, avoiding surfaces in DPB - pub fn get_next_surface(&mut self) -> u32 { - // Find a surface that is NOT currently used as a reference in the DPB - let surface_count = self.config.surface_count; - - for _ in 0..surface_count { - let candidate = self.current_surface; - self.current_surface = (self.current_surface + 1) % surface_count; - - // Check if this surface is in the DPB - let in_dpb = self - .dpb - .iter() - .any(|entry| entry.surface_index == candidate as u8); - - if !in_dpb { - return candidate; - } - } - - // All surfaces are in DPB - this shouldn't happen if DPB size < surface count - // Fall back to evicting the oldest DPB entry - if let Some(oldest) = self.dpb.first() { - let surface = oldest.surface_index as u32; - self.dpb.remove(0); - return surface; - } - - // Last resort: just use current surface - let surface = self.current_surface; - self.current_surface = (self.current_surface + 1) % surface_count; - surface - } - - /// Get the output texture - pub fn output_texture(&self) -> Option<&ID3D11Texture2D> { - self.output_textures.as_ref() - } - - /// Get a specific output view - pub fn output_view(&self, index: u32) -> Option<&ID3D11VideoDecoderOutputView> { - self.output_views.get(index as usize) - } - - /// Get the D3D11 device - pub fn device(&self) -> &ID3D11Device { - &self.device - } - - /// Get the video decoder - pub fn decoder(&self) -> Option<&ID3D11VideoDecoder> { - self.decoder.as_ref() - } - - /// Get the video context - pub fn video_context(&self) -> &ID3D11VideoContext { - &self.video_context - } - - /// Get decoder configuration - pub fn config(&self) -> &DxvaDecoderConfig { - &self.config - } - - /// Check if decoder is initialized - pub fn is_initialized(&self) -> bool { - self.decoder.is_some() && !self.output_views.is_empty() - } - - /// Get output format - pub fn output_format(&self) -> DXGI_FORMAT { - self.output_format - } - - /// Check decoder capabilities for a specific resolution - pub fn check_resolution_support( - codec: DxvaCodec, - width: u32, - height: u32, - is_hdr: bool, - ) -> Result { - // Create temporary device to check capabilities - let (device, _context) = Self::create_d3d11_device()?; - - let video_device: ID3D11VideoDevice = device - .cast() - .map_err(|e| anyhow!("Failed to get ID3D11VideoDevice: {:?}", e))?; - - let config = DxvaDecoderConfig { - codec, - width, - height, - is_hdr, - surface_count: 1, - low_latency: true, - }; - - let (output_format, profile_guid) = Self::get_format_and_profile(&config)?; - - unsafe { - // Check format support - let format_supported = video_device - .CheckVideoDecoderFormat(&profile_guid, output_format) - .map_err(|e| anyhow!("Failed to check decoder format: {:?}", e))?; - - if !format_supported.as_bool() { - return Ok(false); - } - - // Check if we can get decoder configs for this resolution - let desc = D3D11_VIDEO_DECODER_DESC { - Guid: profile_guid, - SampleWidth: width, - SampleHeight: height, - OutputFormat: output_format, - }; - - let config_count = video_device.GetVideoDecoderConfigCount(&desc).unwrap_or(0); - - Ok(config_count > 0) - } - } - - /// Get maximum supported resolution for a codec - pub fn get_max_resolution(codec: DxvaCodec, is_hdr: bool) -> Result<(u32, u32)> { - // Common resolutions to check (from highest to lowest) - let resolutions = [ - (7680, 4320), // 8K - (5120, 2880), // 5K - (3840, 2160), // 4K - (2560, 1440), // 1440p - (1920, 1080), // 1080p - (1280, 720), // 720p - ]; - - for (width, height) in resolutions { - if Self::check_resolution_support(codec, width, height, is_hdr)? { - info!( - "Max resolution for {:?} HDR={}: {}x{}", - codec, is_hdr, width, height - ); - return Ok((width, height)); - } - } - - Err(anyhow!("No supported resolution found for {:?}", codec)) - } -} - -impl Drop for DxvaDecoder { - fn drop(&mut self) { - info!("Dropping DXVA decoder"); - // COM objects are automatically released when dropped - self.output_views.clear(); - self.decoder = None; - self.output_textures = None; - } -} - -// ============================================================================ -// DXVA2 HEVC Structures and Frame Decoding -// ============================================================================ - -/// DXVA Picture Entry for HEVC -/// Matches DXVA_PicEntry_HEVC from dxva.h -#[repr(C)] -#[derive(Debug, Clone, Copy, Default)] -pub struct DxvaPicEntryHevc { - /// Index into the reference picture array (7 bits) + flags (1 bit for RefPicFlags) - /// Bit 7: RefPicFlags (0 = short-term, 1 = long-term) - pub b_pic_entry: u8, -} - -impl DxvaPicEntryHevc { - pub fn new(index: u8, is_long_term: bool) -> Self { - let flags = if is_long_term { 0x80 } else { 0 }; - Self { - b_pic_entry: (index & 0x7F) | flags, - } - } - - pub fn invalid() -> Self { - Self { b_pic_entry: 0xFF } - } -} - -/// DXVA HEVC Picture Parameters (DXVA_PicParams_HEVC) -/// This structure MUST match the Windows SDK dxva.h definition exactly -/// Size should be 440 bytes according to Microsoft spec -#[repr(C, packed)] -#[derive(Clone, Copy)] -pub struct DxvaHevcPicParams { - // Dimensions in minimum coding block units - pub pic_width_in_min_cbs_y: u16, // offset 0 - pub pic_height_in_min_cbs_y: u16, // offset 2 - - // Format and sequence info flags (bitfield packed into u16) - // chroma_format_idc:2, separate_colour_plane_flag:1, bit_depth_luma_minus8:3, - // bit_depth_chroma_minus8:3, log2_max_pic_order_cnt_lsb_minus4:4, - // NoPicReorderingFlag:1, NoBiPredFlag:1, ReservedBits1:1 - pub w_format_and_sequence_info_flags: u16, // offset 4 - - // Current picture - pub curr_pic: DxvaPicEntryHevc, // offset 6 - - // SPS parameters - pub sps_max_dec_pic_buffering_minus1: u8, // offset 7 - pub log2_min_luma_coding_block_size_minus3: u8, // offset 8 - pub log2_diff_max_min_luma_coding_block_size: u8, // offset 9 - pub log2_min_transform_block_size_minus2: u8, // offset 10 - pub log2_diff_max_min_transform_block_size: u8, // offset 11 - pub max_transform_hierarchy_depth_inter: u8, // offset 12 - pub max_transform_hierarchy_depth_intra: u8, // offset 13 - pub num_short_term_ref_pic_sets: u8, // offset 14 - pub num_long_term_ref_pics_sps: u8, // offset 15 - pub num_ref_idx_l0_default_active_minus1: u8, // offset 16 - pub num_ref_idx_l1_default_active_minus1: u8, // offset 17 - pub init_qp_minus26: i8, // offset 18 - pub uc_num_delta_pocs_of_ref_rps_idx: u8, // offset 19 - pub w_num_bits_for_short_term_rps_in_slice: u16, // offset 20 - pub reserved_bits2: u16, // offset 22 - - // Coding param tool flags (bitfield packed into u32) - // scaling_list_enabled_flag:1, amp_enabled_flag:1, sample_adaptive_offset_enabled_flag:1, - // pcm_enabled_flag:1, pcm_sample_bit_depth_luma_minus1:4, pcm_sample_bit_depth_chroma_minus1:4, - // log2_min_pcm_luma_coding_block_size_minus3:2, log2_diff_max_min_pcm_luma_coding_block_size:2, - // pcm_loop_filter_disabled_flag:1, long_term_ref_pics_present_flag:1, sps_temporal_mvp_enabled_flag:1, - // strong_intra_smoothing_enabled_flag:1, dependent_slice_segments_enabled_flag:1, - // output_flag_present_flag:1, num_extra_slice_header_bits:3, sign_data_hiding_enabled_flag:1, - // cabac_init_present_flag:1, ReservedBits3:5 - pub dw_coding_param_tool_flags: u32, // offset 24 - - // Coding setting picture property flags (bitfield packed into u32) - // constrained_intra_pred_flag:1, transform_skip_enabled_flag:1, cu_qp_delta_enabled_flag:1, - // pps_slice_chroma_qp_offsets_present_flag:1, weighted_pred_flag:1, weighted_bipred_flag:1, - // transquant_bypass_enabled_flag:1, tiles_enabled_flag:1, entropy_coding_sync_enabled_flag:1, - // uniform_spacing_flag:1, loop_filter_across_tiles_enabled_flag:1, - // pps_loop_filter_across_slices_enabled_flag:1, deblocking_filter_override_enabled_flag:1, - // pps_deblocking_filter_disabled_flag:1, lists_modification_present_flag:1, - // slice_segment_header_extension_present_flag:1, IrapPicFlag:1, IdrPicFlag:1, IntraPicFlag:1, - // ReservedBits4:13 - pub dw_coding_setting_picture_property_flags: u32, // offset 28 - - // PPS parameters - pub pps_cb_qp_offset: i8, // offset 32 - pub pps_cr_qp_offset: i8, // offset 33 - pub num_tile_columns_minus1: u8, // offset 34 - pub num_tile_rows_minus1: u8, // offset 35 - pub column_width_minus1: [u16; 19], // offset 36 (38 bytes) - pub row_height_minus1: [u16; 21], // offset 74 (42 bytes) - pub diff_cu_qp_delta_depth: u8, // offset 116 - pub pps_beta_offset_div2: i8, // offset 117 - pub pps_tc_offset_div2: i8, // offset 118 - pub log2_parallel_merge_level_minus2: u8, // offset 119 - pub curr_pic_order_cnt_val: i32, // offset 120 - - // Reference picture list (15 entries) - pub ref_pic_list: [DxvaPicEntryHevc; 15], // offset 124 (15 bytes) - pub reserved_bits5: u8, // offset 139 - - // POC values for reference pictures - pub pic_order_cnt_val_list: [i32; 15], // offset 140 (60 bytes) - - // Reference picture sets - pub ref_pic_set_st_curr_before: [u8; 8], // offset 200 - pub ref_pic_set_st_curr_after: [u8; 8], // offset 208 - pub ref_pic_set_lt_curr: [u8; 8], // offset 216 - - pub reserved_bits6: u16, // offset 224 - pub reserved_bits7: u16, // offset 226 - pub status_report_feedback_number: u32, // offset 228 - // Total size: 232 bytes (without alignment padding) -} - -impl Default for DxvaHevcPicParams { - fn default() -> Self { - unsafe { std::mem::zeroed() } - } -} - -impl std::fmt::Debug for DxvaHevcPicParams { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - // Copy values to avoid unaligned references in packed struct - let width = self.pic_width_in_min_cbs_y; - let height = self.pic_height_in_min_cbs_y; - let poc = self.curr_pic_order_cnt_val; - f.debug_struct("DxvaHevcPicParams") - .field("pic_width_in_min_cbs_y", &width) - .field("pic_height_in_min_cbs_y", &height) - .field("curr_pic_order_cnt_val", &poc) - .finish() - } -} - -/// DXVA HEVC Quantization Matrix -#[repr(C)] -#[derive(Debug, Clone)] -pub struct DxvaHevcQMatrix { - pub scaling_list_4x4: [[u8; 16]; 6], - pub scaling_list_8x8: [[u8; 64]; 6], - pub scaling_list_16x16: [[u8; 64]; 6], - pub scaling_list_32x32: [[u8; 64]; 2], - pub scaling_list_dc_16x16: [u8; 6], - pub scaling_list_dc_32x32: [u8; 2], -} - -impl Default for DxvaHevcQMatrix { - fn default() -> Self { - // Initialize with flat scaling (all 16s) - Self { - scaling_list_4x4: [[16; 16]; 6], - scaling_list_8x8: [[16; 64]; 6], - scaling_list_16x16: [[16; 64]; 6], - scaling_list_32x32: [[16; 64]; 2], - scaling_list_dc_16x16: [16; 6], - scaling_list_dc_32x32: [16; 2], - } - } -} - -/// DXVA HEVC Slice Header (short format) -/// This matches the DXVA_Slice_HEVC_Short structure used by FFmpeg and NVIDIA -/// For ConfigBitstreamRaw=1, we submit Annex-B formatted bitstream with start codes -/// Size: 10 bytes (packed) -#[repr(C, packed)] -#[derive(Debug, Clone, Copy, Default)] -pub struct DxvaHevcSliceShort { - /// Position of NAL unit data in the bitstream buffer - pub bs_nal_unit_data_location: u32, - /// Number of bytes in the bitstream buffer for this slice - pub slice_bytes_in_buffer: u32, - /// Bad slice chopping indicator (0 = no chopping) - pub w_bad_slice_chopping: u16, -} - -/// Decoded frame result - zero-copy version -/// The texture remains on GPU and should be used directly for rendering -#[derive(Debug)] -pub struct DxvaDecodedFrame { - /// Texture array containing the decoded frame - pub texture: ID3D11Texture2D, - /// Array slice index within the texture - pub array_index: u32, - /// Frame width - pub width: u32, - /// Frame height - pub height: u32, - /// Is 10-bit HDR - pub is_hdr: bool, - /// Picture order count - pub poc: i32, -} - -/// DXVA2 Buffer types - must match D3D11_VIDEO_DECODER_BUFFER_TYPE enum values -#[repr(i32)] -#[derive(Debug, Clone, Copy)] -pub enum DxvaBufferType { - PictureParameters = 0, // D3D11_VIDEO_DECODER_BUFFER_PICTURE_PARAMETERS - MacroblockControl = 1, // D3D11_VIDEO_DECODER_BUFFER_MACROBLOCK_CONTROL - ResidualDifference = 2, // D3D11_VIDEO_DECODER_BUFFER_RESIDUAL_DIFFERENCE - DeblockingControl = 3, // D3D11_VIDEO_DECODER_BUFFER_DEBLOCKING_CONTROL - InverseQuantizationMatrix = 4, // D3D11_VIDEO_DECODER_BUFFER_INVERSE_QUANTIZATION_MATRIX - SliceControl = 5, // D3D11_VIDEO_DECODER_BUFFER_SLICE_CONTROL - Bitstream = 6, // D3D11_VIDEO_DECODER_BUFFER_BITSTREAM - MotionVector = 7, // D3D11_VIDEO_DECODER_BUFFER_MOTION_VECTOR - FilmGrain = 8, // D3D11_VIDEO_DECODER_BUFFER_FILM_GRAIN -} - -impl DxvaDecoder { - /// Decode a frame using native DXVA2 - /// - /// This function: - /// 1. Parses the HEVC bitstream - /// 2. Fills DXVA picture parameters - /// 3. Submits buffers to the decoder - /// 4. Returns the decoded texture - pub fn decode_frame( - &mut self, - bitstream: &[u8], - parser: &mut super::hevc_parser::HevcParser, - ) -> Result { - if !self.is_initialized() { - return Err(anyhow!("DXVA decoder not initialized")); - } - - // Parse NAL units - let nals = parser.find_nal_units(bitstream); - if nals.is_empty() { - return Err(anyhow!("No NAL units found in bitstream")); - } - - // Process parameter sets - for nal in &nals { - parser.process_nal(nal)?; - } - - // Find slice NAL units - let slice_nals: Vec<_> = nals.iter().filter(|n| n.nal_type.is_slice()).collect(); - if slice_nals.is_empty() { - return Err(anyhow!("No slice NAL units found")); - } - - // Get first slice header to determine PPS/SPS - let first_slice = &slice_nals[0]; - let slice_header = parser.parse_slice_header(first_slice)?; - - let pps = parser.pps[slice_header.pps_id as usize] - .as_ref() - .ok_or_else(|| anyhow!("PPS {} not found", slice_header.pps_id))?; - let sps = parser.sps[pps.sps_id as usize] - .as_ref() - .ok_or_else(|| anyhow!("SPS {} not found", pps.sps_id))?; - - // CRITICAL: Clear DPB BEFORE building pic_params for IDR frames - // IDR frames must not have any reference pictures, so we need to clear - // the DPB before building the picture parameters, not after decoding - if first_slice.nal_type.is_idr() { - self.dpb.clear(); - } - - // Get next output surface FIRST - this must happen before building pic_params - // because curr_pic in pic_params must match the actual output surface - let surface_idx = self.get_next_surface(); - - // Calculate the full POC using H.265 section 8.3.1 algorithm - // max_poc_lsb = 2^log2_max_pic_order_cnt_lsb - let max_poc_lsb = 1i32 << sps.log2_max_poc_lsb; - self.max_poc_lsb = max_poc_lsb; - let is_idr = first_slice.nal_type.is_idr(); - let poc_lsb = slice_header.pic_order_cnt_lsb as i32; - let full_poc = self.calculate_full_poc(poc_lsb, is_idr, max_poc_lsb); - - // Build DXVA picture parameters with the correct surface index and full POC - let pic_params = self.build_hevc_pic_params( - sps, - pps, - first_slice, - &slice_header, - surface_idx, - full_poc, - )?; - let output_view = self - .output_views - .get(surface_idx as usize) - .ok_or_else(|| anyhow!("Invalid surface index {}", surface_idx))?; - - // Get decoder - let decoder = self - .decoder - .as_ref() - .ok_or_else(|| anyhow!("Decoder not available"))?; - - // Build Annex-B formatted bitstream and slice controls - // FFmpeg prepends start codes (0x000001) to each slice NAL unit - let (annex_b_bitstream, slice_controls) = - self.build_annex_b_bitstream_and_slices(&slice_nals, bitstream)?; - let slice_size = (std::mem::size_of::() * slice_controls.len()) as u32; - - unsafe { - // Begin frame - self.video_context - .DecoderBeginFrame(decoder, output_view, 0, None) - .map_err(|e| anyhow!("DecoderBeginFrame failed: {:?}", e))?; - - // Collect buffer descriptors - let mut buffer_descs = Vec::with_capacity(4); - - // 1. Submit picture parameters buffer - let pic_params_size = std::mem::size_of::() as u32; - self.submit_buffer( - decoder, - DxvaBufferType::PictureParameters, - &pic_params as *const _ as *const u8, - pic_params_size, - )?; - buffer_descs.push(D3D11_VIDEO_DECODER_BUFFER_DESC { - BufferType: D3D11_VIDEO_DECODER_BUFFER_PICTURE_PARAMETERS, - BufferIndex: 0, - DataOffset: 0, - DataSize: pic_params_size, - FirstMBaddress: 0, - NumMBsInBuffer: 0, - Width: self.config.width, - Height: self.config.height, - Stride: 0, - ReservedBits: 0, - pIV: std::ptr::null_mut(), - IVSize: 0, - PartialEncryption: false.into(), - EncryptedBlockInfo: D3D11_ENCRYPTED_BLOCK_INFO::default(), - }); - - // 2. Submit quantization matrix buffer only if scaling lists are enabled - if sps.scaling_list_enabled { - let qmatrix = DxvaHevcQMatrix::default(); - let qmatrix_size = std::mem::size_of::() as u32; - self.submit_buffer( - decoder, - DxvaBufferType::InverseQuantizationMatrix, - &qmatrix as *const _ as *const u8, - qmatrix_size, - )?; - buffer_descs.push(D3D11_VIDEO_DECODER_BUFFER_DESC { - BufferType: D3D11_VIDEO_DECODER_BUFFER_INVERSE_QUANTIZATION_MATRIX, - BufferIndex: 0, - DataOffset: 0, - DataSize: qmatrix_size, - FirstMBaddress: 0, - NumMBsInBuffer: 0, - Width: self.config.width, - Height: self.config.height, - Stride: 0, - ReservedBits: 0, - pIV: std::ptr::null_mut(), - IVSize: 0, - PartialEncryption: false.into(), - EncryptedBlockInfo: D3D11_ENCRYPTED_BLOCK_INFO::default(), - }); - } - - // 3. Submit slice control buffers - if !slice_controls.is_empty() { - self.submit_buffer( - decoder, - DxvaBufferType::SliceControl, - slice_controls.as_ptr() as *const u8, - slice_size, - )?; - buffer_descs.push(D3D11_VIDEO_DECODER_BUFFER_DESC { - BufferType: D3D11_VIDEO_DECODER_BUFFER_SLICE_CONTROL, - BufferIndex: 0, - DataOffset: 0, - DataSize: slice_size, - FirstMBaddress: 0, - NumMBsInBuffer: slice_controls.len() as u32, - Width: self.config.width, - Height: self.config.height, - Stride: 0, - ReservedBits: 0, - pIV: std::ptr::null_mut(), - IVSize: 0, - PartialEncryption: false.into(), - EncryptedBlockInfo: D3D11_ENCRYPTED_BLOCK_INFO::default(), - }); - } - - // 4. Submit bitstream buffer (Annex-B formatted with start codes) - let bitstream_size = annex_b_bitstream.len() as u32; - self.submit_buffer( - decoder, - DxvaBufferType::Bitstream, - annex_b_bitstream.as_ptr(), - bitstream_size, - )?; - buffer_descs.push(D3D11_VIDEO_DECODER_BUFFER_DESC { - BufferType: D3D11_VIDEO_DECODER_BUFFER_BITSTREAM, - BufferIndex: 0, - DataOffset: 0, - DataSize: bitstream_size, - FirstMBaddress: 0, - NumMBsInBuffer: 0, - Width: self.config.width, - Height: self.config.height, - Stride: 0, - ReservedBits: 0, - pIV: std::ptr::null_mut(), - IVSize: 0, - PartialEncryption: false.into(), - EncryptedBlockInfo: D3D11_ENCRYPTED_BLOCK_INFO::default(), - }); - - // Execute decode with all buffer descriptors - self.video_context - .SubmitDecoderBuffers(decoder, &buffer_descs) - .map_err(|e| anyhow!("SubmitDecoderBuffers failed: {:?}", e))?; - - // End frame - self.video_context - .DecoderEndFrame(decoder) - .map_err(|e| anyhow!("DecoderEndFrame failed: {:?}", e))?; - - // CRITICAL: Flush the context to ensure decode commands are submitted - // This is needed for proper GPU synchronization before texture is used - self.context.Flush(); - } - - // ZERO-COPY: No CPU staging texture copy needed! - // The texture stays on GPU and will be used directly by the renderer - // via D3D11TextureWrapper and wgpu texture import - - // Determine if this is a reference frame (all non-RASL/RADL frames are reference) - // TrailR (trailing picture, reference) = slice type indicates reference - let is_reference = first_slice.nal_type.is_vcl(); // VCL NALs are video data - - // Update DPB with the decoded frame using the full POC - self.update_dpb(surface_idx, full_poc, is_reference, is_idr); - - // Return decoded frame info - texture stays on GPU - let output_texture = self - .output_textures - .as_ref() - .ok_or_else(|| anyhow!("Output texture not available"))? - .clone(); - - Ok(DxvaDecodedFrame { - texture: output_texture, - array_index: surface_idx, - width: self.config.width, - height: self.config.height, - is_hdr: self.config.is_hdr, - poc: full_poc, - }) - } - - /// Build bitstream and slice controls based on ConfigBitstreamRaw setting - /// - ConfigBitstreamRaw=1: Annex-B format with start codes (0x000001) - /// - ConfigBitstreamRaw=2: Raw NAL units without start codes - fn build_annex_b_bitstream_and_slices( - &self, - slice_nals: &[&super::hevc_parser::HevcNalUnit], - _original_bitstream: &[u8], - ) -> Result<(Vec, Vec)> { - // Start code for Annex-B format (only used when ConfigBitstreamRaw=1) - const START_CODE: [u8; 3] = [0x00, 0x00, 0x01]; - - // Determine whether to include start codes based on ConfigBitstreamRaw - // ConfigBitstreamRaw=1: Include start codes (Annex-B format) - // ConfigBitstreamRaw=2: No start codes (raw NAL units) - let use_start_codes = self.config_bitstream_raw == 1; - let start_code_len = if use_start_codes { START_CODE.len() } else { 0 }; - - // Pre-calculate total size needed - let total_size: usize = slice_nals - .iter() - .map(|nal| start_code_len + nal.data.len()) - .sum(); - - // Add padding to align to 128 bytes (required by DXVA) - let padded_size = (total_size + 127) & !127; - - let mut bitstream = Vec::with_capacity(padded_size); - let mut slice_controls = Vec::with_capacity(slice_nals.len()); - - for nal in slice_nals { - // Record position before adding this slice - let position = bitstream.len() as u32; - - // Add start code only if ConfigBitstreamRaw=1 - if use_start_codes { - bitstream.extend_from_slice(&START_CODE); - } - - // Add NAL unit data (use the pre-parsed data from HevcNalUnit) - bitstream.extend_from_slice(&nal.data); - - // Create slice control (short format) - let slice_size = (start_code_len + nal.data.len()) as u32; - slice_controls.push(DxvaHevcSliceShort { - bs_nal_unit_data_location: position, - slice_bytes_in_buffer: slice_size, - w_bad_slice_chopping: 0, - }); - } - - // Add padding to align to 128 bytes (FFmpeg does this) - while bitstream.len() < padded_size { - bitstream.push(0); - } - - // Update last slice to include padding bytes - if let Some(last_slice) = slice_controls.last_mut() { - let padding = (padded_size - total_size) as u32; - last_slice.slice_bytes_in_buffer += padding; - } - - Ok((bitstream, slice_controls)) - } - - /// Build HEVC picture parameters from parsed data - /// This fills the DXVA_PicParams_HEVC structure according to Microsoft specification - fn build_hevc_pic_params( - &self, - sps: &super::hevc_parser::HevcSps, - pps: &super::hevc_parser::HevcPps, - nal: &super::hevc_parser::HevcNalUnit, - slice_header: &super::hevc_parser::HevcSliceHeader, - surface_idx: u32, - full_poc: i32, - ) -> Result { - let mut pp = DxvaHevcPicParams::default(); - - // Calculate MinCbSizeY = 1 << (log2_min_luma_coding_block_size_minus3 + 3) - let min_cb_log2 = sps.log2_min_luma_coding_block_size; - let min_cb_size = 1u32 << min_cb_log2; - - // PicWidthInMinCbsY = pic_width / MinCbSizeY - // PicHeightInMinCbsY = pic_height / MinCbSizeY - pp.pic_width_in_min_cbs_y = (sps.pic_width / min_cb_size) as u16; - pp.pic_height_in_min_cbs_y = (sps.pic_height / min_cb_size) as u16; - - // wFormatAndSequenceInfoFlags - packed bitfield: - // chroma_format_idc:2, separate_colour_plane_flag:1, bit_depth_luma_minus8:3, - // bit_depth_chroma_minus8:3, log2_max_pic_order_cnt_lsb_minus4:4, - // NoPicReorderingFlag:1, NoBiPredFlag:1, ReservedBits1:1 - let chroma_format = (sps.chroma_format_idc as u16) & 0x3; - let separate_colour_plane = ((sps.separate_colour_plane as u16) & 0x1) << 2; - let bit_depth_luma = (((sps.bit_depth_luma - 8) as u16) & 0x7) << 3; - let bit_depth_chroma = (((sps.bit_depth_chroma - 8) as u16) & 0x7) << 6; - let log2_max_poc = (((sps.log2_max_poc_lsb - 4) as u16) & 0xF) << 9; - // NoPicReorderingFlag and NoBiPredFlag are typically 0 - pp.w_format_and_sequence_info_flags = chroma_format - | separate_colour_plane - | bit_depth_luma - | bit_depth_chroma - | log2_max_poc; - - // Current picture - must match the surface index used in DecoderBeginFrame - pp.curr_pic = DxvaPicEntryHevc::new(surface_idx as u8, false); - - // SPS parameters - // max_dec_pic_buffering not in HevcSps, use a default of 5 (common value) - pp.sps_max_dec_pic_buffering_minus1 = 4; // 5 - 1 = 4 - pp.log2_min_luma_coding_block_size_minus3 = - sps.log2_min_luma_coding_block_size.saturating_sub(3); - pp.log2_diff_max_min_luma_coding_block_size = sps.log2_diff_max_min_luma_coding_block_size; - pp.log2_min_transform_block_size_minus2 = - sps.log2_min_luma_transform_block_size.saturating_sub(2); - pp.log2_diff_max_min_transform_block_size = sps.log2_diff_max_min_luma_transform_block_size; - pp.max_transform_hierarchy_depth_inter = sps.max_transform_hierarchy_depth_inter; - pp.max_transform_hierarchy_depth_intra = sps.max_transform_hierarchy_depth_intra; - pp.num_short_term_ref_pic_sets = sps.num_short_term_ref_pic_sets; - pp.num_long_term_ref_pics_sps = sps.num_long_term_ref_pics_sps; - pp.num_ref_idx_l0_default_active_minus1 = - pps.num_ref_idx_l0_default_active.saturating_sub(1); - pp.num_ref_idx_l1_default_active_minus1 = - pps.num_ref_idx_l1_default_active.saturating_sub(1); - pp.init_qp_minus26 = (pps.init_qp as i8) - 26; - - // dwCodingParamToolFlags - packed bitfield for SPS/PPS tool flags - let mut tool_flags: u32 = 0; - tool_flags |= (sps.scaling_list_enabled as u32) << 0; - tool_flags |= (sps.amp_enabled as u32) << 1; - tool_flags |= (sps.sample_adaptive_offset_enabled as u32) << 2; - tool_flags |= (sps.pcm_enabled as u32) << 3; - if sps.pcm_enabled { - tool_flags |= ((sps.pcm_sample_bit_depth_luma.saturating_sub(1) as u32) & 0xF) << 4; - tool_flags |= ((sps.pcm_sample_bit_depth_chroma.saturating_sub(1) as u32) & 0xF) << 8; - tool_flags |= - ((sps.log2_min_pcm_luma_coding_block_size.saturating_sub(3) as u32) & 0x3) << 12; - tool_flags |= ((sps.log2_diff_max_min_pcm_luma_coding_block_size as u32) & 0x3) << 14; - tool_flags |= (sps.pcm_loop_filter_disabled as u32) << 16; - } - tool_flags |= (sps.long_term_ref_pics_present as u32) << 17; - tool_flags |= (sps.temporal_mvp_enabled as u32) << 18; - tool_flags |= (sps.strong_intra_smoothing_enabled as u32) << 19; - tool_flags |= (pps.dependent_slice_segments_enabled as u32) << 20; - tool_flags |= (pps.output_flag_present as u32) << 21; - tool_flags |= ((pps.num_extra_slice_header_bits as u32) & 0x7) << 22; - tool_flags |= (pps.sign_data_hiding_enabled as u32) << 25; - tool_flags |= (pps.cabac_init_present as u32) << 26; - pp.dw_coding_param_tool_flags = tool_flags; - - // dwCodingSettingPicturePropertyFlags - packed bitfield for picture properties - let mut pic_flags: u32 = 0; - pic_flags |= (pps.constrained_intra_pred as u32) << 0; - pic_flags |= (pps.transform_skip_enabled as u32) << 1; - pic_flags |= (pps.cu_qp_delta_enabled as u32) << 2; - pic_flags |= (pps.slice_chroma_qp_offsets_present as u32) << 3; - pic_flags |= (pps.weighted_pred as u32) << 4; - pic_flags |= (pps.weighted_bipred as u32) << 5; - pic_flags |= (pps.transquant_bypass_enabled as u32) << 6; - pic_flags |= (pps.tiles_enabled as u32) << 7; - pic_flags |= (pps.entropy_coding_sync_enabled as u32) << 8; - pic_flags |= (pps.uniform_spacing as u32) << 9; - pic_flags |= (pps.loop_filter_across_tiles_enabled as u32) << 10; - pic_flags |= (pps.loop_filter_across_slices_enabled as u32) << 11; - pic_flags |= (pps.deblocking_filter_override_enabled as u32) << 12; - pic_flags |= (pps.deblocking_filter_disabled as u32) << 13; - pic_flags |= (pps.lists_modification_present as u32) << 14; - pic_flags |= (pps.slice_segment_header_extension_present as u32) << 15; - // IrapPicFlag, IdrPicFlag, IntraPicFlag - let is_irap = nal.nal_type.is_rap(); - let is_idr = nal.nal_type.is_idr(); - let is_intra = slice_header.slice_type == 2; // I-slice - pic_flags |= (is_irap as u32) << 16; - pic_flags |= (is_idr as u32) << 17; - pic_flags |= (is_intra as u32) << 18; - pp.dw_coding_setting_picture_property_flags = pic_flags; - - // PPS QP offsets - pp.pps_cb_qp_offset = pps.cb_qp_offset; - pp.pps_cr_qp_offset = pps.cr_qp_offset; - - // Tiles - if pps.tiles_enabled { - pp.num_tile_columns_minus1 = pps.num_tile_columns.saturating_sub(1) as u8; - pp.num_tile_rows_minus1 = pps.num_tile_rows.saturating_sub(1) as u8; - // column_width_minus1 and row_height_minus1 arrays would be filled here - // For uniform spacing, these aren't needed - } - - // Deblocking - pp.diff_cu_qp_delta_depth = pps.diff_cu_qp_delta_depth; - pp.pps_beta_offset_div2 = pps.beta_offset / 2; - pp.pps_tc_offset_div2 = pps.tc_offset / 2; - pp.log2_parallel_merge_level_minus2 = pps.log2_parallel_merge_level.saturating_sub(2); - - // Current picture POC - use the full POC (includes MSB) passed from decode_frame - // The full POC is calculated using ITU-T H.265 section 8.3.1 algorithm - let current_poc = full_poc; - pp.curr_pic_order_cnt_val = current_poc; - - // Reference picture list - populate from DPB - // First, mark all as invalid - for i in 0..15 { - pp.ref_pic_list[i] = DxvaPicEntryHevc::invalid(); - pp.pic_order_cnt_val_list[i] = 0; - } - - // Initialize reference picture sets to invalid - for i in 0..8 { - pp.ref_pic_set_st_curr_before[i] = 0xFF; - pp.ref_pic_set_st_curr_after[i] = 0xFF; - pp.ref_pic_set_lt_curr[i] = 0xFF; - } - - // For IDR frames, DPB should already be cleared - no references needed - // For non-IDR frames, fill reference picture list from DPB - if !is_idr && !self.dpb.is_empty() { - // Sort DPB entries by POC for proper reference ordering - // RefPicSetStCurrBefore: short-term refs with POC < current POC (most recent first) - // RefPicSetStCurrAfter: short-term refs with POC > current POC (not used for P-frames) - - let mut ref_idx = 0; - let mut st_curr_before_idx = 0; - let mut st_curr_after_idx = 0; - - // Collect and sort references by POC (descending for before, ascending for after) - let mut refs_before: Vec<_> = self - .dpb - .iter() - .filter(|e| e.is_reference && !e.is_long_term && e.poc < current_poc) - .collect(); - refs_before.sort_by(|a, b| b.poc.cmp(&a.poc)); // Most recent first - - let mut refs_after: Vec<_> = self - .dpb - .iter() - .filter(|e| e.is_reference && !e.is_long_term && e.poc > current_poc) - .collect(); - refs_after.sort_by(|a, b| a.poc.cmp(&b.poc)); // Closest first - - // Add references before current POC - for dpb_entry in &refs_before { - if ref_idx >= 15 { - break; - } - pp.ref_pic_list[ref_idx] = DxvaPicEntryHevc::new(dpb_entry.surface_index, false); - pp.pic_order_cnt_val_list[ref_idx] = dpb_entry.poc; - - if st_curr_before_idx < 8 { - pp.ref_pic_set_st_curr_before[st_curr_before_idx] = ref_idx as u8; - st_curr_before_idx += 1; - } - ref_idx += 1; - } - - // Add references after current POC (for B-frames) - for dpb_entry in &refs_after { - if ref_idx >= 15 { - break; - } - pp.ref_pic_list[ref_idx] = DxvaPicEntryHevc::new(dpb_entry.surface_index, false); - pp.pic_order_cnt_val_list[ref_idx] = dpb_entry.poc; - - if st_curr_after_idx < 8 { - pp.ref_pic_set_st_curr_after[st_curr_after_idx] = ref_idx as u8; - st_curr_after_idx += 1; - } - ref_idx += 1; - } - - // Add long-term references if any - for dpb_entry in &self.dpb { - if ref_idx >= 15 { - break; - } - if dpb_entry.is_reference && dpb_entry.is_long_term { - pp.ref_pic_list[ref_idx] = DxvaPicEntryHevc::new(dpb_entry.surface_index, true); - pp.pic_order_cnt_val_list[ref_idx] = dpb_entry.poc; - // Long-term refs go in ref_pic_set_lt_curr - ref_idx += 1; - } - } - } - - // Status report feedback number (used for debugging) - pp.status_report_feedback_number = 1; - - Ok(pp) - } - - /// Update the DPB (Decoded Picture Buffer) after decoding a frame - fn update_dpb(&mut self, surface_idx: u32, poc: i32, is_reference: bool, is_idr: bool) { - // Increment frame counter - self.frame_count += 1; - - // Clear DPB on IDR frames and reset POC tracking - if is_idr { - self.dpb.clear(); - self.prev_poc_lsb = 0; - self.prev_poc_msb = 0; - } - - // Remove any existing entry with this surface index (surface is being reused) - self.dpb - .retain(|entry| entry.surface_index != surface_idx as u8); - - // Remove oldest entries if DPB is full (keep most recent by frame_num) - while self.dpb.len() >= self.dpb_max_size { - // Find entry with lowest frame_num (oldest) - if let Some(oldest_idx) = self - .dpb - .iter() - .enumerate() - .min_by_key(|(_, e)| e.frame_num) - .map(|(i, _)| i) - { - self.dpb.remove(oldest_idx); - } - } - - // Add current frame to DPB if it's a reference frame - if is_reference { - self.dpb.push(DpbEntry { - surface_index: surface_idx as u8, - poc, - is_reference: true, - is_long_term: false, - frame_num: self.frame_count, - }); - } - } - - /// Clear DPB (call on seek or error recovery) - pub fn clear_dpb(&mut self) { - self.dpb.clear(); - self.prev_poc_lsb = 0; - self.prev_poc_msb = 0; - self.frame_count = 0; - } - - /// Calculate full POC from POC LSB using the POC MSB derivation algorithm - /// This follows ITU-T H.265 section 8.3.1 - fn calculate_full_poc(&mut self, poc_lsb: i32, is_idr: bool, max_poc_lsb: i32) -> i32 { - // For IDR frames, POC is always 0 and we reset tracking - if is_idr { - self.prev_poc_lsb = 0; - self.prev_poc_msb = 0; - return 0; - } - - // Calculate POC MSB according to H.265 spec - let half_max_poc_lsb = max_poc_lsb / 2; - - let poc_msb = if poc_lsb < self.prev_poc_lsb - && (self.prev_poc_lsb - poc_lsb) >= half_max_poc_lsb - { - // POC LSB wrapped around (increased) - self.prev_poc_msb + max_poc_lsb - } else if poc_lsb > self.prev_poc_lsb && (poc_lsb - self.prev_poc_lsb) > half_max_poc_lsb { - // POC LSB wrapped around (decreased) - rare case - self.prev_poc_msb - max_poc_lsb - } else { - self.prev_poc_msb - }; - - let full_poc = poc_msb + poc_lsb; - - // Update tracking for next frame - // Note: Only update for reference frames in a real implementation - self.prev_poc_lsb = poc_lsb; - self.prev_poc_msb = poc_msb; - - full_poc - } - - /// Submit a buffer to the decoder - unsafe fn submit_buffer( - &self, - decoder: &ID3D11VideoDecoder, - buffer_type: DxvaBufferType, - data: *const u8, - size: u32, - ) -> Result<()> { - // Get buffer from decoder - let mut buffer_ptr: *mut std::ffi::c_void = std::ptr::null_mut(); - let mut buffer_size: u32 = 0; - - self.video_context - .GetDecoderBuffer( - decoder, - D3D11_VIDEO_DECODER_BUFFER_TYPE(buffer_type as i32), - &mut buffer_size, - &mut buffer_ptr, - ) - .map_err(|e| anyhow!("GetDecoderBuffer failed for {:?}: {:?}", buffer_type, e))?; - - if buffer_ptr.is_null() { - return Err(anyhow!( - "GetDecoderBuffer returned null for {:?}", - buffer_type - )); - } - - if size > buffer_size { - self.video_context - .ReleaseDecoderBuffer(decoder, D3D11_VIDEO_DECODER_BUFFER_TYPE(buffer_type as i32)) - .ok(); - return Err(anyhow!( - "Buffer too small for {:?}: need {} but got {}", - buffer_type, - size, - buffer_size - )); - } - - // Copy data to buffer - std::ptr::copy_nonoverlapping(data, buffer_ptr as *mut u8, size as usize); - - // Release buffer - self.video_context - .ReleaseDecoderBuffer(decoder, D3D11_VIDEO_DECODER_BUFFER_TYPE(buffer_type as i32)) - .map_err(|e| anyhow!("ReleaseDecoderBuffer failed for {:?}: {:?}", buffer_type, e))?; - - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_check_hevc_support() { - let result = DxvaDecoder::check_resolution_support(DxvaCodec::HEVC, 1920, 1080, false); - println!("HEVC 1080p support: {:?}", result); - } - - #[test] - fn test_check_hevc_4k_support() { - let result = DxvaDecoder::check_resolution_support(DxvaCodec::HEVC, 3840, 2160, false); - println!("HEVC 4K support: {:?}", result); - } - - #[test] - fn test_get_max_resolution() { - let result = DxvaDecoder::get_max_resolution(DxvaCodec::HEVC, false); - println!("HEVC max resolution: {:?}", result); - } - - #[test] - fn test_dxva_struct_sizes() { - // Verify structure sizes match Microsoft DXVA specification - - // DXVA_PicParams_HEVC should be 232 bytes (packed) - let hevc_pic_params_size = std::mem::size_of::(); - println!("DxvaHevcPicParams size: {} bytes", hevc_pic_params_size); - - // DXVA_Slice_HEVC_Short should be 10 bytes - let hevc_slice_short_size = std::mem::size_of::(); - println!("DxvaHevcSliceShort size: {} bytes", hevc_slice_short_size); - - // DXVA_Qmatrix_HEVC - let qmatrix_size = std::mem::size_of::(); - println!("DxvaHevcQMatrix size: {} bytes", qmatrix_size); - - // DxvaPicEntryHevc should be 1 byte - let hevc_pic_entry_size = std::mem::size_of::(); - println!("DxvaPicEntryHevc size: {} bytes", hevc_pic_entry_size); - assert_eq!(hevc_pic_entry_size, 1, "DxvaPicEntryHevc should be 1 byte"); - - // Slice short should be 10 bytes (4 + 4 + 2) - assert_eq!( - hevc_slice_short_size, 10, - "DxvaHevcSliceShort should be 10 bytes" - ); - - // PicParams size check - the packed struct should be around 232 bytes - assert!( - hevc_pic_params_size >= 200 && hevc_pic_params_size <= 256, - "DxvaHevcPicParams size {} is outside expected range 200-256", - hevc_pic_params_size - ); - } -} diff --git a/opennow-streamer/src/media/gstreamer_decoder.rs b/opennow-streamer/src/media/gstreamer_decoder.rs deleted file mode 100644 index 13c37ac..0000000 --- a/opennow-streamer/src/media/gstreamer_decoder.rs +++ /dev/null @@ -1,942 +0,0 @@ -//! GStreamer Video Decoder -//! -//! Hardware-accelerated video decoding using GStreamer. -//! -//! Platform support: -//! - Windows: H.264 via D3D11 hardware acceleration (d3d11h264dec) -//! - Linux (Raspberry Pi): H.264/HEVC via V4L2 (v4l2h264dec, v4l2h265dec) -//! - Linux (Desktop): H.264/HEVC via VA-API (vaapih264dec, vaapih265dec) -//! -//! Pipeline structures: -//! - Windows: appsrc -> h264parse -> d3d11h264dec -> d3d11download -> videoconvert -> appsink -//! - Linux V4L2: appsrc -> h264parse -> v4l2h264dec -> videoconvert -> appsink -//! - Linux VA-API: appsrc -> h264parse -> vaapih264dec -> videoconvert -> appsink -//! -//! ## Windows GStreamer Bundling -//! -//! On Windows, GStreamer can be bundled with the application. The decoder will look for -//! GStreamer in the following locations (in order): -//! 1. `gstreamer/` subdirectory next to the executable -//! 2. System-installed GStreamer (GSTREAMER_1_0_ROOT_MSVC_X86_64 environment variable) -//! 3. Standard GStreamer installation paths - -use anyhow::{anyhow, Result}; -use gstreamer as gst; -use gstreamer::prelude::*; -use gstreamer_app::{AppSink, AppSrc}; -use gstreamer_video as gst_video; -use log::{debug, info, warn}; -use std::str::FromStr; -use std::sync::{Arc, Mutex}; - -use super::{ColorRange, ColorSpace, PixelFormat, TransferFunction, VideoFrame}; - -/// Initialize GStreamer with support for bundled runtime on Windows -/// This function MUST be called before any other GStreamer operations. -/// It sets up the PATH and plugin paths for bundled GStreamer on Windows. -#[cfg(target_os = "windows")] -pub fn init_gstreamer() -> Result<()> { - use std::env; - use std::path::PathBuf; - use std::sync::Once; - - static INIT: Once = Once::new(); - static mut INIT_RESULT: Option> = None; - - // Thread-safe one-time initialization - INIT.call_once(|| { - // Try to find bundled GStreamer FIRST, before calling gst::init() - // The DLLs must be in PATH before GStreamer tries to load them - let exe_dir = env::current_exe() - .ok() - .and_then(|p| p.parent().map(|p| p.to_path_buf())) - .unwrap_or_else(|| PathBuf::from(".")); - - // GStreamer core DLLs are next to the exe (for load-time linking) - // Plugins are in lib/gstreamer-1.0/ subdirectory - let bundled_plugins = exe_dir.join("lib").join("gstreamer-1.0"); - - // Check if we have bundled GStreamer (look for a core DLL next to exe) - let has_bundled_gst = exe_dir.join("gstreamer-1.0-0.dll").exists(); - - if has_bundled_gst { - info!("Found bundled GStreamer DLLs at: {}", exe_dir.display()); - - // Set plugin path - if bundled_plugins.exists() { - env::set_var("GST_PLUGIN_PATH", bundled_plugins.to_str().unwrap_or("")); - info!("Set GST_PLUGIN_PATH to: {}", bundled_plugins.display()); - } - - // Disable plugin scanning outside bundled path for faster startup - env::set_var("GST_PLUGIN_SYSTEM_PATH", ""); - } else { - // Check for system GStreamer - if let Ok(gst_root) = env::var("GSTREAMER_1_0_ROOT_MSVC_X86_64") { - info!("Using system GStreamer from: {}", gst_root); - } else { - warn!("GStreamer not found. Please install GStreamer or bundle it with the app."); - warn!("Download from: https://gstreamer.freedesktop.org/download/"); - } - } - - // Now initialize GStreamer after PATH is set up - unsafe { - INIT_RESULT = Some(gst::init().map_err(|e| e.to_string())); - } - }); - - // Return cached result - unsafe { - match &INIT_RESULT { - Some(Ok(())) => Ok(()), - Some(Err(e)) => Err(anyhow!("Failed to initialize GStreamer: {}", e)), - None => Err(anyhow!("GStreamer initialization not completed")), - } - } -} - -/// Initialize GStreamer (Linux - straightforward) -#[cfg(target_os = "linux")] -pub fn init_gstreamer() -> Result<()> { - gst::init().map_err(|e| anyhow!("Failed to initialize GStreamer: {}", e)) -} - -/// GStreamer codec type -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum GstCodec { - H264, - H265, - AV1, -} - -impl GstCodec { - fn caps_string(&self) -> &'static str { - match self { - GstCodec::H264 => "video/x-h264,stream-format=byte-stream,alignment=au", - GstCodec::H265 => "video/x-h265,stream-format=byte-stream,alignment=au", - GstCodec::AV1 => "video/x-av1,stream-format=obu-stream,alignment=tu", - } - } - - fn parser_element(&self) -> &'static str { - match self { - GstCodec::H264 => "h264parse", - GstCodec::H265 => "h265parse", - GstCodec::AV1 => "av1parse", - } - } - - /// Get the best available decoder element for this codec on the current platform - #[cfg(target_os = "windows")] - fn decoder_element(&self) -> &'static str { - match self { - // Windows: Use D3D11 hardware decoder for best performance - // Falls back to software if D3D11 decoder not available - GstCodec::H264 => "d3d11h264dec", - GstCodec::H265 => "d3d11h265dec", - GstCodec::AV1 => "d3d11av1dec", - } - } - - #[cfg(target_os = "linux")] - fn decoder_element(&self) -> &'static str { - // Linux: V4L2 for embedded (RPi), otherwise VA-API or software - match self { - GstCodec::H264 => "v4l2h264dec", - GstCodec::H265 => "v4l2h265dec", - GstCodec::AV1 => "v4l2av1dec", // Raspberry Pi 5 supports AV1 - } - } - - /// Get fallback software decoder - fn software_decoder(&self) -> &'static str { - match self { - GstCodec::H264 => "avdec_h264", - GstCodec::H265 => "avdec_h265", - GstCodec::AV1 => "av1dec", // dav1d-based decoder (preferred) or avdec_av1 - } - } -} - -/// GStreamer decoder configuration -#[derive(Debug, Clone)] -pub struct GstDecoderConfig { - pub codec: GstCodec, - pub width: u32, - pub height: u32, - /// Enable low latency mode (minimize buffering) - pub low_latency: bool, -} - -impl Default for GstDecoderConfig { - fn default() -> Self { - Self { - codec: GstCodec::H264, - width: 1920, - height: 1080, - low_latency: true, // Default to low latency for streaming - } - } -} - -/// Decoded frame from GStreamer -struct DecodedFrame { - width: u32, - height: u32, - y_plane: Vec, - uv_plane: Vec, - y_stride: u32, - uv_stride: u32, - /// Timestamp when frame was decoded (for latency tracking) - decode_time: std::time::Instant, - /// Color space from GStreamer colorimetry - color_space: ColorSpace, - /// Transfer function (SDR/PQ/HLG) from GStreamer colorimetry - transfer_function: TransferFunction, - /// Color range (Limited/Full) - color_range: ColorRange, -} - -/// GStreamer Video Decoder -/// -/// Cross-platform hardware-accelerated video decoder using GStreamer. -/// - Windows: D3D11 hardware acceleration -/// - Linux: V4L2 (embedded) or VA-API (desktop) -pub struct GStreamerDecoder { - pipeline: gst::Pipeline, - appsrc: AppSrc, - #[allow(dead_code)] - appsink: AppSink, - #[allow(dead_code)] - config: GstDecoderConfig, - frame_count: u64, - last_frame: Arc>>, - /// Last logged transfer function (to avoid log spam) - last_logged_transfer: TransferFunction, -} - -// GStreamer is thread-safe -unsafe impl Send for GStreamerDecoder {} -unsafe impl Sync for GStreamerDecoder {} - -impl GStreamerDecoder { - /// Create a new GStreamer decoder - pub fn new(config: GstDecoderConfig) -> Result { - info!( - "Creating GStreamer decoder: {:?} {}x{}", - config.codec, config.width, config.height - ); - - // Initialize GStreamer (with bundled DLL support on Windows) - init_gstreamer()?; - - // Build platform-specific pipeline - let pipeline_str = Self::build_pipeline_string(&config)?; - info!("GStreamer pipeline: {}", pipeline_str); - - // Parse and create pipeline - let pipeline = gst::parse::launch(&pipeline_str) - .map_err(|e| anyhow!("Failed to create GStreamer pipeline: {}", e))? - .downcast::() - .map_err(|_| anyhow!("Failed to downcast to Pipeline"))?; - - // Get appsrc - let appsrc = pipeline - .by_name("src") - .ok_or_else(|| anyhow!("Failed to get appsrc"))? - .downcast::() - .map_err(|_| anyhow!("Failed to downcast to AppSrc"))?; - - // Configure appsrc for ultra-low latency live streaming - let caps = gst::Caps::from_str(config.codec.caps_string()) - .map_err(|e| anyhow!("Failed to create caps: {}", e))?; - appsrc.set_caps(Some(&caps)); - appsrc.set_format(gst::Format::Time); - - // Critical low-latency settings (from GStreamer docs): - // - stream-type=Stream for live streaming push mode - // - max-bytes=0 disables internal buffering - // - block=false prevents blocking when buffer is full - // - min-latency=0 required when using do-timestamp=true - // (appsrc timestamps based on running-time when buffer arrives) - appsrc.set_stream_type(gstreamer_app::AppStreamType::Stream); - appsrc.set_max_bytes(0); - appsrc.set_property("block", false); - appsrc.set_property("min-latency", 0i64); - appsrc.set_property("max-latency", 0i64); - - // Get appsink - let appsink = pipeline - .by_name("sink") - .ok_or_else(|| anyhow!("Failed to get appsink"))? - .downcast::() - .map_err(|_| anyhow!("Failed to downcast to AppSink"))?; - - // Configure appsink for NV12 output with minimal latency - let sink_caps = gst::Caps::builder("video/x-raw") - .field("format", "NV12") - .build(); - appsink.set_caps(Some(&sink_caps)); - - // Ultra-low latency sink settings: - // - drop=true drops old frames if we can't keep up - // - max-buffers=1 only keeps the latest frame - // - sync=false renders immediately without clock sync - appsink.set_drop(true); - appsink.set_max_buffers(1); - appsink.set_sync(false); - - // Set up frame storage - let last_frame = Arc::new(Mutex::new(None)); - let last_frame_clone = last_frame.clone(); - - // Set up new-sample callback - appsink.set_callbacks( - gstreamer_app::AppSinkCallbacks::builder() - .new_sample(move |sink| { - match sink.pull_sample() { - Ok(sample) => { - if let Some(buffer) = sample.buffer() { - if let Some(caps) = sample.caps() { - if let Ok(video_info) = gst_video::VideoInfo::from_caps(caps) { - let width = video_info.width(); - let height = video_info.height(); - - // Extract colorimetry info for HDR detection - let colorimetry = video_info.colorimetry(); - - // Detect transfer function (SDR vs HDR PQ vs HLG) - // Use raw GLib values to detect PQ/HLG without v1_18 feature - // GST_VIDEO_TRANSFER_SMPTE2084 = 14, GST_VIDEO_TRANSFER_ARIB_STD_B67 = 15 - use gstreamer::glib::translate::IntoGlib; - let transfer_raw = colorimetry.transfer().into_glib(); - let transfer_function = match transfer_raw { - 14 => { - // SMPTE ST 2084 = PQ (HDR10) - TransferFunction::PQ - } - 15 => { - // ARIB STD-B67 = HLG - TransferFunction::HLG - } - _ => TransferFunction::SDR, - }; - - // Detect color space (BT.709 vs BT.2020) - let color_space = match colorimetry.matrix() { - gst_video::VideoColorMatrix::Bt2020 => { - ColorSpace::BT2020 - } - gst_video::VideoColorMatrix::Bt601 => ColorSpace::BT601, - _ => ColorSpace::BT709, - }; - - // Detect color range (Limited vs Full) - let color_range = match colorimetry.range() { - gst_video::VideoColorRange::Range0_255 => { - ColorRange::Full - } - _ => ColorRange::Limited, - }; - - // Map buffer for reading - if let Ok(map) = buffer.map_readable() { - let data = map.as_slice(); - - // NV12 format: Y plane followed by interleaved UV - let y_stride = video_info.stride()[0] as u32; - let uv_stride = video_info.stride()[1] as u32; - let y_size = (y_stride * height) as usize; - let uv_size = (uv_stride * height / 2) as usize; - - if data.len() >= y_size + uv_size { - let y_plane = data[..y_size].to_vec(); - let uv_plane = - data[y_size..y_size + uv_size].to_vec(); - - let frame = DecodedFrame { - width, - height, - y_plane, - uv_plane, - y_stride, - uv_stride, - decode_time: std::time::Instant::now(), - color_space, - transfer_function, - color_range, - }; - - *last_frame_clone.lock().unwrap() = Some(frame); - } - } - } - } - } - Ok(gst::FlowSuccess::Ok) - } - Err(_) => Err(gst::FlowError::Error), - } - }) - .build(), - ); - - // Set up bus message monitoring for errors and state changes - let bus = pipeline.bus().expect("Pipeline has no bus"); - std::thread::spawn(move || { - for msg in bus.iter_timed(gst::ClockTime::NONE) { - use gst::MessageView; - match msg.view() { - MessageView::Error(err) => { - log::error!( - "GStreamer Error from {:?}: {} ({:?})", - err.src().map(|s| s.path_string()), - err.error(), - err.debug() - ); - } - MessageView::Warning(warn) => { - log::warn!( - "GStreamer Warning from {:?}: {} ({:?})", - warn.src().map(|s| s.path_string()), - warn.error(), - warn.debug() - ); - } - MessageView::StateChanged(state) => { - if state - .src() - .map(|s| s.path_string().contains("pipeline")) - .unwrap_or(false) - { - log::debug!( - "GStreamer pipeline state: {:?} -> {:?}", - state.old(), - state.current() - ); - } - } - MessageView::Eos(_) => { - log::warn!("GStreamer: End of stream received"); - } - _ => {} - } - } - }); - - // Start pipeline - pipeline - .set_state(gst::State::Playing) - .map_err(|e| anyhow!("Failed to start pipeline: {:?}", e))?; - - info!("GStreamer decoder initialized successfully"); - - Ok(Self { - pipeline, - appsrc, - appsink, - config, - frame_count: 0, - last_frame, - last_logged_transfer: TransferFunction::SDR, - }) - } - - /// Build the GStreamer pipeline string for the current platform - fn build_pipeline_string(config: &GstDecoderConfig) -> Result { - let parser = config.codec.parser_element(); - let decoder = config.codec.decoder_element(); - - // Low latency sink options - critical for streaming - // sync=false renders frames immediately without clock sync - // max-buffers=1 drop=true ensures we always get the latest frame - // wait-on-eos=false prevents blocking on end-of-stream - let sink_opts = if config.low_latency { - "max-buffers=1 drop=true sync=false wait-on-eos=false" - } else { - "max-buffers=2 drop=false sync=false wait-on-eos=false" - }; - - // Check if the hardware decoder is available - let registry = gst::Registry::get(); - let hw_decoder_available = registry - .find_feature(decoder, gst::ElementFactory::static_type()) - .is_some(); - - #[cfg(target_os = "windows")] - { - if hw_decoder_available { - // Windows D3D11 hardware decoder pipeline - ULTRA LOW LATENCY - // d3d11h264dec outputs D3D11 textures, need d3d11download to copy to system memory - // - // Key optimizations: - // - NO queue element (queues add latency for thread sync) - // - is-live=true on appsrc for real-time behavior - // - sync=false on appsink to render immediately - // - videoconvert with n-threads for parallel color conversion - info!("Using D3D11 hardware decoder: {}", decoder); - Ok(format!( - "appsrc name=src is-live=true format=time do-timestamp=true max-buffers=1 \ - ! {} \ - ! {} \ - ! d3d11download \ - ! videoconvert n-threads=2 \ - ! video/x-raw,format=NV12 \ - ! appsink name=sink emit-signals=true {}", - parser, decoder, sink_opts - )) - } else { - // Fallback to software decoder - still optimized for low latency - let sw_decoder = config.codec.software_decoder(); - warn!( - "D3D11 decoder {} not available, falling back to software: {}", - decoder, sw_decoder - ); - Ok(format!( - "appsrc name=src is-live=true format=time do-timestamp=true max-buffers=1 \ - ! {} \ - ! {} \ - ! videoconvert n-threads=4 \ - ! video/x-raw,format=NV12 \ - ! appsink name=sink emit-signals=true {}", - parser, sw_decoder, sink_opts - )) - } - } - - #[cfg(target_os = "linux")] - { - // Linux decoder priority (from best to fallback): - // 1. V4L2 (Raspberry Pi, embedded devices with hardware codec) - // 2. VA (newer va plugin - vah264dec/vah265dec/vaav1dec) for Intel/AMD - // 3. VAAPI (legacy vaapi plugin - vaapih264dec/vaapih265dec) - // 4. Software (avdec_h264/avdec_h265/av1dec) - - // Check for V4L2 decoder (Raspberry Pi - RPi5 supports AV1) - let v4l2_decoder = match config.codec { - GstCodec::H264 => "v4l2h264dec", - GstCodec::H265 => "v4l2h265dec", - GstCodec::AV1 => "v4l2av1dec", - }; - let v4l2_available = registry - .find_feature(v4l2_decoder, gst::ElementFactory::static_type()) - .is_some(); - - // Check for new VA plugin decoders (preferred for desktop Linux) - // Intel Arc, AMD RDNA2+, and modern Intel iGPUs support AV1 - let va_decoder = match config.codec { - GstCodec::H264 => "vah264dec", - GstCodec::H265 => "vah265dec", - GstCodec::AV1 => "vaav1dec", - }; - let va_available = registry - .find_feature(va_decoder, gst::ElementFactory::static_type()) - .is_some(); - - // Check for legacy VAAPI decoders (fallback for older systems) - // Note: VAAPI AV1 uses same naming as VA plugin - let vaapi_decoder = match config.codec { - GstCodec::H264 => "vaapih264dec", - GstCodec::H265 => "vaapih265dec", - GstCodec::AV1 => "vaapiav1dec", // May not exist on all systems - }; - let vaapi_available = registry - .find_feature(vaapi_decoder, gst::ElementFactory::static_type()) - .is_some(); - - if v4l2_available { - // Raspberry Pi / embedded V4L2 hardware decoder - ULTRA LOW LATENCY - // V4L2 decoders output directly to DMA buffers - info!( - "Using V4L2 hardware decoder: {} (Raspberry Pi / embedded)", - v4l2_decoder - ); - Ok(format!( - "appsrc name=src is-live=true format=time do-timestamp=true max-buffers=1 \ - ! {} \ - ! {} \ - ! videoconvert n-threads=2 \ - ! video/x-raw,format=NV12 \ - ! appsink name=sink emit-signals=true {}", - parser, v4l2_decoder, sink_opts - )) - } else if va_available { - // Modern VA plugin (Intel/AMD desktop Linux) - LOW LATENCY - // va plugin is the newer, preferred method for VAAPI - info!( - "Using VA hardware decoder: {} (Intel/AMD via va plugin)", - va_decoder - ); - Ok(format!( - "appsrc name=src is-live=true format=time do-timestamp=true max-buffers=1 \ - ! {} \ - ! {} \ - ! videoconvert n-threads=2 \ - ! video/x-raw,format=NV12 \ - ! appsink name=sink emit-signals=true {}", - parser, va_decoder, sink_opts - )) - } else if vaapi_available { - // Legacy VAAPI plugin (older systems) - LOW LATENCY - info!("Using legacy VAAPI hardware decoder: {}", vaapi_decoder); - Ok(format!( - "appsrc name=src is-live=true format=time do-timestamp=true max-buffers=1 \ - ! {} \ - ! {} \ - ! videoconvert n-threads=2 \ - ! video/x-raw,format=NV12 \ - ! appsink name=sink emit-signals=true {}", - parser, vaapi_decoder, sink_opts - )) - } else { - // Fallback to software decoder - let sw_decoder = config.codec.software_decoder(); - warn!( - "No hardware decoder available for {:?}, falling back to software: {}", - config.codec, sw_decoder - ); - warn!("For hardware acceleration, install: libva (Intel/AMD) or enable V4L2 (Raspberry Pi)"); - Ok(format!( - "appsrc name=src is-live=true format=time do-timestamp=true max-buffers=1 \ - ! {} \ - ! {} \ - ! videoconvert n-threads=4 \ - ! video/x-raw,format=NV12 \ - ! appsink name=sink emit-signals=true {}", - parser, sw_decoder, sink_opts - )) - } - } - } - - /// Decode a video frame - pub fn decode(&mut self, nal_data: &[u8]) -> Result> { - if nal_data.is_empty() { - return Ok(None); - } - - // Create GStreamer buffer from NAL data - let mut buffer = gst::Buffer::with_size(nal_data.len()) - .map_err(|e| anyhow!("Failed to create buffer: {}", e))?; - - { - let buffer_ref = buffer.get_mut().unwrap(); - let mut map = buffer_ref - .map_writable() - .map_err(|e| anyhow!("Failed to map buffer: {}", e))?; - map.copy_from_slice(nal_data); - } - - // Push buffer to pipeline - match self.appsrc.push_buffer(buffer) { - Ok(_) => {} - Err(e) => { - warn!("Failed to push buffer: {:?}", e); - return Ok(None); - } - } - - self.frame_count += 1; - - // Check for decoded frame - let frame = self.last_frame.lock().unwrap().take(); - - if let Some(decoded) = frame { - debug!( - "Decoded frame {}: {}x{}", - self.frame_count, decoded.width, decoded.height - ); - - // Log when transfer function changes (HDR/SDR switch) - if decoded.transfer_function != self.last_logged_transfer { - log::info!( - "Transfer function changed: {:?} -> {:?} ({:?} {:?})", - self.last_logged_transfer, - decoded.transfer_function, - decoded.color_space, - decoded.color_range - ); - self.last_logged_transfer = decoded.transfer_function; - } - - Ok(Some(VideoFrame { - frame_id: super::next_frame_id(), - width: decoded.width, - height: decoded.height, - y_plane: decoded.y_plane, - u_plane: decoded.uv_plane, - v_plane: Vec::new(), // NV12 has interleaved UV in u_plane - y_stride: decoded.y_stride, - u_stride: decoded.uv_stride, - v_stride: 0, - timestamp_us: 0, - format: PixelFormat::NV12, - color_range: decoded.color_range, - color_space: decoded.color_space, - transfer_function: decoded.transfer_function, - gpu_frame: None, - })) - } else { - Ok(None) - } - } - - /// Get frame count - pub fn frame_count(&self) -> u64 { - self.frame_count - } -} - -impl Drop for GStreamerDecoder { - fn drop(&mut self) { - info!("Stopping GStreamer pipeline"); - let _ = self.pipeline.set_state(gst::State::Null); - } -} - -/// Check if GStreamer hardware decoding is available -#[cfg(target_os = "windows")] -pub fn is_gstreamer_available() -> bool { - // Initialize GStreamer (with bundled DLL support) - if init_gstreamer().is_err() { - return false; - } - - // Check for D3D11 hardware decoders - let registry = gst::Registry::get(); - let d3d11_h264 = registry - .find_feature("d3d11h264dec", gst::ElementFactory::static_type()) - .is_some(); - let d3d11_h265 = registry - .find_feature("d3d11h265dec", gst::ElementFactory::static_type()) - .is_some(); - let d3d11_av1 = registry - .find_feature("d3d11av1dec", gst::ElementFactory::static_type()) - .is_some(); - let avdec_h264 = registry - .find_feature("avdec_h264", gst::ElementFactory::static_type()) - .is_some(); - let av1dec = registry - .find_feature("av1dec", gst::ElementFactory::static_type()) - .is_some(); - - if d3d11_h264 || d3d11_h265 || d3d11_av1 { - info!( - "GStreamer D3D11 decoders available: H.264={}, H.265={}, AV1={}", - d3d11_h264, d3d11_h265, d3d11_av1 - ); - true - } else if avdec_h264 || av1dec { - info!( - "GStreamer software decoders available: H.264={}, AV1={}", - avdec_h264, av1dec - ); - true - } else { - debug!("GStreamer decoders not available"); - false - } -} - -/// Check if GStreamer V4L2 decoding is available (Linux - Raspberry Pi) -#[cfg(target_os = "linux")] -pub fn is_gstreamer_v4l2_available() -> bool { - // Initialize GStreamer if needed - if init_gstreamer().is_err() { - return false; - } - - // Check for V4L2 decoders (RPi5 supports AV1) - let registry = gst::Registry::get(); - let h264_available = registry - .find_feature("v4l2h264dec", gst::ElementFactory::static_type()) - .is_some(); - let h265_available = registry - .find_feature("v4l2h265dec", gst::ElementFactory::static_type()) - .is_some(); - let av1_available = registry - .find_feature("v4l2av1dec", gst::ElementFactory::static_type()) - .is_some(); - - if h264_available || h265_available || av1_available { - info!( - "GStreamer V4L2 decoders available: H.264={}, H.265={}, AV1={}", - h264_available, h265_available, av1_available - ); - true - } else { - debug!("GStreamer V4L2 decoders not available"); - false - } -} - -/// Check if GStreamer VA (VAAPI) decoding is available (Linux - Intel/AMD) -#[cfg(target_os = "linux")] -pub fn is_gstreamer_va_available() -> bool { - // Initialize GStreamer if needed - if init_gstreamer().is_err() { - return false; - } - - let registry = gst::Registry::get(); - - // Check new VA plugin (preferred) - Intel Arc/AMD RDNA2+ support AV1 - let va_h264 = registry - .find_feature("vah264dec", gst::ElementFactory::static_type()) - .is_some(); - let va_h265 = registry - .find_feature("vah265dec", gst::ElementFactory::static_type()) - .is_some(); - let va_av1 = registry - .find_feature("vaav1dec", gst::ElementFactory::static_type()) - .is_some(); - - // Check legacy VAAPI plugin (fallback) - let vaapi_h264 = registry - .find_feature("vaapih264dec", gst::ElementFactory::static_type()) - .is_some(); - let vaapi_h265 = registry - .find_feature("vaapih265dec", gst::ElementFactory::static_type()) - .is_some(); - - if va_h264 || va_h265 || va_av1 { - info!( - "GStreamer VA decoders available: H.264={}, H.265={}, AV1={}", - va_h264, va_h265, va_av1 - ); - true - } else if vaapi_h264 || vaapi_h265 { - info!( - "GStreamer legacy VAAPI decoders available: H.264={}, H.265={}", - vaapi_h264, vaapi_h265 - ); - true - } else { - debug!("GStreamer VA/VAAPI decoders not available"); - false - } -} - -/// Check if any GStreamer hardware decoding is available (Linux) -#[cfg(target_os = "linux")] -pub fn is_gstreamer_available() -> bool { - // Initialize GStreamer if needed - if init_gstreamer().is_err() { - return false; - } - - let registry = gst::Registry::get(); - - // Check all available decoders (H.264, H.265, AV1) - let v4l2_h264 = registry - .find_feature("v4l2h264dec", gst::ElementFactory::static_type()) - .is_some(); - let v4l2_h265 = registry - .find_feature("v4l2h265dec", gst::ElementFactory::static_type()) - .is_some(); - let v4l2_av1 = registry - .find_feature("v4l2av1dec", gst::ElementFactory::static_type()) - .is_some(); - let va_h264 = registry - .find_feature("vah264dec", gst::ElementFactory::static_type()) - .is_some(); - let va_h265 = registry - .find_feature("vah265dec", gst::ElementFactory::static_type()) - .is_some(); - let va_av1 = registry - .find_feature("vaav1dec", gst::ElementFactory::static_type()) - .is_some(); - let vaapi_h264 = registry - .find_feature("vaapih264dec", gst::ElementFactory::static_type()) - .is_some(); - let vaapi_h265 = registry - .find_feature("vaapih265dec", gst::ElementFactory::static_type()) - .is_some(); - let avdec_h264 = registry - .find_feature("avdec_h264", gst::ElementFactory::static_type()) - .is_some(); - let avdec_h265 = registry - .find_feature("avdec_h265", gst::ElementFactory::static_type()) - .is_some(); - let av1dec = registry - .find_feature("av1dec", gst::ElementFactory::static_type()) - .is_some(); - - // Log available decoders - info!("GStreamer Linux decoders:"); - info!( - " V4L2 (Raspberry Pi): H.264={}, H.265={}, AV1={}", - v4l2_h264, v4l2_h265, v4l2_av1 - ); - info!( - " VA (Intel/AMD): H.264={}, H.265={}, AV1={}", - va_h264, va_h265, va_av1 - ); - info!( - " VAAPI (legacy): H.264={}, H.265={}", - vaapi_h264, vaapi_h265 - ); - info!( - " Software: H.264={}, H.265={}, AV1={}", - avdec_h264, avdec_h265, av1dec - ); - - // Return true if any decoder is available - v4l2_h264 - || v4l2_h265 - || v4l2_av1 - || va_h264 - || va_h265 - || va_av1 - || vaapi_h264 - || vaapi_h265 - || avdec_h264 - || avdec_h265 - || av1dec -} - -/// Check if running on Raspberry Pi -#[cfg(target_os = "linux")] -pub fn is_raspberry_pi() -> bool { - if let Ok(model) = std::fs::read_to_string("/proc/device-tree/model") { - model.contains("Raspberry Pi") - } else { - false - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_codec_caps() { - assert!(GstCodec::H264.caps_string().contains("h264")); - assert!(GstCodec::H265.caps_string().contains("h265")); - assert!(GstCodec::AV1.caps_string().contains("av1")); - } - - #[test] - fn test_default_config() { - let config = GstDecoderConfig::default(); - assert_eq!(config.width, 1920); - assert_eq!(config.height, 1080); - assert_eq!(config.codec, GstCodec::H264); - assert!(config.low_latency); - } - - #[test] - fn test_parser_elements() { - assert_eq!(GstCodec::H264.parser_element(), "h264parse"); - assert_eq!(GstCodec::H265.parser_element(), "h265parse"); - assert_eq!(GstCodec::AV1.parser_element(), "av1parse"); - } - - #[test] - fn test_software_decoders() { - assert_eq!(GstCodec::H264.software_decoder(), "avdec_h264"); - assert_eq!(GstCodec::H265.software_decoder(), "avdec_h265"); - assert_eq!(GstCodec::AV1.software_decoder(), "av1dec"); - } -} diff --git a/opennow-streamer/src/media/hevc_parser.rs b/opennow-streamer/src/media/hevc_parser.rs deleted file mode 100644 index 58aa616..0000000 --- a/opennow-streamer/src/media/hevc_parser.rs +++ /dev/null @@ -1,1014 +0,0 @@ -//! HEVC/H.265 NAL Unit Parser for DXVA2 -//! -//! This module parses HEVC bitstreams and extracts the necessary parameter sets -//! (VPS, SPS, PPS) and slice headers needed for DXVA2 hardware decoding. -//! -//! Based on ITU-T H.265 specification. - -use anyhow::{anyhow, Result}; -use log::{debug, trace, warn}; - -/// HEVC NAL unit types (ITU-T H.265 Table 7-1) -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[repr(u8)] -pub enum HevcNalType { - TrailN = 0, - TrailR = 1, - TsaN = 2, - TsaR = 3, - StsaN = 4, - StsaR = 5, - RadlN = 6, - RadlR = 7, - RaslN = 8, - RaslR = 9, - // Reserved 10-15 - BlaWLp = 16, - BlaWRadl = 17, - BlaNLp = 18, - IdrWRadl = 19, - IdrNLp = 20, - CraNut = 21, - // Reserved 22-31 - VpsNut = 32, - SpsNut = 33, - PpsNut = 34, - AudNut = 35, - EosNut = 36, - EobNut = 37, - FdNut = 38, - PrefixSeiNut = 39, - SuffixSeiNut = 40, - // Reserved 41-47 - Unknown = 255, -} - -impl From for HevcNalType { - fn from(val: u8) -> Self { - match val { - 0 => Self::TrailN, - 1 => Self::TrailR, - 2 => Self::TsaN, - 3 => Self::TsaR, - 4 => Self::StsaN, - 5 => Self::StsaR, - 6 => Self::RadlN, - 7 => Self::RadlR, - 8 => Self::RaslN, - 9 => Self::RaslR, - 16 => Self::BlaWLp, - 17 => Self::BlaWRadl, - 18 => Self::BlaNLp, - 19 => Self::IdrWRadl, - 20 => Self::IdrNLp, - 21 => Self::CraNut, - 32 => Self::VpsNut, - 33 => Self::SpsNut, - 34 => Self::PpsNut, - 35 => Self::AudNut, - 36 => Self::EosNut, - 37 => Self::EobNut, - 38 => Self::FdNut, - 39 => Self::PrefixSeiNut, - 40 => Self::SuffixSeiNut, - _ => Self::Unknown, - } - } -} - -impl HevcNalType { - /// Check if this NAL type is a VCL (Video Coding Layer) NAL unit - pub fn is_vcl(&self) -> bool { - (*self as u8) <= 31 - } - - /// Check if this NAL type is an IDR frame - pub fn is_idr(&self) -> bool { - matches!(self, Self::IdrWRadl | Self::IdrNLp) - } - - /// Check if this NAL type is a BLA (Broken Link Access) frame - pub fn is_bla(&self) -> bool { - matches!(self, Self::BlaWLp | Self::BlaWRadl | Self::BlaNLp) - } - - /// Check if this NAL type is a CRA (Clean Random Access) frame - pub fn is_cra(&self) -> bool { - matches!(self, Self::CraNut) - } - - /// Check if this is a random access point (IDR, BLA, or CRA) - pub fn is_rap(&self) -> bool { - self.is_idr() || self.is_bla() || self.is_cra() - } - - /// Check if this NAL contains a slice - pub fn is_slice(&self) -> bool { - self.is_vcl() - } -} - -/// Parsed HEVC NAL unit -#[derive(Debug, Clone)] -pub struct HevcNalUnit { - /// NAL unit type - pub nal_type: HevcNalType, - /// NAL unit header layer ID (nuh_layer_id) - pub layer_id: u8, - /// NAL unit header temporal ID plus 1 (nuh_temporal_id_plus1) - pub temporal_id: u8, - /// Raw NAL unit data (without start code, with header) - pub data: Vec, - /// Offset in original stream - pub offset: usize, -} - -/// HEVC Video Parameter Set (VPS) -#[derive(Debug, Clone, Default)] -pub struct HevcVps { - pub vps_id: u8, - pub max_layers: u8, - pub max_sub_layers: u8, - pub temporal_id_nesting: bool, - // Raw VPS data for DXVA - pub raw_data: Vec, -} - -/// HEVC Sequence Parameter Set (SPS) -#[derive(Debug, Clone, Default)] -pub struct HevcSps { - pub sps_id: u8, - pub vps_id: u8, - pub max_sub_layers: u8, - pub chroma_format_idc: u8, - pub separate_colour_plane: bool, - pub pic_width: u32, - pub pic_height: u32, - pub bit_depth_luma: u8, - pub bit_depth_chroma: u8, - pub log2_max_poc_lsb: u8, - pub num_short_term_ref_pic_sets: u8, - pub long_term_ref_pics_present: bool, - pub num_long_term_ref_pics_sps: u8, - pub temporal_mvp_enabled: bool, - pub strong_intra_smoothing_enabled: bool, - pub scaling_list_enabled: bool, - pub amp_enabled: bool, - pub sample_adaptive_offset_enabled: bool, - pub pcm_enabled: bool, - pub pcm_sample_bit_depth_luma: u8, - pub pcm_sample_bit_depth_chroma: u8, - pub log2_min_pcm_luma_coding_block_size: u8, - pub log2_diff_max_min_pcm_luma_coding_block_size: u8, - pub pcm_loop_filter_disabled: bool, - pub log2_min_luma_coding_block_size: u8, - pub log2_diff_max_min_luma_coding_block_size: u8, - pub log2_min_luma_transform_block_size: u8, - pub log2_diff_max_min_luma_transform_block_size: u8, - pub max_transform_hierarchy_depth_inter: u8, - pub max_transform_hierarchy_depth_intra: u8, - // Raw SPS data for DXVA - pub raw_data: Vec, -} - -/// HEVC Picture Parameter Set (PPS) -#[derive(Debug, Clone, Default)] -pub struct HevcPps { - pub pps_id: u8, - pub sps_id: u8, - pub dependent_slice_segments_enabled: bool, - pub output_flag_present: bool, - pub num_extra_slice_header_bits: u8, - pub sign_data_hiding_enabled: bool, - pub cabac_init_present: bool, - pub num_ref_idx_l0_default_active: u8, - pub num_ref_idx_l1_default_active: u8, - pub init_qp: i8, - pub constrained_intra_pred: bool, - pub transform_skip_enabled: bool, - pub cu_qp_delta_enabled: bool, - pub diff_cu_qp_delta_depth: u8, - pub cb_qp_offset: i8, - pub cr_qp_offset: i8, - pub slice_chroma_qp_offsets_present: bool, - pub weighted_pred: bool, - pub weighted_bipred: bool, - pub transquant_bypass_enabled: bool, - pub tiles_enabled: bool, - pub entropy_coding_sync_enabled: bool, - pub num_tile_columns: u16, - pub num_tile_rows: u16, - pub uniform_spacing: bool, - pub loop_filter_across_tiles_enabled: bool, - pub loop_filter_across_slices_enabled: bool, - pub deblocking_filter_control_present: bool, - pub deblocking_filter_override_enabled: bool, - pub deblocking_filter_disabled: bool, - pub beta_offset: i8, - pub tc_offset: i8, - pub lists_modification_present: bool, - pub log2_parallel_merge_level: u8, - pub slice_segment_header_extension_present: bool, - // Raw PPS data for DXVA - pub raw_data: Vec, -} - -/// HEVC Slice Header (partial parse for DXVA) -#[derive(Debug, Clone, Default)] -pub struct HevcSliceHeader { - pub first_slice_in_pic: bool, - pub no_output_of_prior_pics: bool, - pub pps_id: u8, - pub dependent_slice_segment: bool, - pub slice_segment_address: u32, - pub slice_type: u8, // 0=B, 1=P, 2=I - pub pic_output: bool, - pub colour_plane_id: u8, - pub pic_order_cnt_lsb: u16, - pub short_term_ref_pic_set_sps_flag: bool, - pub short_term_ref_pic_set_idx: u8, - // Reference picture list modification - pub num_ref_idx_l0_active: u8, - pub num_ref_idx_l1_active: u8, -} - -/// Exponential-Golomb bit reader -pub struct BitReader<'a> { - data: &'a [u8], - byte_pos: usize, - bit_pos: u8, -} - -impl<'a> BitReader<'a> { - pub fn new(data: &'a [u8]) -> Self { - Self { - data, - byte_pos: 0, - bit_pos: 0, - } - } - - /// Read a single bit - pub fn read_bit(&mut self) -> Result { - if self.byte_pos >= self.data.len() { - return Err(anyhow!("BitReader: end of data")); - } - let bit = (self.data[self.byte_pos] >> (7 - self.bit_pos)) & 1; - self.bit_pos += 1; - if self.bit_pos >= 8 { - self.bit_pos = 0; - self.byte_pos += 1; - } - Ok(bit) - } - - /// Read N bits as unsigned - pub fn read_bits(&mut self, n: u8) -> Result { - let mut val = 0u32; - for _ in 0..n { - val = (val << 1) | self.read_bit()? as u32; - } - Ok(val) - } - - /// Read unsigned Exp-Golomb coded value - pub fn read_ue(&mut self) -> Result { - let mut leading_zeros = 0u32; - while self.read_bit()? == 0 { - leading_zeros += 1; - if leading_zeros > 31 { - return Err(anyhow!("BitReader: invalid Exp-Golomb code")); - } - } - if leading_zeros == 0 { - return Ok(0); - } - let val = self.read_bits(leading_zeros as u8)?; - Ok((1 << leading_zeros) - 1 + val) - } - - /// Read signed Exp-Golomb coded value - pub fn read_se(&mut self) -> Result { - let ue = self.read_ue()?; - let sign = if ue & 1 == 1 { 1 } else { -1 }; - Ok(sign * ((ue + 1) / 2) as i32) - } - - /// Skip N bits - pub fn skip_bits(&mut self, n: u32) -> Result<()> { - for _ in 0..n { - self.read_bit()?; - } - Ok(()) - } - - /// Get current bit position - pub fn position(&self) -> usize { - self.byte_pos * 8 + self.bit_pos as usize - } - - /// Check if more data is available - pub fn has_more_data(&self) -> bool { - self.byte_pos < self.data.len() - } -} - -/// HEVC Bitstream Parser -pub struct HevcParser { - /// Current VPS (up to 16) - pub vps: [Option; 16], - /// Current SPS (up to 16) - pub sps: [Option; 16], - /// Current PPS (up to 64) - pub pps: Vec>, -} - -impl Default for HevcParser { - fn default() -> Self { - Self::new() - } -} - -impl HevcParser { - pub fn new() -> Self { - Self { - vps: Default::default(), - sps: Default::default(), - pps: vec![None; 64], - } - } - - /// Find all NAL units in a bitstream (Annex B format with start codes) - pub fn find_nal_units(&self, data: &[u8]) -> Vec { - let mut nals = Vec::new(); - let mut i = 0; - - while i < data.len() { - // Look for start code (0x000001 or 0x00000001) - if i + 3 <= data.len() && data[i] == 0 && data[i + 1] == 0 { - let start_code_len; - let nal_start; - - if data[i + 2] == 1 { - // 3-byte start code - start_code_len = 3; - nal_start = i + 3; - } else if i + 4 <= data.len() && data[i + 2] == 0 && data[i + 3] == 1 { - // 4-byte start code - start_code_len = 4; - nal_start = i + 4; - } else { - i += 1; - continue; - } - - // Find the end of this NAL unit (next start code or end of data) - let mut nal_end = data.len(); - for j in nal_start..data.len() - 2 { - if data[j] == 0 - && data[j + 1] == 0 - && (data[j + 2] == 1 - || (j + 3 < data.len() && data[j + 2] == 0 && data[j + 3] == 1)) - { - nal_end = j; - break; - } - } - - if nal_start < nal_end && nal_end - nal_start >= 2 { - // Parse NAL header (2 bytes for HEVC) - let header0 = data[nal_start]; - let header1 = data[nal_start + 1]; - - // forbidden_zero_bit (1) | nal_unit_type (6) | nuh_layer_id (6) | nuh_temporal_id_plus1 (3) - let nal_type = HevcNalType::from((header0 >> 1) & 0x3F); - let layer_id = ((header0 & 1) << 5) | (header1 >> 3); - let temporal_id = header1 & 0x07; - - nals.push(HevcNalUnit { - nal_type, - layer_id, - temporal_id, - data: data[nal_start..nal_end].to_vec(), - offset: i, - }); - - trace!( - "Found NAL: type={:?}, layer={}, temporal={}, size={}", - nal_type, - layer_id, - temporal_id, - nal_end - nal_start - ); - } - - i = nal_end; - } else { - i += 1; - } - } - - nals - } - - /// Remove emulation prevention bytes (0x000003 -> 0x0000) - fn remove_emulation_prevention(data: &[u8]) -> Vec { - let mut result = Vec::with_capacity(data.len()); - let mut i = 0; - - while i < data.len() { - if i + 2 < data.len() && data[i] == 0 && data[i + 1] == 0 && data[i + 2] == 3 { - result.push(0); - result.push(0); - i += 3; // Skip the 0x03 byte - } else { - result.push(data[i]); - i += 1; - } - } - - result - } - - /// Parse VPS NAL unit - pub fn parse_vps(&mut self, nal: &HevcNalUnit) -> Result { - let rbsp = Self::remove_emulation_prevention(&nal.data[2..]); // Skip NAL header - let mut reader = BitReader::new(&rbsp); - - let vps_id = reader.read_bits(4)? as u8; - reader.skip_bits(2)?; // vps_base_layer_internal_flag, vps_base_layer_available_flag - let max_layers = reader.read_bits(6)? as u8 + 1; - let max_sub_layers = reader.read_bits(3)? as u8 + 1; - let temporal_id_nesting = reader.read_bit()? != 0; - - let vps = HevcVps { - vps_id, - max_layers, - max_sub_layers, - temporal_id_nesting, - raw_data: nal.data.clone(), - }; - - debug!( - "Parsed VPS {}: max_layers={}, max_sub_layers={}", - vps_id, max_layers, max_sub_layers - ); - self.vps[vps_id as usize] = Some(vps); - Ok(vps_id) - } - - /// Parse SPS NAL unit - pub fn parse_sps(&mut self, nal: &HevcNalUnit) -> Result { - let rbsp = Self::remove_emulation_prevention(&nal.data[2..]); // Skip NAL header - let mut reader = BitReader::new(&rbsp); - - let vps_id = reader.read_bits(4)? as u8; - let max_sub_layers = reader.read_bits(3)? as u8 + 1; - let _temporal_id_nesting = reader.read_bit()?; - - // Skip profile_tier_level (complex structure) - self.skip_profile_tier_level(&mut reader, true, max_sub_layers)?; - - let sps_id = reader.read_ue()? as u8; - let chroma_format_idc = reader.read_ue()? as u8; - - let separate_colour_plane = if chroma_format_idc == 3 { - reader.read_bit()? != 0 - } else { - false - }; - - let pic_width = reader.read_ue()?; - let pic_height = reader.read_ue()?; - - // conformance_window_flag - let conformance_window = reader.read_bit()? != 0; - if conformance_window { - reader.read_ue()?; // left - reader.read_ue()?; // right - reader.read_ue()?; // top - reader.read_ue()?; // bottom - } - - let bit_depth_luma = reader.read_ue()? as u8 + 8; - let bit_depth_chroma = reader.read_ue()? as u8 + 8; - let log2_max_poc_lsb = reader.read_ue()? as u8 + 4; - - // sub_layer_ordering_info_present_flag - let sub_layer_ordering = reader.read_bit()? != 0; - let start = if sub_layer_ordering { - 0 - } else { - max_sub_layers - 1 - }; - for _ in start..max_sub_layers { - reader.read_ue()?; // max_dec_pic_buffering - reader.read_ue()?; // max_num_reorder_pics - reader.read_ue()?; // max_latency_increase - } - - let log2_min_luma_coding_block_size = reader.read_ue()? as u8 + 3; - let log2_diff_max_min_luma_coding_block_size = reader.read_ue()? as u8; - let log2_min_luma_transform_block_size = reader.read_ue()? as u8 + 2; - let log2_diff_max_min_luma_transform_block_size = reader.read_ue()? as u8; - let max_transform_hierarchy_depth_inter = reader.read_ue()? as u8; - let max_transform_hierarchy_depth_intra = reader.read_ue()? as u8; - - let scaling_list_enabled = reader.read_bit()? != 0; - if scaling_list_enabled { - let scaling_list_data_present = reader.read_bit()? != 0; - if scaling_list_data_present { - self.skip_scaling_list_data(&mut reader)?; - } - } - - let amp_enabled = reader.read_bit()? != 0; - let sample_adaptive_offset_enabled = reader.read_bit()? != 0; - - let pcm_enabled = reader.read_bit()? != 0; - let ( - pcm_sample_bit_depth_luma, - pcm_sample_bit_depth_chroma, - log2_min_pcm_luma_coding_block_size, - log2_diff_max_min_pcm_luma_coding_block_size, - pcm_loop_filter_disabled, - ) = if pcm_enabled { - let luma = reader.read_bits(4)? as u8 + 1; - let chroma = reader.read_bits(4)? as u8 + 1; - let min = reader.read_ue()? as u8 + 3; - let diff = reader.read_ue()? as u8; - let disabled = reader.read_bit()? != 0; - (luma, chroma, min, diff, disabled) - } else { - (0, 0, 0, 0, false) - }; - - let num_short_term_ref_pic_sets = reader.read_ue()? as u8; - // Skip short term ref pic sets (complex) - for i in 0..num_short_term_ref_pic_sets { - self.skip_short_term_ref_pic_set(&mut reader, i, num_short_term_ref_pic_sets)?; - } - - let long_term_ref_pics_present = reader.read_bit()? != 0; - let num_long_term_ref_pics_sps = if long_term_ref_pics_present { - let num = reader.read_ue()? as u8; - for _ in 0..num { - reader.skip_bits(log2_max_poc_lsb as u32)?; // lt_ref_pic_poc_lsb_sps - reader.read_bit()?; // used_by_curr_pic_lt_sps_flag - } - num - } else { - 0 - }; - - let temporal_mvp_enabled = reader.read_bit()? != 0; - let strong_intra_smoothing_enabled = reader.read_bit()? != 0; - - let sps = HevcSps { - sps_id, - vps_id, - max_sub_layers, - chroma_format_idc, - separate_colour_plane, - pic_width, - pic_height, - bit_depth_luma, - bit_depth_chroma, - log2_max_poc_lsb, - num_short_term_ref_pic_sets, - long_term_ref_pics_present, - num_long_term_ref_pics_sps, - temporal_mvp_enabled, - strong_intra_smoothing_enabled, - scaling_list_enabled, - amp_enabled, - sample_adaptive_offset_enabled, - pcm_enabled, - pcm_sample_bit_depth_luma, - pcm_sample_bit_depth_chroma, - log2_min_pcm_luma_coding_block_size, - log2_diff_max_min_pcm_luma_coding_block_size, - pcm_loop_filter_disabled, - log2_min_luma_coding_block_size, - log2_diff_max_min_luma_coding_block_size, - log2_min_luma_transform_block_size, - log2_diff_max_min_luma_transform_block_size, - max_transform_hierarchy_depth_inter, - max_transform_hierarchy_depth_intra, - raw_data: nal.data.clone(), - }; - - debug!( - "Parsed SPS {}: {}x{}, bit_depth={}/{}", - sps_id, pic_width, pic_height, bit_depth_luma, bit_depth_chroma - ); - self.sps[sps_id as usize] = Some(sps); - Ok(sps_id) - } - - /// Parse PPS NAL unit - pub fn parse_pps(&mut self, nal: &HevcNalUnit) -> Result { - let rbsp = Self::remove_emulation_prevention(&nal.data[2..]); // Skip NAL header - let mut reader = BitReader::new(&rbsp); - - let pps_id = reader.read_ue()? as u8; - let sps_id = reader.read_ue()? as u8; - - let dependent_slice_segments_enabled = reader.read_bit()? != 0; - let output_flag_present = reader.read_bit()? != 0; - let num_extra_slice_header_bits = reader.read_bits(3)? as u8; - let sign_data_hiding_enabled = reader.read_bit()? != 0; - let cabac_init_present = reader.read_bit()? != 0; - - let num_ref_idx_l0_default_active = reader.read_ue()? as u8 + 1; - let num_ref_idx_l1_default_active = reader.read_ue()? as u8 + 1; - let init_qp = reader.read_se()? as i8 + 26; - - let constrained_intra_pred = reader.read_bit()? != 0; - let transform_skip_enabled = reader.read_bit()? != 0; - - let cu_qp_delta_enabled = reader.read_bit()? != 0; - let diff_cu_qp_delta_depth = if cu_qp_delta_enabled { - reader.read_ue()? as u8 - } else { - 0 - }; - - let cb_qp_offset = reader.read_se()? as i8; - let cr_qp_offset = reader.read_se()? as i8; - - let slice_chroma_qp_offsets_present = reader.read_bit()? != 0; - let weighted_pred = reader.read_bit()? != 0; - let weighted_bipred = reader.read_bit()? != 0; - let transquant_bypass_enabled = reader.read_bit()? != 0; - let tiles_enabled = reader.read_bit()? != 0; - let entropy_coding_sync_enabled = reader.read_bit()? != 0; - - let (num_tile_columns, num_tile_rows, uniform_spacing, loop_filter_across_tiles_enabled) = - if tiles_enabled { - let cols = reader.read_ue()? as u16 + 1; - let rows = reader.read_ue()? as u16 + 1; - let uniform = reader.read_bit()? != 0; - if !uniform { - for _ in 0..cols - 1 { - reader.read_ue()?; - } - for _ in 0..rows - 1 { - reader.read_ue()?; - } - } - let across = reader.read_bit()? != 0; - (cols, rows, uniform, across) - } else { - (1, 1, true, false) - }; - - let loop_filter_across_slices_enabled = reader.read_bit()? != 0; - - let deblocking_filter_control_present = reader.read_bit()? != 0; - let ( - deblocking_filter_override_enabled, - deblocking_filter_disabled, - beta_offset, - tc_offset, - ) = if deblocking_filter_control_present { - let override_enabled = reader.read_bit()? != 0; - let disabled = reader.read_bit()? != 0; - let (beta, tc) = if !disabled { - (reader.read_se()? as i8 * 2, reader.read_se()? as i8 * 2) - } else { - (0, 0) - }; - (override_enabled, disabled, beta, tc) - } else { - (false, false, 0, 0) - }; - - // scaling_list_data_present_flag - if self.sps[sps_id as usize] - .as_ref() - .map_or(false, |s| s.scaling_list_enabled) - { - let scaling_list_data_present = reader.read_bit()? != 0; - if scaling_list_data_present { - self.skip_scaling_list_data(&mut reader)?; - } - } - - let lists_modification_present = reader.read_bit()? != 0; - let log2_parallel_merge_level = reader.read_ue()? as u8 + 2; - let slice_segment_header_extension_present = reader.read_bit()? != 0; - - let pps = HevcPps { - pps_id, - sps_id, - dependent_slice_segments_enabled, - output_flag_present, - num_extra_slice_header_bits, - sign_data_hiding_enabled, - cabac_init_present, - num_ref_idx_l0_default_active, - num_ref_idx_l1_default_active, - init_qp, - constrained_intra_pred, - transform_skip_enabled, - cu_qp_delta_enabled, - diff_cu_qp_delta_depth, - cb_qp_offset, - cr_qp_offset, - slice_chroma_qp_offsets_present, - weighted_pred, - weighted_bipred, - transquant_bypass_enabled, - tiles_enabled, - entropy_coding_sync_enabled, - num_tile_columns, - num_tile_rows, - uniform_spacing, - loop_filter_across_tiles_enabled, - loop_filter_across_slices_enabled, - deblocking_filter_control_present, - deblocking_filter_override_enabled, - deblocking_filter_disabled, - beta_offset, - tc_offset, - lists_modification_present, - log2_parallel_merge_level, - slice_segment_header_extension_present, - raw_data: nal.data.clone(), - }; - - debug!( - "Parsed PPS {}: sps={}, tiles={}x{}", - pps_id, sps_id, num_tile_columns, num_tile_rows - ); - self.pps[pps_id as usize] = Some(pps); - Ok(pps_id) - } - - /// Parse slice header (partial, for DXVA) - pub fn parse_slice_header(&self, nal: &HevcNalUnit) -> Result { - let rbsp = Self::remove_emulation_prevention(&nal.data[2..]); // Skip NAL header - let mut reader = BitReader::new(&rbsp); - - let first_slice_in_pic = reader.read_bit()? != 0; - - let no_output_of_prior_pics = if nal.nal_type.is_rap() { - reader.read_bit()? != 0 - } else { - false - }; - - let pps_id = reader.read_ue()? as u8; - let pps = self.pps[pps_id as usize] - .as_ref() - .ok_or_else(|| anyhow!("PPS {} not found", pps_id))?; - let sps = self.sps[pps.sps_id as usize] - .as_ref() - .ok_or_else(|| anyhow!("SPS {} not found", pps.sps_id))?; - - let mut header = HevcSliceHeader { - first_slice_in_pic, - no_output_of_prior_pics, - pps_id, - ..Default::default() - }; - - if !first_slice_in_pic { - if pps.dependent_slice_segments_enabled { - header.dependent_slice_segment = reader.read_bit()? != 0; - } - // Calculate slice_segment_address bits - let ctb_size = 1u32 - << (sps.log2_min_luma_coding_block_size - + sps.log2_diff_max_min_luma_coding_block_size); - let pic_width_in_ctb = (sps.pic_width + ctb_size - 1) / ctb_size; - let pic_height_in_ctb = (sps.pic_height + ctb_size - 1) / ctb_size; - let num_ctb = pic_width_in_ctb * pic_height_in_ctb; - let address_bits = (32 - num_ctb.leading_zeros()) as u8; - header.slice_segment_address = reader.read_bits(address_bits)?; - } - - if !header.dependent_slice_segment { - // Skip extra slice header bits - reader.skip_bits(pps.num_extra_slice_header_bits as u32)?; - - header.slice_type = reader.read_ue()? as u8; - - if pps.output_flag_present { - header.pic_output = reader.read_bit()? != 0; - } else { - header.pic_output = true; - } - - if sps.separate_colour_plane { - header.colour_plane_id = reader.read_bits(2)? as u8; - } - - if !nal.nal_type.is_idr() { - header.pic_order_cnt_lsb = reader.read_bits(sps.log2_max_poc_lsb as u8)? as u16; - header.short_term_ref_pic_set_sps_flag = reader.read_bit()? != 0; - // Additional ref pic set parsing would go here - } - - // Default ref idx counts from PPS - header.num_ref_idx_l0_active = pps.num_ref_idx_l0_default_active; - header.num_ref_idx_l1_active = pps.num_ref_idx_l1_default_active; - } - - Ok(header) - } - - /// Process a NAL unit (parse if parameter set, return slice info if slice) - pub fn process_nal(&mut self, nal: &HevcNalUnit) -> Result<()> { - match nal.nal_type { - HevcNalType::VpsNut => { - self.parse_vps(nal)?; - } - HevcNalType::SpsNut => { - self.parse_sps(nal)?; - } - HevcNalType::PpsNut => { - self.parse_pps(nal)?; - } - _ => {} - } - Ok(()) - } - - // Helper functions for skipping complex structures - - fn skip_profile_tier_level( - &self, - reader: &mut BitReader, - profile_present: bool, - max_sub_layers: u8, - ) -> Result<()> { - if profile_present { - // general_profile_space (2) + general_tier_flag (1) + general_profile_idc (5) = 8 bits - // general_profile_compatibility_flag[32] = 32 bits - // general_progressive_source_flag + general_interlaced_source_flag + - // general_non_packed_constraint_flag + general_frame_only_constraint_flag = 4 bits - // general_max_12bit_constraint_flag...general_reserved_zero_34bits = 44 bits - // Total constraint flags = 48 bits - // general_level_idc = 8 bits - // Total = 2 + 1 + 5 + 32 + 48 + 8 = 96 bits - reader.skip_bits(96)?; - } - - // sub_layer_profile_present_flag and sub_layer_level_present_flag - let mut sub_layer_profile_present = vec![false; 8]; - let mut sub_layer_level_present = vec![false; 8]; - - for i in 0..(max_sub_layers - 1) as usize { - sub_layer_profile_present[i] = reader.read_bit()? != 0; - sub_layer_level_present[i] = reader.read_bit()? != 0; - } - - // Reserved bits when max_sub_layers - 1 < 8 - if max_sub_layers > 1 && max_sub_layers < 8 { - for _ in (max_sub_layers - 1)..8 { - reader.skip_bits(2)?; // reserved_zero_2bits - } - } - - // Sub-layer profile/level info - for i in 0..(max_sub_layers - 1) as usize { - if sub_layer_profile_present[i] { - // sub_layer profile: 2 + 1 + 5 + 32 + 48 = 88 bits - reader.skip_bits(88)?; - } - if sub_layer_level_present[i] { - reader.skip_bits(8)?; // sub_layer_level_idc - } - } - - Ok(()) - } - - fn skip_scaling_list_data(&self, reader: &mut BitReader) -> Result<()> { - for size_id in 0..4 { - let num_matrix = if size_id == 3 { 2 } else { 6 }; - for matrix_id in 0..num_matrix { - let scaling_list_pred_mode = reader.read_bit()? != 0; - if !scaling_list_pred_mode { - reader.read_ue()?; // delta - } else { - let coef_num = std::cmp::min(64, 1 << (4 + (size_id << 1))); - if size_id > 1 { - reader.read_se()?; // dc coef - } - for _ in 0..coef_num { - reader.read_se()?; - } - } - let _ = matrix_id; // suppress warning - } - } - Ok(()) - } - - fn skip_short_term_ref_pic_set( - &self, - reader: &mut BitReader, - idx: u8, - num_sets: u8, - ) -> Result<()> { - let inter_ref_pic_set_prediction = if idx > 0 { - reader.read_bit()? != 0 - } else { - false - }; - - if inter_ref_pic_set_prediction { - if idx == num_sets { - reader.read_ue()?; // delta_idx - } - reader.read_bit()?; // delta_rps_sign - reader.read_ue()?; // abs_delta_rps - // Would need previous ref pic set info to know how many to skip - // For now, we'll rely on the parsing stopping before this gets complex - } else { - let num_negative = reader.read_ue()?; - let num_positive = reader.read_ue()?; - for _ in 0..num_negative { - reader.read_ue()?; // delta_poc - reader.read_bit()?; // used - } - for _ in 0..num_positive { - reader.read_ue()?; // delta_poc - reader.read_bit()?; // used - } - } - Ok(()) - } - - /// Get active SPS for a PPS - pub fn get_sps_for_pps(&self, pps_id: u8) -> Option<&HevcSps> { - let pps = self.pps[pps_id as usize].as_ref()?; - self.sps[pps.sps_id as usize].as_ref() - } - - /// Get active VPS for an SPS - pub fn get_vps_for_sps(&self, sps_id: u8) -> Option<&HevcVps> { - let sps = self.sps[sps_id as usize].as_ref()?; - self.vps[sps.vps_id as usize].as_ref() - } - - /// Get video dimensions and HDR status from the first available SPS - /// Returns (width, height, is_hdr) or None if no SPS is available - pub fn get_dimensions(&self) -> Option<(u32, u32, bool)> { - // Find the first valid SPS - for sps in self.sps.iter().flatten() { - let width = sps.pic_width; - let height = sps.pic_height; - // HDR is indicated by 10-bit depth (Main10 profile) - let is_hdr = sps.bit_depth_luma > 8 || sps.bit_depth_chroma > 8; - return Some((width, height, is_hdr)); - } - None - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_nal_type_parsing() { - assert_eq!(HevcNalType::from(32), HevcNalType::VpsNut); - assert_eq!(HevcNalType::from(33), HevcNalType::SpsNut); - assert_eq!(HevcNalType::from(34), HevcNalType::PpsNut); - assert_eq!(HevcNalType::from(19), HevcNalType::IdrWRadl); - assert!(HevcNalType::IdrWRadl.is_idr()); - assert!(HevcNalType::IdrWRadl.is_rap()); - } - - #[test] - fn test_bit_reader() { - let data = [0b10110100, 0b01100000]; - let mut reader = BitReader::new(&data); - - assert_eq!(reader.read_bit().unwrap(), 1); - assert_eq!(reader.read_bit().unwrap(), 0); - assert_eq!(reader.read_bits(4).unwrap(), 0b1101); - } - - #[test] - fn test_exp_golomb() { - // 1 -> 0 (single 1 bit = 0) - let data = [0b10000000]; - let mut reader = BitReader::new(&data); - assert_eq!(reader.read_ue().unwrap(), 0); - - // 010 -> 1 - let data = [0b01000000]; - let mut reader = BitReader::new(&data); - assert_eq!(reader.read_ue().unwrap(), 1); - - // 011 -> 2 - let data = [0b01100000]; - let mut reader = BitReader::new(&data); - assert_eq!(reader.read_ue().unwrap(), 2); - } -} diff --git a/opennow-streamer/src/media/mod.rs b/opennow-streamer/src/media/mod.rs deleted file mode 100644 index 2b76aad..0000000 --- a/opennow-streamer/src/media/mod.rs +++ /dev/null @@ -1,359 +0,0 @@ -//! Media Pipeline -//! -//! Video decoding, audio decoding, and rendering. - -use std::sync::atomic::{AtomicU64, Ordering}; - -mod audio; -mod rtp; -mod video; - -/// Global frame ID counter for unique frame identification -/// Used to avoid redundant GPU texture uploads -static FRAME_ID_COUNTER: AtomicU64 = AtomicU64::new(1); - -/// Generate a new unique frame ID -pub fn next_frame_id() -> u64 { - FRAME_ID_COUNTER.fetch_add(1, Ordering::Relaxed) -} - -#[cfg(target_os = "macos")] -pub mod videotoolbox; - -#[cfg(target_os = "windows")] -pub mod d3d11; - -#[cfg(target_os = "windows")] -pub mod dxva_decoder; - -#[cfg(target_os = "windows")] -pub mod hevc_parser; - -#[cfg(target_os = "windows")] -pub mod native_video; - -#[cfg(target_os = "linux")] -pub mod vaapi; - -#[cfg(target_os = "linux")] -pub mod v4l2; - -// GStreamer decoder available on Linux and Windows x64 -// Note: GStreamer ARM64 Windows binaries are not available -#[cfg(any(target_os = "linux", all(windows, target_arch = "x86_64")))] -pub mod gstreamer_decoder; - -pub use audio::*; -pub use rtp::{DepacketizerCodec, RtpDepacketizer}; -pub use video::{get_supported_decoder_backends, DecodeStats, UnifiedVideoDecoder, VideoDecoder}; - -#[cfg(target_os = "macos")] -pub use videotoolbox::{ - CVMetalTexture, CVPixelBufferWrapper, LockedPlanes, MetalVideoRenderer, ZeroCopyFrame, - ZeroCopyTextureManager, -}; - -#[cfg(target_os = "windows")] -pub use d3d11::{D3D11TextureWrapper, D3D11ZeroCopyManager, LockedPlanes as D3D11LockedPlanes}; - -#[cfg(target_os = "windows")] -pub use dxva_decoder::{DxvaCodec, DxvaDecoder, DxvaDecoderConfig}; - -#[cfg(target_os = "windows")] -pub use native_video::{NativeDecodeStats, NativeVideoDecoder}; - -#[cfg(target_os = "linux")] -pub use vaapi::{LockedPlanes as VaapiLockedPlanes, VAAPISurfaceWrapper, VaapiZeroCopyManager}; - -#[cfg(target_os = "linux")] -pub use v4l2::{ - get_pi_model, get_recommended_codec, is_raspberry_pi, is_v4l2_available, - LockedPlanes as V4L2LockedPlanes, V4L2BufferWrapper, V4L2Codec, V4L2PixelFormat, - V4L2ZeroCopyManager, -}; - -#[cfg(target_os = "linux")] -pub use gstreamer_decoder::{ - init_gstreamer, is_gstreamer_v4l2_available, GStreamerDecoder, GstCodec, GstDecoderConfig, -}; - -// GStreamer only available on Windows x64 (no ARM64 binaries) -#[cfg(all(windows, target_arch = "x86_64"))] -pub use gstreamer_decoder::{ - init_gstreamer, is_gstreamer_available, GStreamerDecoder, GstCodec, GstDecoderConfig, -}; - -/// Pixel format of decoded video frame -#[derive(Debug, Clone, Copy, PartialEq, Default)] -pub enum PixelFormat { - /// YUV 4:2:0 planar (Y, U, V separate planes) - #[default] - YUV420P, - /// NV12 semi-planar (Y plane + interleaved UV plane) - /// More efficient on macOS VideoToolbox - skip CPU conversion - NV12, - /// P010 10-bit HDR (Y plane + interleaved UV, 10 bits per sample in 16-bit words) - P010, -} - -/// Decoded video frame -#[derive(Debug, Clone)] -pub struct VideoFrame { - /// Unique frame ID for tracking (monotonically increasing) - /// Used to avoid redundant GPU uploads of the same frame - pub frame_id: u64, - pub width: u32, - pub height: u32, - /// Y plane (luma) - full resolution - pub y_plane: Vec, - /// U plane (Cb chroma) - for YUV420P: half resolution - /// For NV12: this contains interleaved UV data - pub u_plane: Vec, - /// V plane (Cr chroma) - for YUV420P: half resolution - /// For NV12: this is empty (UV is interleaved in u_plane) - pub v_plane: Vec, - pub y_stride: u32, - pub u_stride: u32, - pub v_stride: u32, - pub timestamp_us: u64, - /// Pixel format (YUV420P or NV12) - pub format: PixelFormat, - /// Color range (Limited or Full) - pub color_range: ColorRange, - /// Color space (matrix coefficients) - pub color_space: ColorSpace, - /// Transfer function (SDR gamma vs HDR PQ/HLG) - pub transfer_function: TransferFunction, - /// Zero-copy GPU buffer (macOS VideoToolbox only) - /// When present, y_plane/u_plane are empty and rendering uses this directly - #[cfg(target_os = "macos")] - pub gpu_frame: Option>, - /// Zero-copy GPU texture (Windows D3D11VA only) - /// When present, y_plane/u_plane are empty and rendering imports this directly - #[cfg(target_os = "windows")] - pub gpu_frame: Option>, - /// Zero-copy GPU surface (Linux VAAPI only) - /// When present, y_plane/u_plane are empty and rendering imports this directly - #[cfg(target_os = "linux")] - pub gpu_frame: Option>, -} - -/// Video color range -#[derive(Debug, Clone, Copy, PartialEq, Default)] -pub enum ColorRange { - /// Limited range (16-235 for Y, 16-240 for UV) - Standard for TV/Video - #[default] - Limited, - /// Full range (0-255) - Standard for PC/JPEG - Full, -} - -/// Video color space (matrix coefficients) -#[derive(Debug, Clone, Copy, PartialEq, Default)] -pub enum ColorSpace { - /// BT.709 (HDTV) - Default - #[default] - BT709, - /// BT.601 (SDTV) - BT601, - /// BT.2020 (UHDTV) - BT2020, -} - -/// Video transfer function (EOTF - Electro-Optical Transfer Function) -#[derive(Debug, Clone, Copy, PartialEq, Default)] -pub enum TransferFunction { - /// SDR gamma (~2.4) - BT.709/BT.601/sRGB - #[default] - SDR, - /// HDR PQ (Perceptual Quantizer) - SMPTE ST 2084 / HDR10 - PQ, - /// HDR HLG (Hybrid Log-Gamma) - ARIB STD-B67 - HLG, -} - -impl VideoFrame { - /// Create empty frame (YUV420P format) - pub fn empty(width: u32, height: u32) -> Self { - let y_size = (width * height) as usize; - let uv_size = y_size / 4; - - Self { - frame_id: next_frame_id(), - width, - height, - y_plane: vec![0; y_size], - u_plane: vec![128; uv_size], - v_plane: vec![128; uv_size], - y_stride: width, - u_stride: width / 2, - v_stride: width / 2, - timestamp_us: 0, - format: PixelFormat::YUV420P, - color_range: ColorRange::Limited, - color_space: ColorSpace::BT709, - transfer_function: TransferFunction::SDR, - #[cfg(target_os = "macos")] - gpu_frame: None, - #[cfg(target_os = "windows")] - gpu_frame: None, - #[cfg(target_os = "linux")] - gpu_frame: None, - } - } - - /// Convert YUV to RGB (for CPU rendering fallback) - pub fn to_rgb(&self) -> Vec { - let mut rgb = Vec::with_capacity((self.width * self.height * 3) as usize); - - for row in 0..self.height { - for col in 0..self.width { - let yi = (row * self.y_stride + col) as usize; - let ui = ((row / 2) * self.u_stride + col / 2) as usize; - let vi = ((row / 2) * self.v_stride + col / 2) as usize; - - let y = self.y_plane.get(yi).copied().unwrap_or(0) as f32; - let u = self.u_plane.get(ui).copied().unwrap_or(128) as f32 - 128.0; - let v = self.v_plane.get(vi).copied().unwrap_or(128) as f32 - 128.0; - - // BT.601 YUV to RGB - let r = (y + 1.402 * v).clamp(0.0, 255.0) as u8; - let g = (y - 0.344 * u - 0.714 * v).clamp(0.0, 255.0) as u8; - let b = (y + 1.772 * u).clamp(0.0, 255.0) as u8; - - rgb.push(r); - rgb.push(g); - rgb.push(b); - } - } - - rgb - } - - /// Convert YUV to RGBA - optimized with integer math - pub fn to_rgba(&self) -> Vec { - let pixel_count = (self.width * self.height) as usize; - let mut rgba = vec![0u8; pixel_count * 4]; - - // Pre-calculate constants for BT.601 YUV->RGB (scaled by 256 for integer math) - // R = Y + 1.402*V -> Y + (359*V)/256 - // G = Y - 0.344*U - 0.714*V -> Y - (88*U + 183*V)/256 - // B = Y + 1.772*U -> Y + (454*U)/256 - - let width = self.width as usize; - let height = self.height as usize; - let y_stride = self.y_stride as usize; - let u_stride = self.u_stride as usize; - let _v_stride = self.v_stride as usize; - - for row in 0..height { - let y_row_offset = row * y_stride; - let uv_row_offset = (row / 2) * u_stride; - let rgba_row_offset = row * width * 4; - - for col in 0..width { - let yi = y_row_offset + col; - let uvi = uv_row_offset + col / 2; - let rgba_i = rgba_row_offset + col * 4; - - // Safe bounds check with defaults - let y = *self.y_plane.get(yi).unwrap_or(&0) as i32; - let u = *self.u_plane.get(uvi).unwrap_or(&128) as i32 - 128; - let v = *self.v_plane.get(uvi).unwrap_or(&128) as i32 - 128; - - // Integer math conversion (faster than float) - let r = (y + ((359 * v) >> 8)).clamp(0, 255) as u8; - let g = (y - ((88 * u + 183 * v) >> 8)).clamp(0, 255) as u8; - let b = (y + ((454 * u) >> 8)).clamp(0, 255) as u8; - - rgba[rgba_i] = r; - rgba[rgba_i + 1] = g; - rgba[rgba_i + 2] = b; - rgba[rgba_i + 3] = 255; - } - } - - rgba - } -} - -/// Stream statistics -#[derive(Debug, Clone, Default)] -pub struct StreamStats { - /// Video resolution - pub resolution: String, - /// Current decoded FPS (frames decoded per second) - pub fps: f32, - /// Render FPS (frames actually rendered to screen per second) - pub render_fps: f32, - /// Target FPS - pub target_fps: u32, - /// Video bitrate in Mbps - pub bitrate_mbps: f32, - /// Network latency in ms - pub latency_ms: f32, - /// Frame decode time in ms - pub decode_time_ms: f32, - /// Frame render time in ms - pub render_time_ms: f32, - /// Input latency in ms (time from event creation to transmission) - pub input_latency_ms: f32, - /// Video codec name - pub codec: String, - /// GPU type - pub gpu_type: String, - /// Server region - pub server_region: String, - /// Packet loss percentage - pub packet_loss: f32, - /// Network jitter in ms - pub jitter_ms: f32, - /// Network RTT (round-trip time) in ms from ICE candidate pair - pub rtt_ms: f32, - /// Total frames received - pub frames_received: u64, - /// Total frames decoded - pub frames_decoded: u64, - /// Total frames dropped - pub frames_dropped: u64, - /// Total frames rendered - pub frames_rendered: u64, - /// Input events sent per second - pub input_rate: f32, - /// Frame delivery latency (RTP arrival to decode complete) in ms - pub frame_delivery_ms: f32, - /// Estimated end-to-end latency in ms (decode_time + estimated network) - pub estimated_e2e_ms: f32, - /// Audio buffer level in ms - pub audio_buffer_ms: f32, - /// HDR mode (true = HDR/PQ, false = SDR) - pub is_hdr: bool, - /// Color space (e.g., "BT.709", "BT.2020") - pub color_space: String, - /// Number of racing wheels detected (0 = none) - pub wheel_count: usize, -} - -impl StreamStats { - pub fn new() -> Self { - Self::default() - } - - /// Format resolution string - pub fn format_resolution(&self) -> String { - if self.resolution.is_empty() { - "N/A".to_string() - } else { - self.resolution.clone() - } - } - - /// Format bitrate string - pub fn format_bitrate(&self) -> String { - if self.bitrate_mbps > 0.0 { - format!("{:.1} Mbps", self.bitrate_mbps) - } else { - "N/A".to_string() - } - } -} diff --git a/opennow-streamer/src/media/native_video.rs b/opennow-streamer/src/media/native_video.rs deleted file mode 100644 index 58f8956..0000000 --- a/opennow-streamer/src/media/native_video.rs +++ /dev/null @@ -1,350 +0,0 @@ -//! Native Video Decoder Wrapper -//! -//! This module provides a VideoDecoder-compatible interface for the native -//! D3D11 Video decoder (DXVA2), which bypasses FFmpeg entirely. -//! -//! Benefits: -//! - No MAX_SLICES=32 limitation (FFmpeg hardcoded limit) -//! - Native texture array (RTArray) support like NVIDIA's client -//! - Better compatibility with NVIDIA drivers -//! - Zero-copy output to D3D11 textures - -use anyhow::{anyhow, Result}; -use log::{info, warn}; -use std::sync::mpsc; -use std::sync::Arc; -use std::thread; -use tokio::sync::mpsc as tokio_mpsc; - -use super::dxva_decoder::{DxvaCodec, DxvaDecoder, DxvaDecoderConfig}; -use super::hevc_parser::HevcParser; -use super::{ColorRange, ColorSpace, PixelFormat, TransferFunction, VideoFrame}; -use crate::app::{SharedFrame, VideoCodec}; - -/// Stats from the native decoder thread -#[derive(Debug, Clone)] -pub struct NativeDecodeStats { - /// Time from packet receive to decode complete (ms) - pub decode_time_ms: f32, - /// Whether a frame was produced - pub frame_produced: bool, - /// Whether a keyframe is needed - pub needs_keyframe: bool, -} - -/// Commands sent to the native decoder thread -enum NativeDecoderCommand { - /// Decode a packet (async mode) - DecodeAsync { - data: Vec, - receive_time: std::time::Instant, - }, - /// Update decoder configuration (resolution change) - Configure { - width: u32, - height: u32, - is_hdr: bool, - }, - /// Stop the decoder - Stop, -} - -/// Native D3D11 Video Decoder wrapper -/// -/// Provides the same interface as VideoDecoder but uses native DXVA2 -/// instead of FFmpeg, avoiding the MAX_SLICES limitation. -pub struct NativeVideoDecoder { - cmd_tx: mpsc::Sender, - frames_decoded: u64, - shared_frame: Option>, -} - -impl NativeVideoDecoder { - /// Create a new native video decoder for async mode - /// - /// Note: Only HEVC (H.265) is supported. H.264 streams should use - /// FFmpeg-based decoders (D3D11VA, DXVA2) instead. - pub fn new_async( - codec: VideoCodec, - shared_frame: Arc, - ) -> Result<(Self, tokio_mpsc::Receiver)> { - // Only HEVC is supported by the native decoder - if codec != VideoCodec::H265 { - return Err(anyhow!( - "Native DXVA decoder only supports HEVC. Use D3D11VA or DXVA2 for H.264." - )); - } - - info!("Creating native DXVA HEVC decoder"); - let dxva_codec = DxvaCodec::HEVC; - - // Create channels for communication - let (cmd_tx, cmd_rx) = mpsc::channel::(); - let (stats_tx, stats_rx) = tokio_mpsc::channel::(64); - - // Spawn decoder thread - let shared_frame_clone = shared_frame.clone(); - Self::spawn_decoder_thread(dxva_codec, cmd_rx, shared_frame_clone, stats_tx)?; - - let decoder = Self { - cmd_tx, - frames_decoded: 0, - shared_frame: Some(shared_frame), - }; - - Ok((decoder, stats_rx)) - } - - /// Spawn the native decoder thread - fn spawn_decoder_thread( - _codec: DxvaCodec, - cmd_rx: mpsc::Receiver, - shared_frame: Arc, - stats_tx: tokio_mpsc::Sender, - ) -> Result<()> { - thread::spawn(move || { - // HEVC NAL unit parser - let mut hevc_parser = HevcParser::new(); - - // Decoder will be initialized on first frame when we know dimensions - let mut decoder: Option = None; - let mut current_width = 0u32; - let mut current_height = 0u32; - let mut is_hdr = false; - - let mut frames_decoded = 0u64; - let mut consecutive_failures = 0u32; - const KEYFRAME_REQUEST_THRESHOLD: u32 = 3; // Lowered from 10 for faster recovery after focus loss - - while let Ok(cmd) = cmd_rx.recv() { - match cmd { - NativeDecoderCommand::DecodeAsync { data, receive_time } => { - // Parse HEVC NAL units to extract SPS for dimensions - let nals = hevc_parser.find_nal_units(&data); - for nal in &nals { - let _ = hevc_parser.process_nal(nal); - } - let (width, height, hdr) = - hevc_parser.get_dimensions().unwrap_or((0, 0, false)); - - // Initialize or reconfigure decoder if dimensions changed - if width > 0 && height > 0 { - if decoder.is_none() - || width != current_width - || height != current_height - || hdr != is_hdr - { - let config = DxvaDecoderConfig { - codec: DxvaCodec::HEVC, - width, - height, - is_hdr: hdr, - surface_count: 25, // Increased for high bitrate streams - low_latency: true, // Enable low latency for streaming - }; - - match DxvaDecoder::new(config) { - Ok(dec) => { - info!( - "Native DXVA HEVC decoder initialized: {}x{} HDR={}", - width, height, hdr - ); - decoder = Some(dec); - current_width = width; - current_height = height; - is_hdr = hdr; - } - Err(e) => { - warn!("Failed to create DXVA decoder: {:?}", e); - let _ = stats_tx.try_send(NativeDecodeStats { - decode_time_ms: receive_time.elapsed().as_secs_f32() - * 1000.0, - frame_produced: false, - needs_keyframe: true, - }); - continue; - } - } - } - } - - // Decode frame if decoder is ready - let mut frame_produced = false; - let mut needs_keyframe = false; - - if let Some(ref mut dec) = decoder { - // Decode HEVC frame - match dec.decode_frame(&data, &mut hevc_parser) { - Ok(decoded) => { - frames_decoded += 1; - frame_produced = true; - consecutive_failures = 0; - - // Convert to VideoFrame and write to SharedFrame - // Zero-copy: GPU texture passed directly to renderer - let video_frame = Self::convert_decoded_frame(&decoded, is_hdr); - if let Some(frame) = video_frame { - shared_frame.write(frame); - } - } - Err(e) => { - consecutive_failures += 1; - // Log first few failures and then periodically - if consecutive_failures <= 5 || consecutive_failures % 100 == 0 - { - warn!( - "Native HEVC decode failed (failure #{}): {:?}", - consecutive_failures, e - ); - } - if consecutive_failures >= KEYFRAME_REQUEST_THRESHOLD { - needs_keyframe = true; - } - } - } - } else { - // Decoder not ready yet (waiting for SPS) - consecutive_failures += 1; - } - - // Send stats - let _ = stats_tx.try_send(NativeDecodeStats { - decode_time_ms: receive_time.elapsed().as_secs_f32() * 1000.0, - frame_produced, - needs_keyframe, - }); - } - - NativeDecoderCommand::Configure { - width, - height, - is_hdr: hdr, - } => { - let config = DxvaDecoderConfig { - codec: DxvaCodec::HEVC, - width, - height, - is_hdr: hdr, - surface_count: 25, - low_latency: true, // Enable low latency for streaming - }; - - if let Ok(dec) = DxvaDecoder::new(config) { - decoder = Some(dec); - current_width = width; - current_height = height; - is_hdr = hdr; - } - } - - NativeDecoderCommand::Stop => { - break; - } - } - } - }); - - Ok(()) - } - - /// Convert decoded DXVA frame to VideoFrame - /// - /// CRITICAL: We must copy the frame data IMMEDIATELY after decoding, before - /// the decoder moves on to the next frame. The DXVA decoder uses a texture - /// array with surface recycling - if we don't copy now, the surface may be - /// reused for the next decode before the renderer can read it, causing - /// frame repetition or corruption. - fn convert_decoded_frame( - decoded: &super::dxva_decoder::DxvaDecodedFrame, - is_hdr: bool, - ) -> Option { - use super::d3d11::D3D11TextureWrapper; - - info!( - "Converting decoded frame: {}x{}, array_index={}, poc={}", - decoded.width, decoded.height, decoded.array_index, decoded.poc - ); - - // Create wrapper to access the texture - let gpu_texture = - D3D11TextureWrapper::from_texture(decoded.texture.clone(), decoded.array_index); - - // CRITICAL: Copy frame data NOW before decoder reuses the surface - // This prevents frame repetition caused by surface recycling - match gpu_texture.lock_and_get_planes() { - Ok(planes) => { - info!( - "Frame copied: poc={}, y_size={}, uv_size={}, stride={}", - decoded.poc, - planes.y_plane.len(), - planes.uv_plane.len(), - planes.y_stride - ); - - // Return VideoFrame with CPU plane data - // The renderer will upload this to GPU textures - Some(VideoFrame { - frame_id: super::next_frame_id(), - width: decoded.width, - height: decoded.height, - // NV12 format: Y plane + interleaved UV plane - y_plane: planes.y_plane, - u_plane: planes.uv_plane, // UV interleaved in NV12 - v_plane: Vec::new(), // Empty for NV12 (UV is interleaved) - y_stride: planes.y_stride, - u_stride: planes.uv_stride, - v_stride: 0, - timestamp_us: 0, - format: if is_hdr { - PixelFormat::P010 - } else { - PixelFormat::NV12 - }, - color_range: ColorRange::Limited, - color_space: if is_hdr { - ColorSpace::BT2020 - } else { - ColorSpace::BT709 - }, - transfer_function: if is_hdr { - TransferFunction::PQ - } else { - TransferFunction::SDR - }, - // No GPU frame - we've copied to CPU planes - gpu_frame: None, - }) - } - Err(e) => { - warn!( - "Failed to copy decoded frame (poc={}): {:?}", - decoded.poc, e - ); - None - } - } - } - - /// Send a packet for async decoding - pub fn decode_async(&self, data: Vec, receive_time: std::time::Instant) { - let _ = self - .cmd_tx - .send(NativeDecoderCommand::DecodeAsync { data, receive_time }); - } - - /// Get frames decoded count - pub fn frames_decoded(&self) -> u64 { - self.frames_decoded - } - - /// Check if using hardware acceleration (always true for native DXVA) - pub fn is_hw_accel(&self) -> bool { - true - } -} - -impl Drop for NativeVideoDecoder { - fn drop(&mut self) { - let _ = self.cmd_tx.send(NativeDecoderCommand::Stop); - } -} diff --git a/opennow-streamer/src/media/rtp.rs b/opennow-streamer/src/media/rtp.rs deleted file mode 100644 index e3379e2..0000000 --- a/opennow-streamer/src/media/rtp.rs +++ /dev/null @@ -1,647 +0,0 @@ -//! RTP Depacketizer -//! -//! Depacketizes RTP payloads for H.264, H.265/HEVC, and AV1 video codecs. - -use log::{debug, warn}; - -/// Codec type for depacketizer -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum DepacketizerCodec { - H264, - H265, - AV1, -} - -/// RTP depacketizer supporting H.264, H.265/HEVC, and AV1 -pub struct RtpDepacketizer { - codec: DepacketizerCodec, - buffer: Vec, - fragments: Vec>, - in_fragment: bool, - /// Cached VPS NAL unit (H.265 only) - vps: Option>, - /// Cached SPS NAL unit - sps: Option>, - /// Cached PPS NAL unit - pps: Option>, - /// Accumulated OBUs for current AV1 frame (sent when marker bit is set) - av1_frame_buffer: Vec, - /// Cached AV1 SEQUENCE_HEADER OBU - must be present at start of each frame - av1_sequence_header: Option>, - /// Accumulated NAL units for current H.264/H.265 frame (sent when marker bit is set) - nal_frame_buffer: Vec, -} - -impl RtpDepacketizer { - pub fn new() -> Self { - Self::with_codec(DepacketizerCodec::H264) - } - - pub fn with_codec(codec: DepacketizerCodec) -> Self { - Self { - codec, - buffer: Vec::with_capacity(64 * 1024), - fragments: Vec::new(), - in_fragment: false, - vps: None, - sps: None, - pps: None, - av1_frame_buffer: Vec::with_capacity(256 * 1024), - av1_sequence_header: None, - nal_frame_buffer: Vec::with_capacity(256 * 1024), - } - } - - /// Set the codec type - pub fn set_codec(&mut self, codec: DepacketizerCodec) { - self.codec = codec; - // Clear cached parameter sets when codec changes - self.vps = None; - self.sps = None; - self.pps = None; - self.buffer.clear(); - self.in_fragment = false; - self.av1_frame_buffer.clear(); - self.av1_sequence_header = None; - self.nal_frame_buffer.clear(); - } - - /// Reset depacketizer state (call after decode errors to resync) - /// Preserves cached SEQUENCE_HEADER but clears all fragment state - pub fn reset_state(&mut self) { - self.buffer.clear(); - self.in_fragment = false; - self.av1_frame_buffer.clear(); - self.nal_frame_buffer.clear(); - // Keep av1_sequence_header cached - we need it for recovery - debug!("RTP depacketizer state reset"); - } - - /// Process AV1 RTP payload and accumulate directly to frame buffer - /// This handles GFN's non-standard AV1 RTP which has continuation packets - /// that don't properly follow RFC 9000 fragmentation rules - pub fn process_av1_raw(&mut self, payload: &[u8]) { - if payload.is_empty() { - return; - } - - let agg_header = payload[0]; - let z_flag = (agg_header & 0x80) != 0; - let y_flag = (agg_header & 0x40) != 0; - let w_field = (agg_header >> 4) & 0x03; - let n_flag = (agg_header & 0x08) != 0; - - if n_flag { - // New coded video sequence - clear everything - self.av1_frame_buffer.clear(); - self.buffer.clear(); - self.in_fragment = false; - } - - let mut offset = 1; - - // GFN bug workaround: When we're in the middle of accumulating a large OBU - // (like TILE_GROUP), treat ALL subsequent packets as raw continuation data - // until marker bit arrives. GFN doesn't properly set Z=1 flag. - if self.in_fragment { - // Just append raw data - don't try to parse aggregation header semantics - self.buffer.extend_from_slice(&payload[offset..]); - // Stay in fragment mode until marker bit triggers flush - return; - } - - if z_flag { - // Standard continuation packet (Z=1) - self.buffer.extend_from_slice(&payload[offset..]); - - if y_flag { - // Fragment complete - try to reconstruct OBU - if !self.buffer.is_empty() { - if let Some(obu) = Self::reconstruct_obu_with_size(&self.buffer) { - self.av1_frame_buffer.extend_from_slice(&obu); - } - } - self.buffer.clear(); - self.in_fragment = false; - } - return; - } - - // Not a continuation - parse OBU elements - let obu_count = if w_field == 0 { 1 } else { w_field as usize }; - - for i in 0..obu_count { - if offset >= payload.len() { - break; - } - - let obu_size = if w_field > 0 && i < obu_count - 1 { - let (size, bytes_read) = Self::read_leb128(&payload[offset..]); - offset += bytes_read; - size as usize - } else { - payload.len() - offset - }; - - if offset + obu_size > payload.len() { - break; - } - - let obu_data = &payload[offset..offset + obu_size]; - let obu_type = if !obu_data.is_empty() { (obu_data[0] >> 3) & 0x0F } else { 0 }; - - // Check if last OBU is fragmented or is a large OBU type that might span packets - // GFN bug: sometimes marks Y=1 even when TILE_GROUP/FRAME spans packets - let is_last = i == obu_count - 1; - let is_large_obu = obu_type == 4 || obu_type == 6; // TILE_GROUP or FRAME - - if is_last && (!y_flag || is_large_obu) { - // Start/continue fragmented OBU - save for potential continuation - self.buffer.clear(); - self.buffer.extend_from_slice(obu_data); - self.in_fragment = true; - } else if !obu_data.is_empty() { - // Complete OBU - reconstruct with size field and accumulate - if let Some(obu) = Self::reconstruct_obu_with_size(obu_data) { - self.av1_frame_buffer.extend_from_slice(&obu); - } - } - - offset += obu_size; - } - } - - /// Accumulate a NAL unit for the current H.264/H.265 frame - /// Each NAL unit is prefixed with Annex B start code (0x00 0x00 0x00 0x01) - /// Call take_nal_frame() when marker bit is set to get complete frame - pub fn accumulate_nal(&mut self, nal: Vec) { - // Add Annex B start code before each NAL unit - self.nal_frame_buffer.extend_from_slice(&[0x00, 0x00, 0x00, 0x01]); - self.nal_frame_buffer.extend_from_slice(&nal); - } - - /// Take the accumulated H.264/H.265 frame data (all NAL units with start codes) - /// Returns None if no data accumulated - pub fn take_nal_frame(&mut self) -> Option> { - if self.nal_frame_buffer.is_empty() { - return None; - } - let frame = std::mem::take(&mut self.nal_frame_buffer); - // Pre-allocate for next frame - self.nal_frame_buffer = Vec::with_capacity(256 * 1024); - Some(frame) - } - - /// Flush any pending OBU fragment to the frame buffer - /// Call this when marker bit is set before take_accumulated_frame() - pub fn flush_pending_obu(&mut self) { - if self.in_fragment && !self.buffer.is_empty() { - if let Some(obu) = Self::reconstruct_obu_with_size(&self.buffer) { - self.av1_frame_buffer.extend_from_slice(&obu); - } - self.buffer.clear(); - self.in_fragment = false; - } - } - - /// Take the accumulated AV1 frame data (all OBUs concatenated) - /// Returns None if no data accumulated or if frame doesn't contain picture data - pub fn take_accumulated_frame(&mut self) -> Option> { - if self.av1_frame_buffer.is_empty() { - return None; - } - let mut frame = std::mem::take(&mut self.av1_frame_buffer); - // Pre-allocate for next frame - self.av1_frame_buffer = Vec::with_capacity(256 * 1024); - - // Validate that frame contains actual picture data (TILE_GROUP or FRAME OBU) - // Without this, we'd send headers-only to decoder which can crash CUVID - if !Self::av1_frame_has_picture_data(&frame) { - // But still extract and cache SEQUENCE_HEADER if present - if let Some(seq_hdr) = Self::extract_sequence_header(&frame) { - self.av1_sequence_header = Some(seq_hdr); - } - return None; - } - - // Check if frame already has a SEQUENCE_HEADER - let has_sequence_header = Self::av1_frame_has_sequence_header(&frame); - - // If frame has SEQUENCE_HEADER, cache it for future frames - if has_sequence_header { - if let Some(seq_hdr) = Self::extract_sequence_header(&frame) { - self.av1_sequence_header = Some(seq_hdr); - } - } else if let Some(ref seq_hdr) = self.av1_sequence_header { - // Prepend cached SEQUENCE_HEADER to frame - let mut new_frame = Vec::with_capacity(seq_hdr.len() + frame.len()); - new_frame.extend_from_slice(seq_hdr); - new_frame.extend_from_slice(&frame); - frame = new_frame; - } - - Some(frame) - } - - /// Check if an AV1 frame contains actual picture data (TILE_GROUP or FRAME OBU) - /// Frames with only SEQUENCE_HEADER, FRAME_HEADER, etc. are not decodable - fn av1_frame_has_picture_data(data: &[u8]) -> bool { - Self::av1_find_obu_types(data).iter().any(|&t| t == 4 || t == 6) - } - - /// Check if an AV1 frame contains a SEQUENCE_HEADER OBU - fn av1_frame_has_sequence_header(data: &[u8]) -> bool { - Self::av1_find_obu_types(data).contains(&1) - } - - /// Find all OBU types in an AV1 bitstream - fn av1_find_obu_types(data: &[u8]) -> Vec { - let mut types = Vec::new(); - let mut offset = 0; - - while offset < data.len() { - // Parse OBU header - let header = data[offset]; - let obu_type = (header >> 3) & 0x0F; - let has_extension = (header & 0x04) != 0; - let has_size = (header & 0x02) != 0; - - types.push(obu_type); - - // Move to next OBU - let header_size = if has_extension { 2 } else { 1 }; - offset += header_size; - - if has_size && offset < data.len() { - let (size, bytes_read) = Self::read_leb128(&data[offset..]); - offset += bytes_read + size as usize; - } else { - // No size field - OBU extends to end of data - break; - } - } - types - } - - /// Extract the SEQUENCE_HEADER OBU from an AV1 bitstream - fn extract_sequence_header(data: &[u8]) -> Option> { - let mut offset = 0; - - while offset < data.len() { - let start_offset = offset; - - // Parse OBU header - let header = data[offset]; - let obu_type = (header >> 3) & 0x0F; - let has_extension = (header & 0x04) != 0; - let has_size = (header & 0x02) != 0; - - // Move past header - let header_size = if has_extension { 2 } else { 1 }; - offset += header_size; - - if has_size && offset < data.len() { - let (size, bytes_read) = Self::read_leb128(&data[offset..]); - offset += bytes_read; - - // If this is SEQUENCE_HEADER (type 1), extract it - if obu_type == 1 { - let end_offset = offset + size as usize; - if end_offset <= data.len() { - return Some(data[start_offset..end_offset].to_vec()); - } - } - - offset += size as usize; - } else { - // No size field - OBU extends to end of data - if obu_type == 1 { - return Some(data[start_offset..].to_vec()); - } - break; - } - } - None - } - - /// Process an RTP payload and return complete NAL units - /// Note: For AV1, use process_av1_raw() instead - this returns empty for AV1 - pub fn process(&mut self, payload: &[u8]) -> Vec> { - match self.codec { - DepacketizerCodec::H264 => self.process_h264(payload), - DepacketizerCodec::H265 => self.process_h265(payload), - DepacketizerCodec::AV1 => Vec::new(), // Use process_av1_raw() for AV1 - } - } - - /// Process H.264 RTP payload - fn process_h264(&mut self, payload: &[u8]) -> Vec> { - let mut result = Vec::new(); - - if payload.is_empty() { - return result; - } - - let nal_type = payload[0] & 0x1F; - - match nal_type { - // Single NAL unit (1-23) - 1..=23 => { - // Cache SPS/PPS for later use - if nal_type == 7 { - debug!("H264: Caching SPS ({} bytes)", payload.len()); - self.sps = Some(payload.to_vec()); - } else if nal_type == 8 { - debug!("H264: Caching PPS ({} bytes)", payload.len()); - self.pps = Some(payload.to_vec()); - } - result.push(payload.to_vec()); - } - - // STAP-A (24) - Single-time aggregation packet - 24 => { - let mut offset = 1; - debug!("H264 STAP-A packet: {} bytes total", payload.len()); - - while offset + 2 <= payload.len() { - let size = u16::from_be_bytes([payload[offset], payload[offset + 1]]) as usize; - offset += 2; - - if offset + size > payload.len() { - warn!("H264 STAP-A: invalid size {} at offset {}", size, offset); - break; - } - - let nal_data = payload[offset..offset + size].to_vec(); - let inner_nal_type = nal_data.first().map(|b| b & 0x1F).unwrap_or(0); - - // Cache SPS/PPS - if inner_nal_type == 7 { - self.sps = Some(nal_data.clone()); - } else if inner_nal_type == 8 { - self.pps = Some(nal_data.clone()); - } - - result.push(nal_data); - offset += size; - } - } - - // FU-A (28) - Fragmentation unit - 28 => { - if payload.len() < 2 { - return result; - } - - let fu_header = payload[1]; - let start = (fu_header & 0x80) != 0; - let end = (fu_header & 0x40) != 0; - let inner_nal_type = fu_header & 0x1F; - - if start { - self.buffer.clear(); - self.in_fragment = true; - let nal_header = (payload[0] & 0xE0) | inner_nal_type; - self.buffer.push(nal_header); - self.buffer.extend_from_slice(&payload[2..]); - } else if self.in_fragment { - self.buffer.extend_from_slice(&payload[2..]); - } - - if end && self.in_fragment { - self.in_fragment = false; - let inner_nal_type = self.buffer.first().map(|b| b & 0x1F).unwrap_or(0); - - // For IDR frames, prepend SPS/PPS - if inner_nal_type == 5 { - if let (Some(sps), Some(pps)) = (&self.sps, &self.pps) { - result.push(sps.clone()); - result.push(pps.clone()); - } - } - - result.push(self.buffer.clone()); - } - } - - _ => { - debug!("H264: Unknown NAL type: {}", nal_type); - } - } - - result - } - - /// Process H.265/HEVC RTP payload (RFC 7798) - fn process_h265(&mut self, payload: &[u8]) -> Vec> { - let mut result = Vec::new(); - - if payload.len() < 2 { - return result; - } - - // H.265 NAL unit header is 2 bytes - // Type is in bits 1-6 of first byte: (byte0 >> 1) & 0x3F - let nal_type = (payload[0] >> 1) & 0x3F; - - match nal_type { - // Single NAL unit (0-47, but 48 and 49 are special) - 0..=47 => { - // Cache VPS/SPS/PPS for later use - match nal_type { - 32 => { - debug!("H265: Caching VPS ({} bytes)", payload.len()); - self.vps = Some(payload.to_vec()); - } - 33 => { - debug!("H265: Caching SPS ({} bytes)", payload.len()); - self.sps = Some(payload.to_vec()); - } - 34 => { - debug!("H265: Caching PPS ({} bytes)", payload.len()); - self.pps = Some(payload.to_vec()); - } - _ => {} - } - result.push(payload.to_vec()); - } - - // AP (48) - Aggregation Packet - 48 => { - let mut offset = 2; // Skip the 2-byte NAL unit header - debug!("H265 AP packet: {} bytes total", payload.len()); - - while offset + 2 <= payload.len() { - let size = u16::from_be_bytes([payload[offset], payload[offset + 1]]) as usize; - offset += 2; - - if offset + size > payload.len() { - warn!("H265 AP: invalid size {} at offset {}", size, offset); - break; - } - - let nal_data = payload[offset..offset + size].to_vec(); - - if nal_data.len() >= 2 { - let inner_nal_type = (nal_data[0] >> 1) & 0x3F; - // Cache VPS/SPS/PPS - match inner_nal_type { - 32 => self.vps = Some(nal_data.clone()), - 33 => self.sps = Some(nal_data.clone()), - 34 => self.pps = Some(nal_data.clone()), - _ => {} - } - } - - result.push(nal_data); - offset += size; - } - } - - // FU (49) - Fragmentation Unit - 49 => { - if payload.len() < 3 { - return result; - } - - // FU header is at byte 2 - let fu_header = payload[2]; - let start = (fu_header & 0x80) != 0; - let end = (fu_header & 0x40) != 0; - let inner_nal_type = fu_header & 0x3F; - - if start { - self.buffer.clear(); - self.in_fragment = true; - - // Reconstruct NAL unit header from original header + inner type - // H265 NAL header: forbidden_zero_bit(1) | nal_unit_type(6) | nuh_layer_id(6) | nuh_temporal_id_plus1(3) - // First byte: (forbidden_zero_bit << 7) | (inner_nal_type << 1) | (layer_id >> 5) - // Second byte: (layer_id << 3) | temporal_id - let layer_id = payload[0] & 0x01; // lowest bit of first byte - let temporal_id = payload[1]; // second byte - - let nal_header_byte0 = (inner_nal_type << 1) | layer_id; - let nal_header_byte1 = temporal_id; - - self.buffer.push(nal_header_byte0); - self.buffer.push(nal_header_byte1); - self.buffer.extend_from_slice(&payload[3..]); - } else if self.in_fragment { - self.buffer.extend_from_slice(&payload[3..]); - } - - if end && self.in_fragment { - self.in_fragment = false; - - if self.buffer.len() >= 2 { - let inner_nal_type = (self.buffer[0] >> 1) & 0x3F; - - // For IDR frames (types 19 and 20), prepend VPS/SPS/PPS - if inner_nal_type == 19 || inner_nal_type == 20 { - if let Some(vps) = &self.vps { - result.push(vps.clone()); - } - if let Some(sps) = &self.sps { - result.push(sps.clone()); - } - if let Some(pps) = &self.pps { - result.push(pps.clone()); - } - } - } - - result.push(self.buffer.clone()); - } - } - - _ => { - debug!("H265: Unknown NAL type: {}", nal_type); - } - } - - result - } - - /// Reconstruct an OBU with the obu_size field included - /// RTP format strips the size field, but decoders need it - fn reconstruct_obu_with_size(obu_data: &[u8]) -> Option> { - if obu_data.is_empty() { - return None; - } - - // Parse OBU header - let header = obu_data[0]; - let has_extension = (header & 0x04) != 0; - let has_size_field = (header & 0x02) != 0; - - // If it already has a size field, return as-is - if has_size_field { - return Some(obu_data.to_vec()); - } - - // Calculate payload size (everything after header and optional extension) - let header_size = if has_extension { 2 } else { 1 }; - if obu_data.len() < header_size { - return None; - } - - let payload_size = obu_data.len() - header_size; - - // Build new OBU with size field - let mut new_obu = Vec::with_capacity(obu_data.len() + 8); - - // Modified header with has_size_field = 1 - new_obu.push(header | 0x02); - - // Copy extension byte if present - if has_extension && obu_data.len() > 1 { - new_obu.push(obu_data[1]); - } - - // Write payload size as LEB128 - Self::write_leb128(&mut new_obu, payload_size as u64); - - // Copy payload - new_obu.extend_from_slice(&obu_data[header_size..]); - - Some(new_obu) - } - - /// Read LEB128 encoded unsigned integer - fn read_leb128(data: &[u8]) -> (u64, usize) { - let mut value: u64 = 0; - let mut bytes_read = 0; - - for (i, &byte) in data.iter().enumerate().take(8) { - value |= ((byte & 0x7F) as u64) << (i * 7); - bytes_read = i + 1; - if (byte & 0x80) == 0 { - break; - } - } - - (value, bytes_read) - } - - /// Write LEB128 encoded unsigned integer - fn write_leb128(output: &mut Vec, mut value: u64) { - loop { - let mut byte = (value & 0x7F) as u8; - value >>= 7; - if value != 0 { - byte |= 0x80; // More bytes follow - } - output.push(byte); - if value == 0 { - break; - } - } - } -} - -impl Default for RtpDepacketizer { - fn default() -> Self { - Self::new() - } -} diff --git a/opennow-streamer/src/media/v4l2.rs b/opennow-streamer/src/media/v4l2.rs deleted file mode 100644 index b249cb5..0000000 --- a/opennow-streamer/src/media/v4l2.rs +++ /dev/null @@ -1,363 +0,0 @@ -//! V4L2 Video Decoder Support for Linux (Raspberry Pi) -//! -//! This module provides hardware video decoding on Raspberry Pi using V4L2 M2M -//! (Memory-to-Memory) stateful codec interface. -//! -//! Supported hardware: -//! - Raspberry Pi 4: H.264 decode via bcm2835-codec -//! - Raspberry Pi 5: H.264 and HEVC decode via rpivid (stateless, limited FFmpeg support) -//! -//! Note: Pi 5's HEVC decoder uses stateless API which requires special handling. -//! For best compatibility, H.264 is recommended on Raspberry Pi. -//! -//! Flow: -//! 1. FFmpeg v4l2m2m decodes to DMA-BUF backed buffer -//! 2. We extract the DMA-BUF fd from the V4L2 buffer -//! 3. Import into Vulkan/GL via EGL_EXT_image_dma_buf_import -//! 4. Render via wgpu -//! -//! Fallback: If zero-copy fails, we mmap the buffer and copy to CPU memory. - -use anyhow::{anyhow, Result}; -use log::{debug, info, warn}; -use std::os::unix::io::RawFd; -use std::path::Path; - -/// V4L2 buffer wrapper from FFmpeg hardware decoder -pub struct V4L2BufferWrapper { - /// DMA-BUF file descriptor - dmabuf_fd: RawFd, - /// Buffer dimensions - pub width: u32, - pub height: u32, - /// Pixel format (NV12 for Pi decoders) - pub format: V4L2PixelFormat, - /// Whether we own the fd (should close on drop) - owns_fd: bool, -} - -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum V4L2PixelFormat { - NV12, - NV21, - YUV420, - Unknown, -} - -// Safety: DMA-BUF fds can be shared across threads -unsafe impl Send for V4L2BufferWrapper {} -unsafe impl Sync for V4L2BufferWrapper {} - -impl std::fmt::Debug for V4L2BufferWrapper { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("V4L2BufferWrapper") - .field("dmabuf_fd", &self.dmabuf_fd) - .field("width", &self.width) - .field("height", &self.height) - .field("format", &self.format) - .finish() - } -} - -impl V4L2BufferWrapper { - /// Create a wrapper from a DMA-BUF fd - pub fn new(dmabuf_fd: RawFd, width: u32, height: u32, format: V4L2PixelFormat) -> Self { - Self { - dmabuf_fd, - width, - height, - format, - owns_fd: false, // FFmpeg owns the fd - } - } - - /// Get the DMA-BUF fd for import - pub fn dmabuf_fd(&self) -> RawFd { - self.dmabuf_fd - } - - /// Lock the buffer and copy planes to CPU memory (fallback path) - pub fn lock_and_get_planes(&self) -> Result { - unsafe { - // Calculate sizes based on NV12 format - let y_size = (self.width * self.height) as usize; - let uv_size = y_size / 2; - let total_size = y_size + uv_size; - - // mmap the DMA-BUF - let ptr = libc::mmap( - std::ptr::null_mut(), - total_size, - libc::PROT_READ, - libc::MAP_SHARED, - self.dmabuf_fd, - 0, - ); - - if ptr == libc::MAP_FAILED { - return Err(anyhow!("mmap failed: {}", std::io::Error::last_os_error())); - } - - // Copy the data - let data = std::slice::from_raw_parts(ptr as *const u8, total_size); - let y_plane = data[..y_size].to_vec(); - let uv_plane = data[y_size..].to_vec(); - - // Unmap - libc::munmap(ptr, total_size); - - Ok(LockedPlanes { - y_plane, - uv_plane, - y_stride: self.width, - uv_stride: self.width, - width: self.width, - height: self.height, - }) - } - } -} - -impl Drop for V4L2BufferWrapper { - fn drop(&mut self) { - if self.owns_fd && self.dmabuf_fd >= 0 { - unsafe { - libc::close(self.dmabuf_fd); - } - } - } -} - -/// Locked plane data from V4L2 buffer -pub struct LockedPlanes { - pub y_plane: Vec, - pub uv_plane: Vec, - pub y_stride: u32, - pub uv_stride: u32, - pub width: u32, - pub height: u32, -} - -/// Detect if running on Raspberry Pi -pub fn is_raspberry_pi() -> bool { - // Check for Pi-specific indicators - if Path::new("/sys/firmware/devicetree/base/model").exists() { - if let Ok(model) = std::fs::read_to_string("/sys/firmware/devicetree/base/model") { - let model_lower = model.to_lowercase(); - return model_lower.contains("raspberry pi"); - } - } - - // Check for bcm2835/bcm2711/bcm2712 in /proc/cpuinfo - if let Ok(cpuinfo) = std::fs::read_to_string("/proc/cpuinfo") { - let cpuinfo_lower = cpuinfo.to_lowercase(); - return cpuinfo_lower.contains("bcm2835") - || cpuinfo_lower.contains("bcm2711") - || cpuinfo_lower.contains("bcm2712"); - } - - false -} - -/// Get Raspberry Pi model (4, 5, etc.) -pub fn get_pi_model() -> Option { - if let Ok(model) = std::fs::read_to_string("/sys/firmware/devicetree/base/model") { - if model.contains("Raspberry Pi 5") { - return Some(5); - } else if model.contains("Raspberry Pi 4") { - return Some(4); - } else if model.contains("Raspberry Pi 3") { - return Some(3); - } else if model.contains("Raspberry Pi") { - return Some(0); // Unknown Pi version - } - } - None -} - -/// Find the V4L2 M2M decoder device for the given codec -pub fn find_v4l2_decoder_device(codec: V4L2Codec) -> Option { - // Common V4L2 M2M device paths on Raspberry Pi - let search_paths = match codec { - V4L2Codec::H264 => vec![ - "/dev/video10", // bcm2835-codec on Pi 4 - "/dev/video11", - "/dev/video19", // rpivid on Pi 5 - ], - V4L2Codec::HEVC => vec![ - "/dev/video19", // rpivid HEVC on Pi 5 - "/dev/video10", - ], - }; - - for path in search_paths { - if Path::new(path).exists() { - // Try to query the device capabilities - if let Ok(file) = std::fs::File::open(path) { - use std::os::unix::io::AsRawFd; - let fd = file.as_raw_fd(); - - // Query V4L2 capabilities (simplified check) - if query_v4l2_caps(fd, codec) { - info!("Found V4L2 M2M decoder for {:?} at {}", codec, path); - return Some(path.to_string()); - } - } - } - } - - None -} - -#[derive(Debug, Clone, Copy)] -pub enum V4L2Codec { - H264, - HEVC, -} - -/// Query V4L2 device capabilities (simplified) -fn query_v4l2_caps(fd: RawFd, _codec: V4L2Codec) -> bool { - // V4L2 ioctl numbers - const VIDIOC_QUERYCAP: libc::c_ulong = 0x80685600; - - #[repr(C)] - struct v4l2_capability { - driver: [u8; 16], - card: [u8; 32], - bus_info: [u8; 32], - version: u32, - capabilities: u32, - device_caps: u32, - reserved: [u32; 3], - } - - const V4L2_CAP_VIDEO_M2M_MPLANE: u32 = 0x00004000; - const V4L2_CAP_VIDEO_M2M: u32 = 0x00008000; - - unsafe { - let mut caps: v4l2_capability = std::mem::zeroed(); - let ret = libc::ioctl(fd, VIDIOC_QUERYCAP, &mut caps); - - if ret < 0 { - return false; - } - - let device_caps = if caps.device_caps != 0 { - caps.device_caps - } else { - caps.capabilities - }; - - // Check for M2M capability - let is_m2m = (device_caps & V4L2_CAP_VIDEO_M2M) != 0 - || (device_caps & V4L2_CAP_VIDEO_M2M_MPLANE) != 0; - - if is_m2m { - let driver = String::from_utf8_lossy(&caps.driver); - let card = String::from_utf8_lossy(&caps.card); - debug!( - "V4L2 device: driver={}, card={}", - driver.trim_end_matches('\0'), - card.trim_end_matches('\0') - ); - return true; - } - } - - false -} - -/// Manager for V4L2 zero-copy buffers -pub struct V4L2ZeroCopyManager { - enabled: bool, - pi_model: Option, -} - -impl V4L2ZeroCopyManager { - pub fn new() -> Self { - let pi_model = get_pi_model(); - if let Some(model) = pi_model { - info!( - "Raspberry Pi {} detected - V4L2 hardware decoding available", - model - ); - } - - Self { - enabled: pi_model.is_some(), - pi_model, - } - } - - pub fn is_enabled(&self) -> bool { - self.enabled - } - - pub fn pi_model(&self) -> Option { - self.pi_model - } - - pub fn disable(&mut self) { - warn!("V4L2 zero-copy disabled, falling back to CPU path"); - self.enabled = false; - } -} - -impl Default for V4L2ZeroCopyManager { - fn default() -> Self { - Self::new() - } -} - -/// Check if V4L2 M2M decoding is available for the given codec -pub fn is_v4l2_available(codec: V4L2Codec) -> bool { - if !is_raspberry_pi() { - return false; - } - - find_v4l2_decoder_device(codec).is_some() -} - -/// Get recommended video codec for this Raspberry Pi -pub fn get_recommended_codec() -> Option { - match get_pi_model() { - Some(5) => { - // Pi 5 has HEVC hardware decoder (stateless via rpivid) - // But H.264 is more reliable with FFmpeg - if is_v4l2_available(V4L2Codec::HEVC) { - Some(V4L2Codec::HEVC) - } else if is_v4l2_available(V4L2Codec::H264) { - Some(V4L2Codec::H264) - } else { - None - } - } - Some(4) | Some(3) => { - // Pi 4/3 only have H.264 hardware decoder - if is_v4l2_available(V4L2Codec::H264) { - Some(V4L2Codec::H264) - } else { - None - } - } - _ => None, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_pi_detection() { - // This will only pass on actual Pi hardware - let is_pi = is_raspberry_pi(); - println!("Is Raspberry Pi: {}", is_pi); - - if is_pi { - println!("Pi Model: {:?}", get_pi_model()); - println!("H264 available: {}", is_v4l2_available(V4L2Codec::H264)); - println!("HEVC available: {}", is_v4l2_available(V4L2Codec::HEVC)); - } - } -} diff --git a/opennow-streamer/src/media/vaapi.rs b/opennow-streamer/src/media/vaapi.rs deleted file mode 100644 index 032bead..0000000 --- a/opennow-streamer/src/media/vaapi.rs +++ /dev/null @@ -1,601 +0,0 @@ -//! VA-API Zero-Copy Video Support for Linux -//! -//! This module provides zero-copy video rendering on Linux by keeping -//! decoded frames on GPU as VA surfaces and sharing them with Vulkan. -//! -//! Supports: -//! - AMD GPUs via RADV/AMDGPU-PRO drivers -//! - Intel GPUs via Intel Media Driver (iHD) or i965 -//! - NVIDIA GPUs via nouveau (limited) or nvidia-vaapi-driver -//! -//! Flow: -//! 1. FFmpeg VAAPI decodes to VASurface (GPU VRAM) -//! 2. We export the surface as a DMA-BUF fd -//! 3. Import into Vulkan via VK_EXT_external_memory_dma_buf -//! 4. Bind to wgpu texture for rendering -//! -//! This eliminates the expensive GPU->CPU->GPU round-trip. - -use anyhow::{anyhow, Result}; -use log::{debug, info, warn}; -use parking_lot::Mutex; -use std::os::unix::io::RawFd; -use std::sync::OnceLock; - -/// VA surface format (matches VA-API definitions) -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum VASurfaceFormat { - NV12, // 8-bit 4:2:0 - P010, // 10-bit 4:2:0 (HDR) - Unknown, -} - -/// Wrapper for a VA-API surface from FFmpeg hardware decoder -/// Holds the surface reference and provides DMA-BUF export -pub struct VAAPISurfaceWrapper { - /// VA display handle (void* to libva VADisplay) - va_display: *mut std::ffi::c_void, - /// VA surface ID - surface_id: u32, - /// DMA-BUF file descriptor (lazily exported) - dmabuf_fd: Mutex>, - /// Surface dimensions - pub width: u32, - pub height: u32, - /// Surface format - pub format: VASurfaceFormat, - /// DRM format fourcc (for Vulkan import) - pub drm_format: u32, - /// DRM modifier (for tiled formats) - pub drm_modifier: u64, - /// Plane info for multi-planar formats - pub planes: Vec, -} - -/// Information about a single plane in a multi-planar surface -#[derive(Debug, Clone)] -pub struct PlaneInfo { - pub offset: u32, - pub pitch: u32, -} - -// Safety: VA-API surfaces can be shared across threads when properly synchronized -unsafe impl Send for VAAPISurfaceWrapper {} -unsafe impl Sync for VAAPISurfaceWrapper {} - -impl std::fmt::Debug for VAAPISurfaceWrapper { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("VAAPISurfaceWrapper") - .field("surface_id", &self.surface_id) - .field("width", &self.width) - .field("height", &self.height) - .field("format", &self.format) - .field("has_dmabuf", &self.dmabuf_fd.lock().is_some()) - .finish() - } -} - -// DRM fourcc codes -const DRM_FORMAT_NV12: u32 = 0x3231564E; // NV12 -const DRM_FORMAT_P010: u32 = 0x30313050; // P010 - -// Cached libva library handle -static LIBVA: OnceLock> = OnceLock::new(); - -/// Get or load the libva library (cached) -fn get_libva() -> Result<&'static libloading::Library> { - let result = LIBVA.get_or_init(|| unsafe { - libloading::Library::new("libva.so.2") - .or_else(|_| libloading::Library::new("libva.so")) - .map_err(|e| format!("Failed to load libva: {}", e)) - }); - - match result { - Ok(lib) => Ok(lib), - Err(e) => Err(anyhow!("{}", e)), - } -} - -// VA-API FFI bindings (minimal set for surface export) -// Full bindings would come from libva-sys crate -mod ffi { - use std::ffi::c_void; - - pub type VADisplay = *mut c_void; - pub type VASurfaceID = u32; - pub type VAStatus = i32; - pub type VABufferID = u32; - pub type VAImageID = u32; - - pub const VA_STATUS_SUCCESS: VAStatus = 0; - - // VA surface export flags - pub const VA_EXPORT_SURFACE_READ_ONLY: u32 = 0x0001; - pub const VA_EXPORT_SURFACE_SEPARATE_LAYERS: u32 = 0x0004; - - // VAImage structure for vaDeriveImage - #[repr(C)] - pub struct VAImage { - pub image_id: VAImageID, - pub format: VAImageFormat, - pub buf: VABufferID, - pub width: u16, - pub height: u16, - pub data_size: u32, - pub num_planes: u32, - pub pitches: [u32; 3], - pub offsets: [u32; 3], - pub num_palette_entries: i32, - pub entry_bytes: i32, - pub component_order: [i8; 4], - } - - #[repr(C)] - pub struct VAImageFormat { - pub fourcc: u32, - pub byte_order: u32, - pub bits_per_pixel: u32, - pub depth: u32, - pub red_mask: u32, - pub green_mask: u32, - pub blue_mask: u32, - pub alpha_mask: u32, - pub va_reserved: [u32; 4], - } - - // VADRMPRIMESurfaceDescriptor structure - #[repr(C)] - pub struct VADRMPRIMESurfaceDescriptor { - pub fourcc: u32, - pub width: u32, - pub height: u32, - pub num_objects: u32, - pub objects: [VADRMPRIMEObject; 4], - pub num_layers: u32, - pub layers: [VADRMPRIMELayer; 4], - } - - #[repr(C)] - pub struct VADRMPRIMEObject { - pub fd: i32, - pub size: u32, - pub drm_format_modifier: u64, - } - - #[repr(C)] - pub struct VADRMPRIMELayer { - pub drm_format: u32, - pub num_planes: u32, - pub object_index: [u32; 4], - pub offset: [u32; 4], - pub pitch: [u32; 4], - } - - // VA-API function types - pub type VaExportSurfaceHandle = unsafe extern "C" fn( - dpy: VADisplay, - surface: VASurfaceID, - mem_type: u32, - flags: u32, - descriptor: *mut VADRMPRIMESurfaceDescriptor, - ) -> VAStatus; - - pub type VaSyncSurface = unsafe extern "C" fn(dpy: VADisplay, surface: VASurfaceID) -> VAStatus; - - pub type VaDeriveImage = - unsafe extern "C" fn(dpy: VADisplay, surface: VASurfaceID, image: *mut VAImage) -> VAStatus; - - pub type VaMapBuffer = unsafe extern "C" fn( - dpy: VADisplay, - buf_id: VABufferID, - pbuf: *mut *mut c_void, - ) -> VAStatus; - - pub type VaUnmapBuffer = unsafe extern "C" fn(dpy: VADisplay, buf_id: VABufferID) -> VAStatus; - - pub type VaDestroyImage = unsafe extern "C" fn(dpy: VADisplay, image: VAImageID) -> VAStatus; - - // Memory type for DRM PRIME export - pub const VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2: u32 = 0x40000000; -} - -impl VAAPISurfaceWrapper { - /// Create a new wrapper from FFmpeg's VAAPI frame data - /// - /// # Safety - /// The va_display and surface_id must be from a valid VAAPI decoded frame - pub unsafe fn from_ffmpeg_frame( - va_display: *mut std::ffi::c_void, - surface_id: u32, - width: u32, - height: u32, - ) -> Option { - if va_display.is_null() || surface_id == 0 { - warn!( - "Invalid VAAPI surface: display={:?}, surface_id={}", - va_display, surface_id - ); - return None; - } - - debug!( - "VAAPI surface wrapper: {}x{}, surface_id={}", - width, height, surface_id - ); - - Some(Self { - va_display, - surface_id, - dmabuf_fd: Mutex::new(None), - width, - height, - format: VASurfaceFormat::NV12, // Default, will be updated on export - drm_format: DRM_FORMAT_NV12, - drm_modifier: 0, - planes: Vec::new(), - }) - } - - /// Export the surface as a DMA-BUF for Vulkan import - /// Returns the file descriptor and updates format info - pub fn export_dmabuf(&self) -> Result { - let mut guard = self.dmabuf_fd.lock(); - if let Some(fd) = *guard { - return Ok(fd); - } - - unsafe { - // Get cached libva - let libva = get_libva()?; - - // Get function pointers - let va_sync_surface: libloading::Symbol = libva - .get(b"vaSyncSurface\0") - .map_err(|e| anyhow!("vaSyncSurface not found: {}", e))?; - - let va_export_surface_handle: libloading::Symbol = libva - .get(b"vaExportSurfaceHandle\0") - .map_err(|e| anyhow!("vaExportSurfaceHandle not found: {}", e))?; - - // Sync the surface before export (wait for decode to complete) - let status = va_sync_surface(self.va_display, self.surface_id); - if status != ffi::VA_STATUS_SUCCESS { - return Err(anyhow!("vaSyncSurface failed with status {}", status)); - } - - // Export as DRM PRIME (DMA-BUF) - let mut desc: ffi::VADRMPRIMESurfaceDescriptor = std::mem::zeroed(); - let status = va_export_surface_handle( - self.va_display, - self.surface_id, - ffi::VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2, - ffi::VA_EXPORT_SURFACE_READ_ONLY | ffi::VA_EXPORT_SURFACE_SEPARATE_LAYERS, - &mut desc, - ); - - if status != ffi::VA_STATUS_SUCCESS { - return Err(anyhow!( - "vaExportSurfaceHandle failed with status {}", - status - )); - } - - if desc.num_objects == 0 { - return Err(anyhow!("No DMA-BUF objects exported")); - } - - // Get the primary fd (first object) - let fd = desc.objects[0].fd; - if fd < 0 { - return Err(anyhow!("Invalid DMA-BUF fd: {}", fd)); - } - - debug!( - "VAAPI DMA-BUF export: fd={}, fourcc={:08x}, modifier={:x}, layers={}", - fd, desc.fourcc, desc.objects[0].drm_format_modifier, desc.num_layers - ); - - *guard = Some(fd); - Ok(fd) - } - } - - /// Get the surface ID (for FFmpeg/VA-API operations) - pub fn surface_id(&self) -> u32 { - self.surface_id - } - - /// Check if this is a 10-bit HDR surface - pub fn is_10bit(&self) -> bool { - self.format == VASurfaceFormat::P010 - } - - /// Lock the surface and copy Y and UV planes to CPU memory - /// This is the fallback path when zero-copy import fails - /// - /// Uses vaDeriveImage + vaMapBuffer for proper stride handling - pub fn lock_and_get_planes(&self) -> Result { - unsafe { - // Get cached libva - let libva = get_libva()?; - - // Sync surface first - let va_sync_surface: libloading::Symbol = libva - .get(b"vaSyncSurface\0") - .map_err(|e| anyhow!("vaSyncSurface not found: {}", e))?; - - let status = va_sync_surface(self.va_display, self.surface_id); - if status != ffi::VA_STATUS_SUCCESS { - return Err(anyhow!("vaSyncSurface failed: {}", status)); - } - - // Try vaDeriveImage for proper stride handling - let va_derive_image: Result, _> = - libva.get(b"vaDeriveImage\0"); - let va_map_buffer: Result, _> = - libva.get(b"vaMapBuffer\0"); - let va_unmap_buffer: Result, _> = - libva.get(b"vaUnmapBuffer\0"); - let va_destroy_image: Result, _> = - libva.get(b"vaDestroyImage\0"); - - if let (Ok(derive), Ok(map), Ok(unmap), Ok(destroy)) = ( - va_derive_image, - va_map_buffer, - va_unmap_buffer, - va_destroy_image, - ) { - // Use vaDeriveImage path - proper stride handling - let mut image: ffi::VAImage = std::mem::zeroed(); - let status = derive(self.va_display, self.surface_id, &mut image); - - if status == ffi::VA_STATUS_SUCCESS { - let mut buf_ptr: *mut std::ffi::c_void = std::ptr::null_mut(); - let status = map(self.va_display, image.buf, &mut buf_ptr); - - if status == ffi::VA_STATUS_SUCCESS && !buf_ptr.is_null() { - // Extract planes with proper stride - let y_stride = image.pitches[0] as usize; - let uv_stride = image.pitches[1] as usize; - let y_offset = image.offsets[0] as usize; - let uv_offset = image.offsets[1] as usize; - let height = self.height as usize; - let width = self.width as usize; - let uv_height = height / 2; - - // Copy Y plane (row by row if stride != width) - let mut y_plane = Vec::with_capacity(width * height); - let src = buf_ptr as *const u8; - for row in 0..height { - let row_start = y_offset + row * y_stride; - let row_data = std::slice::from_raw_parts(src.add(row_start), width); - y_plane.extend_from_slice(row_data); - } - - // Copy UV plane (row by row if stride != width) - let mut uv_plane = Vec::with_capacity(width * uv_height); - for row in 0..uv_height { - let row_start = uv_offset + row * uv_stride; - let row_data = std::slice::from_raw_parts(src.add(row_start), width); - uv_plane.extend_from_slice(row_data); - } - - // Cleanup - unmap(self.va_display, image.buf); - destroy(self.va_display, image.image_id); - - return Ok(LockedPlanes { - y_plane, - uv_plane, - y_stride: width as u32, - uv_stride: width as u32, - width: self.width, - height: self.height, - }); - } - - // Cleanup on failure - destroy(self.va_display, image.image_id); - } - } - - // Fallback: DMA-BUF mmap (may not handle stride correctly for all drivers) - warn!("vaDeriveImage failed, falling back to DMA-BUF mmap"); - let fd = self.export_dmabuf()?; - - // Assume linear layout with no padding (works for most cases) - // Real stride info should come from the DRM PRIME descriptor - let y_size = (self.width * self.height) as usize; - let uv_size = y_size / 2; - let total_size = y_size + uv_size; - - let ptr = libc::mmap( - std::ptr::null_mut(), - total_size, - libc::PROT_READ, - libc::MAP_SHARED, - fd, - 0, - ); - - if ptr == libc::MAP_FAILED { - return Err(anyhow!("mmap failed: {}", std::io::Error::last_os_error())); - } - - let data = std::slice::from_raw_parts(ptr as *const u8, total_size); - let y_plane = data[..y_size].to_vec(); - let uv_plane = data[y_size..].to_vec(); - - libc::munmap(ptr, total_size); - - Ok(LockedPlanes { - y_plane, - uv_plane, - y_stride: self.width, - uv_stride: self.width, - width: self.width, - height: self.height, - }) - } - } -} - -impl Drop for VAAPISurfaceWrapper { - fn drop(&mut self) { - // Close the DMA-BUF fd if we exported one - if let Some(fd) = self.dmabuf_fd.lock().take() { - unsafe { - libc::close(fd); - } - } - // Note: The VA surface itself is owned by FFmpeg and will be released - // when the AVFrame is freed - } -} - -/// Locked plane data from VAAPI surface -pub struct LockedPlanes { - pub y_plane: Vec, - pub uv_plane: Vec, - pub y_stride: u32, - pub uv_stride: u32, - pub width: u32, - pub height: u32, -} - -/// Manager for VAAPI zero-copy surfaces -/// Handles Vulkan interop setup -pub struct VaapiZeroCopyManager { - /// Whether zero-copy is enabled - enabled: bool, - /// VA display (cached for surface operations) - va_display: Option<*mut std::ffi::c_void>, -} - -// Safety: VA display pointer is thread-safe when properly synchronized -unsafe impl Send for VaapiZeroCopyManager {} -unsafe impl Sync for VaapiZeroCopyManager {} - -impl VaapiZeroCopyManager { - /// Create a new manager - pub fn new() -> Self { - info!("VAAPI zero-copy manager created"); - Self { - enabled: true, - va_display: None, - } - } - - /// Set the VA display handle (from FFmpeg decoder context) - pub fn set_va_display(&mut self, display: *mut std::ffi::c_void) { - self.va_display = Some(display); - } - - /// Check if zero-copy is enabled - pub fn is_enabled(&self) -> bool { - self.enabled - } - - /// Disable zero-copy (fallback to CPU path) - pub fn disable(&mut self) { - warn!("VAAPI zero-copy disabled, falling back to CPU path"); - self.enabled = false; - } -} - -impl Default for VaapiZeroCopyManager { - fn default() -> Self { - Self::new() - } -} - -/// Extract VAAPI surface from FFmpeg frame data pointers -/// -/// FFmpeg VAAPI frame layout: -/// - data[3] = VASurfaceID (stored as pointer-sized value) -/// - hw_frames_ctx contains VADisplay -/// -/// # Safety -/// The data pointers must be from a valid VAAPI decoded frame -pub unsafe fn extract_vaapi_surface_from_frame( - data3: *mut u8, - va_display: *mut std::ffi::c_void, - width: u32, - height: u32, -) -> Option { - if data3.is_null() || va_display.is_null() { - return None; - } - - // data[3] contains VASurfaceID as a pointer-sized value - let surface_id = data3 as usize as u32; - - VAAPISurfaceWrapper::from_ffmpeg_frame(va_display, surface_id, width, height) -} - -/// Check if VAAPI is available on this system -pub fn is_vaapi_available() -> bool { - // Try to load libva - unsafe { - match libloading::Library::new("libva.so.2") { - Ok(_) => true, - Err(_) => match libloading::Library::new("libva.so") { - Ok(_) => true, - Err(_) => { - debug!("libva not found - VAAPI not available"); - false - } - }, - } - } -} - -/// Get the VAAPI driver name for the current GPU -pub fn get_vaapi_driver_name() -> Option { - // Check environment variable first - if let Ok(driver) = std::env::var("LIBVA_DRIVER_NAME") { - return Some(driver); - } - - // Try to detect from DRI device - if let Ok(entries) = std::fs::read_dir("/dev/dri") { - for entry in entries.filter_map(|e| e.ok()) { - let path = entry.path(); - if let Some(name) = path.file_name().and_then(|n| n.to_str()) { - if name.starts_with("renderD") { - // Found a render node - VAAPI is likely available - return Some("auto".to_string()); - } - } - } - } - - None -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_locked_planes_layout() { - // Test NV12 plane calculations - let width = 1920u32; - let height = 1080u32; - - // Y plane: full resolution - let y_size = width * height; - assert_eq!(y_size, 2073600); - - // UV plane: half height, same width (interleaved) - let uv_size = width * (height / 2); - assert_eq!(uv_size, 1036800); - } - - #[test] - fn test_drm_format_codes() { - // Verify fourcc codes - assert_eq!(DRM_FORMAT_NV12, 0x3231564E); - assert_eq!(DRM_FORMAT_P010, 0x30313050); - } -} diff --git a/opennow-streamer/src/media/video.rs b/opennow-streamer/src/media/video.rs deleted file mode 100644 index a7a6e3a..0000000 --- a/opennow-streamer/src/media/video.rs +++ /dev/null @@ -1,2587 +0,0 @@ -//! Video Decoder -//! -//! Hardware-accelerated H.264/H.265 decoding. -//! -//! Platform-specific backends: -//! - Windows: Native DXVA (D3D11 Video API) -//! - macOS: FFmpeg with VideoToolbox -//! - Linux: Native Vulkan Video or GStreamer -//! -//! This module provides both blocking and non-blocking decode modes: -//! - Blocking: `decode()` - waits for result (legacy, causes latency) -//! - Non-blocking: `decode_async()` - fire-and-forget, writes to SharedFrame - -use anyhow::{anyhow, Result}; -use log::{debug, info, warn}; -use std::sync::mpsc; -use std::sync::Arc; -use std::thread; -use tokio::sync::mpsc as tokio_mpsc; - -#[cfg(target_os = "windows")] -use std::path::Path; - -use super::{ColorRange, ColorSpace, PixelFormat, TransferFunction, VideoFrame}; -use crate::app::{config::VideoDecoderBackend, SharedFrame, VideoCodec}; - -// FFmpeg imports - only for macOS -#[cfg(target_os = "macos")] -extern crate ffmpeg_next as ffmpeg; - -#[cfg(target_os = "macos")] -use ffmpeg::codec::{context::Context as CodecContext, decoder}; -#[cfg(target_os = "macos")] -use ffmpeg::format::Pixel; -#[cfg(target_os = "macos")] -use ffmpeg::software::scaling::{context::Context as ScalerContext, flag::Flags as ScalerFlags}; -#[cfg(target_os = "macos")] -use ffmpeg::util::frame::video::Video as FfmpegFrame; -#[cfg(target_os = "macos")] -use ffmpeg::Packet; - -/// GPU Vendor for decoder optimization -#[derive(Debug, PartialEq, Clone, Copy)] -pub enum GpuVendor { - Nvidia, - Intel, - Amd, - Apple, - Broadcom, // Raspberry Pi VideoCore - Other, - Unknown, -} - -/// Cached GPU vendor -static GPU_VENDOR: std::sync::OnceLock = std::sync::OnceLock::new(); - -/// Detect the primary GPU vendor using wgpu, prioritizing discrete GPUs -pub fn detect_gpu_vendor() -> GpuVendor { - *GPU_VENDOR.get_or_init(|| { - // blocked_on because we are in a sync context (VideoDecoder::new) - // but wgpu adapter request is async - pollster::block_on(async { - let instance = wgpu::Instance::new(&wgpu::InstanceDescriptor::default()); // Needs borrow - - // Enumerate all available adapters (wgpu 28 returns a Future) - let adapters = instance.enumerate_adapters(wgpu::Backends::all()).await; - - let mut best_score = -1; - let mut best_vendor = GpuVendor::Unknown; - - info!("Available GPU adapters:"); - - for adapter in adapters { - let info = adapter.get_info(); - let name = info.name.to_lowercase(); - let mut score = 0; - let mut vendor = GpuVendor::Other; - - // Identify vendor - if name.contains("nvidia") || name.contains("geforce") || name.contains("quadro") { - vendor = GpuVendor::Nvidia; - score += 100; - } else if name.contains("amd") || name.contains("adeon") || name.contains("ryzen") { - vendor = GpuVendor::Amd; - score += 80; - } else if name.contains("intel") - || name.contains("uhd") - || name.contains("iris") - || name.contains("arc") - { - vendor = GpuVendor::Intel; - score += 50; - } else if name.contains("apple") - || name.contains("m1") - || name.contains("m2") - || name.contains("m3") - { - vendor = GpuVendor::Apple; - score += 90; // Apple Silicon is high perf - } else if name.contains("videocore") - || name.contains("broadcom") - || name.contains("v3d") - || name.contains("vc4") - { - vendor = GpuVendor::Broadcom; - score += 30; // Raspberry Pi - low power device - } - - // Prioritize discrete GPUs - match info.device_type { - wgpu::DeviceType::DiscreteGpu => { - score += 50; - } - wgpu::DeviceType::IntegratedGpu => { - score += 10; - } - _ => {} - } - - info!( - " - {} ({:?}, Vendor: {:?}, Score: {})", - info.name, info.device_type, vendor, score - ); - - if score > best_score { - best_score = score; - best_vendor = vendor; - } - } - - if best_vendor != GpuVendor::Unknown { - info!("Selected best GPU vendor: {:?}", best_vendor); - best_vendor - } else { - // Fallback to default request if enumeration fails - warn!("Adapter enumeration yielded no results, trying default request"); - - let adapter_result = instance - .request_adapter(&wgpu::RequestAdapterOptions { - power_preference: wgpu::PowerPreference::HighPerformance, - compatible_surface: None, - force_fallback_adapter: false, - }) - .await; - - // Handle Result - if let Ok(adapter) = adapter_result { - let info = adapter.get_info(); - let name = info.name.to_lowercase(); - - if name.contains("nvidia") { - GpuVendor::Nvidia - } else if name.contains("intel") { - GpuVendor::Intel - } else if name.contains("amd") { - GpuVendor::Amd - } else if name.contains("apple") { - GpuVendor::Apple - } else if name.contains("videocore") - || name.contains("broadcom") - || name.contains("v3d") - { - GpuVendor::Broadcom - } else { - GpuVendor::Other - } - } else { - GpuVendor::Unknown - } - } - }) - }) -} - -/// Check if Intel QSV runtime is available on the system -/// Returns true if the required DLLs are found -#[cfg(target_os = "windows")] -fn is_qsv_runtime_available() -> bool { - use std::env; - - // Intel Media SDK / oneVPL runtime DLLs to look for - let runtime_dlls = [ - "libmfx-gen.dll", // Intel oneVPL runtime (11th gen+, newer) - "libmfxhw64.dll", // Intel Media SDK runtime (older) - "mfxhw64.dll", // Alternative naming - "libmfx64.dll", // Another variant - ]; - - // Check common paths where Intel runtimes are installed - let search_paths: Vec = vec![ - // System32 (most common for driver-installed runtimes) - env::var("SystemRoot") - .map(|s| Path::new(&s).join("System32")) - .unwrap_or_default(), - // SysWOW64 for 32-bit - env::var("SystemRoot") - .map(|s| Path::new(&s).join("SysWOW64")) - .unwrap_or_default(), - // Intel Media SDK default install - Path::new( - "C:\\Program Files\\Intel\\Media SDK 2023 R1\\Software Development Kit\\bin\\x64", - ) - .to_path_buf(), - Path::new("C:\\Program Files\\Intel\\Media SDK\\bin\\x64").to_path_buf(), - // oneVPL default install - Path::new("C:\\Program Files (x86)\\Intel\\oneAPI\\vpl\\latest\\bin").to_path_buf(), - // Application directory (for bundled DLLs) - env::current_exe() - .ok() - .and_then(|p| p.parent().map(|p| p.to_path_buf())) - .unwrap_or_default(), - ]; - - for dll in &runtime_dlls { - for path in &search_paths { - let full_path = path.join(dll); - if full_path.exists() { - info!("Found Intel QSV runtime: {}", full_path.display()); - return true; - } - } - } - - // Also try loading via Windows DLL search path - // If Intel drivers are installed, the DLLs should be in PATH - if let Ok(output) = std::process::Command::new("where") - .arg("libmfx-gen.dll") - .output() - { - if output.status.success() { - let path = String::from_utf8_lossy(&output.stdout); - info!("Found Intel QSV runtime via PATH: {}", path.trim()); - return true; - } - } - - debug!("Intel QSV runtime not found - QSV decoder will be skipped"); - false -} - -#[cfg(not(target_os = "windows"))] -fn is_qsv_runtime_available() -> bool { - // On Linux, check for libmfx.so or libvpl.so - use std::process::Command; - - // QSV is only supported on Intel architectures - if !cfg!(target_arch = "x86") && !cfg!(target_arch = "x86_64") { - return false; - } - - if let Ok(output) = Command::new("ldconfig").arg("-p").output() { - let libs = String::from_utf8_lossy(&output.stdout); - if libs.contains("libmfx") || libs.contains("libvpl") { - info!("Found Intel QSV runtime on Linux"); - return true; - } - } - - debug!("Intel QSV runtime not found on Linux"); - false -} - -/// Cached QSV availability check (only check once at startup) -static QSV_AVAILABLE: std::sync::OnceLock = std::sync::OnceLock::new(); - -fn check_qsv_available() -> bool { - *QSV_AVAILABLE.get_or_init(|| { - let available = is_qsv_runtime_available(); - if available { - info!("Intel QuickSync Video (QSV) runtime detected - QSV decoding enabled"); - } else { - info!("Intel QSV runtime not detected - QSV decoding disabled (install Intel GPU drivers for QSV support)"); - } - available - }) -} - -/// Cached Intel GPU name for QSV capability detection -static INTEL_GPU_NAME: std::sync::OnceLock = std::sync::OnceLock::new(); - -/// Get the Intel GPU name from wgpu adapter info -fn get_intel_gpu_name() -> String { - INTEL_GPU_NAME - .get_or_init(|| { - pollster::block_on(async { - let instance = wgpu::Instance::new(&wgpu::InstanceDescriptor::default()); - let adapters = instance.enumerate_adapters(wgpu::Backends::all()).await; - - for adapter in adapters { - let info = adapter.get_info(); - let name = info.name.to_lowercase(); - if name.contains("intel") { - return info.name.clone(); - } - } - String::new() - }) - }) - .clone() -} - -/// Check if the Intel GPU supports QSV decoding for the given codec (macOS only) -/// Older Intel GPUs have limited QSV codec support: -/// - Gen 7 (Ivy Bridge/HD 4000, 2012): Only H.264 -/// - Gen 8 (Haswell, 2013): H.264 + limited HEVC -/// - Gen 9 (Skylake, 2015+): H.264 + HEVC -#[cfg(target_os = "macos")] -fn is_qsv_supported_for_codec(codec_id: ffmpeg::codec::Id) -> bool { - // First check if QSV runtime is even available - if !check_qsv_available() { - return false; - } - - let gpu_name = get_intel_gpu_name(); - let gpu_lower = gpu_name.to_lowercase(); - - // Detect older Intel GPU generations that have limited QSV support - let is_gen7_or_older = gpu_lower.contains("hd graphics 4000") - || gpu_lower.contains("hd 4000") - || gpu_lower.contains("hd graphics 2500") - || gpu_lower.contains("hd 2500") - || gpu_lower.contains("ivy bridge") - || gpu_lower.contains("sandy bridge") - || gpu_lower.contains("hd graphics 3000") - || gpu_lower.contains("hd 3000"); - - match codec_id { - ffmpeg::codec::Id::H264 => { - // H.264 supported on all Intel QSV generations - true - } - ffmpeg::codec::Id::HEVC => { - // HEVC not supported on Gen 7 (Ivy Bridge) and older - if is_gen7_or_older { - info!("Intel GPU '{}' (Gen 7 or older) does not support HEVC QSV - using software decoder", gpu_name); - return false; - } - // Gen 8 has limited HEVC support (decode only, 8-bit only) - true - } - _ => true, // Unknown codecs - try QSV - } -} - -/// Cached supported decoder backends -static SUPPORTED_BACKENDS: std::sync::OnceLock> = - std::sync::OnceLock::new(); - -/// Get list of supported decoder backends for the current system -pub fn get_supported_decoder_backends() -> Vec { - SUPPORTED_BACKENDS - .get_or_init(|| { - let mut backends = vec![VideoDecoderBackend::Auto]; - - // Always check what's actually available - #[cfg(target_os = "macos")] - { - backends.push(VideoDecoderBackend::VideoToolbox); - } - - #[cfg(target_os = "windows")] - { - let gpu = detect_gpu_vendor(); - let qsv = check_qsv_available(); - - // GStreamer D3D11 decoder - supports both H.264 and HEVC - // This is the recommended decoder for Windows (stable, works on all GPUs) - backends.push(VideoDecoderBackend::Dxva); - - // Native D3D11VA decoder (HEVC only) - EXPERIMENTAL - // Uses direct D3D11 Video API for zero-copy decoding - // WARNING: Only supports HEVC. H.264 streams will fail! - backends.push(VideoDecoderBackend::NativeDxva); - - // GPU-specific accelerators - if gpu == GpuVendor::Nvidia { - backends.push(VideoDecoderBackend::Cuvid); - } - - if qsv || gpu == GpuVendor::Intel { - backends.push(VideoDecoderBackend::Qsv); - } - } - - #[cfg(target_os = "linux")] - { - let gpu = detect_gpu_vendor(); - let qsv = check_qsv_available(); - - // GStreamer with hardware acceleration is the preferred decoder on Linux - // It automatically selects the best available hardware decoder (VAAPI, NVDEC, etc.) - if super::gstreamer_decoder::is_gstreamer_available() { - backends.push(VideoDecoderBackend::VulkanVideo); // GStreamer-based hardware decode - } - - if gpu == GpuVendor::Nvidia { - backends.push(VideoDecoderBackend::Cuvid); - } - - if qsv || gpu == GpuVendor::Intel { - backends.push(VideoDecoderBackend::Qsv); - } - - // VAAPI is generally available on Linux (AMD/Intel) - backends.push(VideoDecoderBackend::Vaapi); - } - - backends.push(VideoDecoderBackend::Software); - backends - }) - .clone() -} - -/// Commands sent to the decoder thread -enum DecoderCommand { - /// Decode a packet and return result via channel (blocking mode) - Decode(Vec), - /// Decode a packet and write directly to SharedFrame (non-blocking mode) - DecodeAsync { - data: Vec, - receive_time: std::time::Instant, - }, - Stop, -} - -/// Stats from the decoder thread -#[derive(Debug, Clone)] -pub struct DecodeStats { - /// Time from packet receive to decode complete (ms) - pub decode_time_ms: f32, - /// Whether a frame was produced - pub frame_produced: bool, - /// Whether a keyframe is needed (too many consecutive decode failures) - pub needs_keyframe: bool, -} - -/// Video decoder using FFmpeg with hardware acceleration -/// Uses a dedicated thread for decoding since FFmpeg types are not Send -pub struct VideoDecoder { - cmd_tx: mpsc::Sender, - frame_rx: mpsc::Receiver>, - /// Stats receiver for non-blocking mode - stats_rx: Option>, - hw_accel: bool, - frames_decoded: u64, - /// SharedFrame for non-blocking writes (set via set_shared_frame) - shared_frame: Option>, -} - -impl VideoDecoder { - /// Create a new video decoder with hardware acceleration - /// Note: On Linux, use new_async() instead - Linux uses native Vulkan Video decoder - #[cfg(target_os = "macos")] - pub fn new(codec: VideoCodec, backend: VideoDecoderBackend) -> Result { - // Initialize FFmpeg - ffmpeg::init().map_err(|e| anyhow!("Failed to initialize FFmpeg: {:?}", e))?; - - // Suppress FFmpeg's "no frame" info messages (EAGAIN is normal for H.264) - unsafe { - ffmpeg::ffi::av_log_set_level(ffmpeg::ffi::AV_LOG_ERROR as i32); - } - - info!( - "Creating FFmpeg video decoder for {:?} (backend: {:?})", - codec, backend - ); - - // Find the decoder - let decoder_id = match codec { - VideoCodec::H264 => ffmpeg::codec::Id::H264, - VideoCodec::H265 => ffmpeg::codec::Id::HEVC, - VideoCodec::AV1 => ffmpeg::codec::Id::AV1, - }; - - // Create channels for communication with decoder thread - let (cmd_tx, cmd_rx) = mpsc::channel::(); - let (frame_tx, frame_rx) = mpsc::channel::>(); - - // Create decoder in a separate thread (FFmpeg types are not Send) - let hw_accel = - Self::spawn_decoder_thread(decoder_id, cmd_rx, frame_tx, None, None, backend)?; - - if hw_accel { - info!("Using hardware-accelerated decoder"); - } else { - info!("Using software decoder (hardware acceleration not available)"); - } - - Ok(Self { - cmd_tx, - frame_rx, - stats_rx: None, - hw_accel, - frames_decoded: 0, - shared_frame: None, - }) - } - - /// Create a new video decoder configured for non-blocking async mode - /// Decoded frames are written directly to the SharedFrame - pub fn new_async( - codec: VideoCodec, - backend: VideoDecoderBackend, - shared_frame: Arc, - ) -> Result<(Self, tokio_mpsc::Receiver)> { - // On Windows, use native DXVA decoder (no FFmpeg) - // This uses D3D11 Video API directly for hardware acceleration - #[cfg(target_os = "windows")] - { - return Err(anyhow!( - "VideoDecoder::new_async not supported on Windows. Use UnifiedVideoDecoder::new_async instead." - )); - } - - // On Linux, use GStreamer for hardware-accelerated decoding - // GStreamer automatically selects the best available backend (VAAPI, NVDEC, V4L2, etc.) - #[cfg(target_os = "linux")] - { - // Use GStreamer decoder (auto-selects V4L2/VAAPI/NVDEC/software) - // The GStreamer decoder automatically selects the best available backend - if super::gstreamer_decoder::is_gstreamer_available() { - info!( - "Using GStreamer decoder for {:?} (auto-selects V4L2/VA/VAAPI/software)", - codec - ); - - let gst_codec = match codec { - VideoCodec::H264 => super::gstreamer_decoder::GstCodec::H264, - VideoCodec::H265 => super::gstreamer_decoder::GstCodec::H265, - VideoCodec::AV1 => super::gstreamer_decoder::GstCodec::AV1, - }; - - let config = super::gstreamer_decoder::GstDecoderConfig { - codec: gst_codec, - width: 1920, - height: 1080, - low_latency: true, // Enable low latency for streaming - }; - - let gst_decoder = super::gstreamer_decoder::GStreamerDecoder::new(config) - .map_err(|e| anyhow!("Failed to create GStreamer decoder: {}", e))?; - - info!("GStreamer decoder created successfully!"); - - let (cmd_tx, cmd_rx) = mpsc::channel::(); - let (frame_tx, frame_rx) = mpsc::channel::>(); - let (stats_tx, stats_rx) = tokio_mpsc::channel::(64); - - let shared_frame_clone = shared_frame.clone(); - - thread::spawn(move || { - info!("GStreamer decoder thread started"); - let mut decoder = gst_decoder; - let mut frames_decoded = 0u64; - let mut consecutive_failures = 0u32; - const KEYFRAME_REQUEST_THRESHOLD: u32 = 3; // Lowered from 10 for faster recovery - const FRAMES_TO_SKIP: u64 = 5; - - while let Ok(cmd) = cmd_rx.recv() { - match cmd { - DecoderCommand::Decode(data) => { - let result = decoder.decode(&data); - let _ = frame_tx.send(result.ok().flatten()); - } - DecoderCommand::DecodeAsync { data, receive_time } => { - let result = decoder.decode(&data); - let decode_time_ms = receive_time.elapsed().as_secs_f32() * 1000.0; - - let frame_produced = matches!(&result, Ok(Some(_))); - - let needs_keyframe = if frame_produced { - consecutive_failures = 0; - false - } else { - consecutive_failures += 1; - consecutive_failures == KEYFRAME_REQUEST_THRESHOLD - }; - - if let Ok(Some(frame)) = result { - frames_decoded += 1; - if frames_decoded > FRAMES_TO_SKIP { - shared_frame_clone.write(frame); - } - } - - let _ = stats_tx.try_send(DecodeStats { - decode_time_ms, - frame_produced, - needs_keyframe, - }); - } - DecoderCommand::Stop => break, - } - } - info!("GStreamer decoder thread stopped"); - }); - - let decoder = Self { - cmd_tx, - frame_rx, - stats_rx: None, - hw_accel: true, - frames_decoded: 0, - shared_frame: Some(shared_frame), - }; - - return Ok((decoder, stats_rx)); - } - - // No decoder available - return Err(anyhow!( - "No video decoder available on Linux. Requires either:\n\ - - Vulkan Video support (Intel Arc, NVIDIA RTX, AMD RDNA2+)\n\ - - GStreamer with hardware decoding:\n\ - * V4L2 (Raspberry Pi / embedded)\n\ - * VA plugin (Intel/AMD desktop - vah264dec)\n\ - * VAAPI plugin (legacy Intel/AMD - vaapih264dec)\n\ - * Software fallback (avdec_h264)\n\ - Run 'vulkaninfo | grep video' to check Vulkan Video support.\n\ - Run 'gst-inspect-1.0 | grep -E \"v4l2|va|vaapi|avdec\"' to check GStreamer decoders." - )); - } - - // FFmpeg path (Windows/macOS only) - #[cfg(target_os = "macos")] - { - // Initialize FFmpeg - ffmpeg::init().map_err(|e| anyhow!("Failed to initialize FFmpeg: {:?}", e))?; - - // Suppress FFmpeg's "no frame" info messages (EAGAIN is normal for H.264) - unsafe { - ffmpeg::ffi::av_log_set_level(ffmpeg::ffi::AV_LOG_ERROR as i32); - } - - info!( - "Creating FFmpeg video decoder (async mode) for {:?} (backend: {:?})", - codec, backend - ); - - // Find the decoder - let decoder_id = match codec { - VideoCodec::H264 => ffmpeg::codec::Id::H264, - VideoCodec::H265 => ffmpeg::codec::Id::HEVC, - VideoCodec::AV1 => ffmpeg::codec::Id::AV1, - }; - - // Create channels for communication with decoder thread - let (cmd_tx, cmd_rx) = mpsc::channel::(); - let (frame_tx, frame_rx) = mpsc::channel::>(); - - // Stats channel for async mode (non-blocking stats updates) - let (stats_tx, stats_rx) = tokio_mpsc::channel::(64); - - // Create decoder in a separate thread with SharedFrame - let hw_accel = Self::spawn_decoder_thread( - decoder_id, - cmd_rx, - frame_tx, - Some(shared_frame.clone()), - Some(stats_tx), - backend, - )?; - - if hw_accel { - info!("Using hardware-accelerated decoder (async mode)"); - } else { - info!("Using software decoder (async mode)"); - } - - let decoder = Self { - cmd_tx, - frame_rx, - stats_rx: None, // Stats come via the returned receiver - hw_accel, - frames_decoded: 0, - shared_frame: Some(shared_frame), - }; - - Ok((decoder, stats_rx)) - } // end #[cfg(target_os = "macos")] - } - - /// Spawn a dedicated decoder thread (FFmpeg-based, not used on Linux) - #[cfg(target_os = "macos")] - fn spawn_decoder_thread( - codec_id: ffmpeg::codec::Id, - cmd_rx: mpsc::Receiver, - frame_tx: mpsc::Sender>, - shared_frame: Option>, - stats_tx: Option>, - backend: VideoDecoderBackend, - ) -> Result { - // Create decoder synchronously to report hw_accel status - info!("Creating decoder for codec {:?}...", codec_id); - let (decoder, hw_accel) = Self::create_decoder(codec_id, backend)?; - info!("Decoder created, hw_accel={}", hw_accel); - - // Spawn thread to handle decoding - thread::spawn(move || { - info!("Decoder thread started for {:?}", codec_id); - let mut decoder = decoder; - let mut scaler: Option = None; - let mut width = 0u32; - let mut height = 0u32; - let mut frames_decoded = 0u64; - let mut consecutive_failures = 0u32; - let mut packets_received = 0u64; - const KEYFRAME_REQUEST_THRESHOLD: u32 = 3; // Lowered from 10 for faster recovery after focus loss - const FRAMES_TO_SKIP: u64 = 5; // Skip first N frames to let decoder settle with reference frames - - while let Ok(cmd) = cmd_rx.recv() { - match cmd { - DecoderCommand::Decode(data) => { - // Blocking mode - send result back via channel - let result = Self::decode_frame( - &mut decoder, - &mut scaler, - &mut width, - &mut height, - &mut frames_decoded, - &data, - codec_id, - false, // No recovery tracking for blocking mode - ); - let _ = frame_tx.send(result); - } - DecoderCommand::DecodeAsync { data, receive_time } => { - packets_received += 1; - - // Check if we're in recovery mode (waiting for keyframe) - let in_recovery = consecutive_failures >= KEYFRAME_REQUEST_THRESHOLD; - - // Non-blocking mode - write directly to SharedFrame - let result = Self::decode_frame( - &mut decoder, - &mut scaler, - &mut width, - &mut height, - &mut frames_decoded, - &data, - codec_id, - in_recovery, - ); - - let decode_time_ms = receive_time.elapsed().as_secs_f32() * 1000.0; - let frame_produced = result.is_some(); - - // Track consecutive decode failures for PLI request - // Note: EAGAIN (no frame) is normal for H.264 - decoder buffers B-frames - let needs_keyframe = if frame_produced { - // Only log recovery for significant failures (>5), not normal buffering - if consecutive_failures > 5 { - info!( - "Decoder: recovered after {} packets without output", - consecutive_failures - ); - } - consecutive_failures = 0; - false - } else { - consecutive_failures += 1; - - // Only log at higher thresholds - low counts are normal H.264 buffering - if consecutive_failures == 30 { - debug!( - "Decoder: {} packets without frame (packets: {}, decoded: {})", - consecutive_failures, packets_received, frames_decoded - ); - } - - if consecutive_failures == KEYFRAME_REQUEST_THRESHOLD { - warn!("Decoder: {} consecutive frames without output - requesting keyframe (packets: {}, decoded: {})", - consecutive_failures, packets_received, frames_decoded); - true - } else if consecutive_failures > KEYFRAME_REQUEST_THRESHOLD - && consecutive_failures % 20 == 0 - { - // Keep requesting every 20 frames if still failing (~166ms at 120fps) - warn!("Decoder: still failing after {} frames - requesting keyframe again", consecutive_failures); - true - } else { - false - } - }; - - // Write frame directly to SharedFrame (zero-copy handoff) - // Skip first few frames to let decoder settle with proper reference frames - // This prevents green/corrupted frames during stream startup - if let Some(frame) = result { - if frames_decoded > FRAMES_TO_SKIP { - if let Some(ref sf) = shared_frame { - sf.write(frame); - } - } else { - debug!( - "Skipping frame {} (waiting for decoder to settle)", - frames_decoded - ); - } - } - - // Send stats update (non-blocking) - if let Some(ref tx) = stats_tx { - let _ = tx.try_send(DecodeStats { - decode_time_ms, - frame_produced, - needs_keyframe, - }); - } - } - DecoderCommand::Stop => break, - } - } - }); - - Ok(hw_accel) - } - - /// FFI Callback for format negotiation (VideoToolbox) - #[cfg(target_os = "macos")] - unsafe extern "C" fn get_videotoolbox_format( - _ctx: *mut ffmpeg::ffi::AVCodecContext, - mut fmt: *const ffmpeg::ffi::AVPixelFormat, - ) -> ffmpeg::ffi::AVPixelFormat { - use ffmpeg::ffi::*; - - // Log all available formats for debugging - let mut available_formats = Vec::new(); - let mut check_fmt = fmt; - while *check_fmt != AVPixelFormat::AV_PIX_FMT_NONE { - available_formats.push(*check_fmt as i32); - check_fmt = check_fmt.add(1); - } - info!( - "get_format callback: available formats: {:?} (VIDEOTOOLBOX={}, NV12={}, YUV420P={})", - available_formats, - AVPixelFormat::AV_PIX_FMT_VIDEOTOOLBOX as i32, - AVPixelFormat::AV_PIX_FMT_NV12 as i32, - AVPixelFormat::AV_PIX_FMT_YUV420P as i32 - ); - - while *fmt != AVPixelFormat::AV_PIX_FMT_NONE { - if *fmt == AVPixelFormat::AV_PIX_FMT_VIDEOTOOLBOX { - info!("get_format: selecting VIDEOTOOLBOX hardware format"); - return AVPixelFormat::AV_PIX_FMT_VIDEOTOOLBOX; - } - fmt = fmt.add(1); - } - - info!("get_format: VIDEOTOOLBOX not available, falling back to NV12"); - AVPixelFormat::AV_PIX_FMT_NV12 - } - - /// FFI Callback for D3D11VA format negotiation (works on all Windows GPUs including NVIDIA) - /// This produces D3D11 textures that can be shared with wgpu via DXGI handles - /// - /// CRITICAL: This callback must set up hw_frames_ctx for D3D11VA to work! - /// Based on NVIDIA GeForce NOW client's DXVADecoder implementation. - /// - /// Key insight: NVIDIA drivers are strict about texture dimensions. - /// - `coded_width/height` includes encoder padding (e.g., 3840x2176) - /// - Actual video dimensions are `width/height` (e.g., 3840x2160) - /// - We must use dimensions that are multiples of the codec's macroblock size - /// - For HEVC: 32x32 CTU alignment, for H.264: 16x16 MB alignment - /// - /// Note: This is only used on macOS where FFmpeg is used for video decoding. - /// Windows uses native DXVA, Linux uses Vulkan Video. - #[cfg(target_os = "macos")] - #[allow(dead_code)] - unsafe extern "C" fn get_d3d11va_format( - ctx: *mut ffmpeg::ffi::AVCodecContext, - fmt: *const ffmpeg::ffi::AVPixelFormat, - ) -> ffmpeg::ffi::AVPixelFormat { - use ffmpeg::ffi::*; - - // Log all available formats for debugging - let mut available_formats = Vec::new(); - let mut check_fmt = fmt; - while *check_fmt != AVPixelFormat::AV_PIX_FMT_NONE { - available_formats.push(*check_fmt as i32); - check_fmt = check_fmt.add(1); - } - info!( - "get_d3d11va_format: available formats: {:?} (D3D11={}, NV12={}, P010={})", - available_formats, - AVPixelFormat::AV_PIX_FMT_D3D11 as i32, - AVPixelFormat::AV_PIX_FMT_NV12 as i32, - AVPixelFormat::AV_PIX_FMT_P010LE as i32 - ); - - // Check if D3D11 format is available - let mut has_d3d11 = false; - check_fmt = fmt; - while *check_fmt != AVPixelFormat::AV_PIX_FMT_NONE { - if *check_fmt == AVPixelFormat::AV_PIX_FMT_D3D11 { - has_d3d11 = true; - break; - } - check_fmt = check_fmt.add(1); - } - - if !has_d3d11 { - warn!("get_d3d11va_format: D3D11 not in available formats list"); - return *fmt; - } - - // We need hw_device_ctx to create hw_frames_ctx - if (*ctx).hw_device_ctx.is_null() { - warn!("get_d3d11va_format: hw_device_ctx is null, cannot use D3D11VA"); - return *fmt; - } - - // Check if hw_frames_ctx already exists (might be called multiple times) - if !(*ctx).hw_frames_ctx.is_null() { - info!("get_d3d11va_format: hw_frames_ctx already set, selecting D3D11"); - return AVPixelFormat::AV_PIX_FMT_D3D11; - } - - // Determine sw_format based on codec and bit depth - // HEVC Main10/AV1 10-bit needs P010, others use NV12 - // Check multiple indicators for 10-bit content - let is_10bit = (*ctx).profile == 2 // HEVC Main10 profile - || (*ctx).pix_fmt == AVPixelFormat::AV_PIX_FMT_YUV420P10LE - || (*ctx).pix_fmt == AVPixelFormat::AV_PIX_FMT_YUV420P10BE - || (*ctx).pix_fmt == AVPixelFormat::AV_PIX_FMT_P010LE - || ((*ctx).codec_id == AVCodecID::AV_CODEC_ID_AV1 && (*ctx).profile >= 1); // AV1 High/Professional profile - - // Calculate proper dimensions for D3D11VA texture - // NVIDIA drivers are strict: they don't like coded_width/height with encoder padding - // - // The issue: coded_width/height includes encoder alignment padding - // - 4K video (3840x2160) may have coded dimensions of 3840x2176 (16 pixels padding for HEVC CTU) - // - This padding causes D3D11VA texture creation to fail on NVIDIA with error 80070057 - // - // Solution: Remove the padding by detecting common video resolutions - // Standard resolutions: 2160p (4K), 1440p, 1080p, 720p, etc. - let coded_w = (*ctx).coded_width; - let coded_h = (*ctx).coded_height; - - // Calculate actual video height by removing encoder padding - // HEVC uses 32-pixel CTU, H.264 uses 16-pixel MB - // Common pattern: encoder adds 16-32 pixels of padding to height - let actual_height = if coded_h > 2160 && coded_h <= 2176 { - 2160 // 4K UHD - } else if coded_h > 1440 && coded_h <= 1472 { - 1440 // QHD - } else if coded_h > 1080 && coded_h <= 1088 { - 1080 // Full HD - } else if coded_h > 720 && coded_h <= 736 { - 720 // HD - } else if coded_h > 480 && coded_h <= 496 { - 480 // SD - } else { - // No standard padding detected, use coded height - coded_h - }; - - // Width is usually already correct (16:9 widths are typically aligned) - let actual_width = coded_w; - - info!( - "get_d3d11va_format: codec={:?}, profile={}, pix_fmt={:?}, coded={}x{}, actual={}x{}, is_10bit={}", - (*ctx).codec_id, - (*ctx).profile, - (*ctx).pix_fmt as i32, - coded_w, - coded_h, - actual_width, - actual_height, - is_10bit - ); - - // Try formats in order of preference - // NVIDIA's DXVADecoder supports: NV12, P010, YUV444, YUV444_10 - let formats_to_try = if is_10bit { - vec![ - AVPixelFormat::AV_PIX_FMT_P010LE, - AVPixelFormat::AV_PIX_FMT_NV12, - ] - } else { - vec![ - AVPixelFormat::AV_PIX_FMT_NV12, - AVPixelFormat::AV_PIX_FMT_P010LE, - ] - }; - - // Try with actual dimensions first (without padding), then fall back to coded dimensions - let dimensions_to_try = [(actual_width, actual_height), (coded_w, coded_h)]; - - for (width, height) in dimensions_to_try { - for sw_format in &formats_to_try { - // Allocate fresh hw_frames_ctx for each attempt - let hw_frames_ref = av_hwframe_ctx_alloc((*ctx).hw_device_ctx); - if hw_frames_ref.is_null() { - warn!("get_d3d11va_format: Failed to allocate hw_frames_ctx"); - continue; - } - - // Configure the frames context - let frames_ctx = (*hw_frames_ref).data as *mut AVHWFramesContext; - (*frames_ctx).format = AVPixelFormat::AV_PIX_FMT_D3D11; - (*frames_ctx).sw_format = *sw_format; - (*frames_ctx).width = width; - (*frames_ctx).height = height; - - // NVIDIA-compatible pool size - // NVIDIA's DXVADecoder uses texture arrays (RTArray) with ~16-20 surfaces - // This matches their "allocating RTArrays" approach - (*frames_ctx).initial_pool_size = 20; - - info!( - "get_d3d11va_format: Trying hw_frames_ctx: {}x{}, sw_format={:?}, pool_size=20", - width, height, *sw_format as i32 - ); - - // Initialize the frames context - let ret = av_hwframe_ctx_init(hw_frames_ref); - if ret >= 0 { - // Success! Attach to codec context - (*ctx).hw_frames_ctx = av_buffer_ref(hw_frames_ref); - av_buffer_unref(&mut (hw_frames_ref as *mut _)); - - let format_name = if *sw_format == AVPixelFormat::AV_PIX_FMT_P010LE { - "P010 (10-bit HDR)" - } else { - "NV12 (8-bit SDR)" - }; - info!( - "get_d3d11va_format: D3D11VA hw_frames_ctx initialized with {} at {}x{} - zero-copy enabled!", - format_name, width, height - ); - return AVPixelFormat::AV_PIX_FMT_D3D11; - } - - // Failed, clean up and try next format - warn!( - "get_d3d11va_format: Failed to init hw_frames_ctx {}x{} with sw_format={:?} (error {})", - width, height, *sw_format as i32, ret - ); - av_buffer_unref(&mut (hw_frames_ref as *mut _)); - } - } - - // All formats failed - warn!("get_d3d11va_format: All D3D11VA formats failed, falling back to software"); - *fmt - } - - /// FFI Callback for CUDA format negotiation (NVIDIA CUVID) - /// CRITICAL: This must set up hw_frames_ctx for proper frame buffer management - /// - /// Note: This is only used on macOS where FFmpeg is used for video decoding. - /// Windows uses native DXVA, Linux uses Vulkan Video. - #[cfg(target_os = "macos")] - #[allow(dead_code)] - unsafe extern "C" fn get_cuda_format( - ctx: *mut ffmpeg::ffi::AVCodecContext, - fmt: *const ffmpeg::ffi::AVPixelFormat, - ) -> ffmpeg::ffi::AVPixelFormat { - use ffmpeg::ffi::*; - - // Check if CUDA format is available - let mut has_cuda = false; - let mut check_fmt = fmt; - while *check_fmt != AVPixelFormat::AV_PIX_FMT_NONE { - if *check_fmt == AVPixelFormat::AV_PIX_FMT_CUDA { - has_cuda = true; - break; - } - check_fmt = check_fmt.add(1); - } - - if !has_cuda { - info!("get_format: CUDA not in available formats, falling back to NV12"); - return AVPixelFormat::AV_PIX_FMT_NV12; - } - - // We need hw_device_ctx to create hw_frames_ctx - if (*ctx).hw_device_ctx.is_null() { - warn!("get_format: hw_device_ctx is null, cannot use CUDA"); - return AVPixelFormat::AV_PIX_FMT_NV12; - } - - // Check if hw_frames_ctx already exists - if !(*ctx).hw_frames_ctx.is_null() { - info!("get_format: hw_frames_ctx already set, selecting CUDA"); - return AVPixelFormat::AV_PIX_FMT_CUDA; - } - - // Allocate hw_frames_ctx from hw_device_ctx - let hw_frames_ref = av_hwframe_ctx_alloc((*ctx).hw_device_ctx); - if hw_frames_ref.is_null() { - warn!("get_format: Failed to allocate hw_frames_ctx for CUDA"); - return AVPixelFormat::AV_PIX_FMT_NV12; - } - - // Configure the frames context - let frames_ctx = (*hw_frames_ref).data as *mut AVHWFramesContext; - (*frames_ctx).format = AVPixelFormat::AV_PIX_FMT_CUDA; - (*frames_ctx).sw_format = AVPixelFormat::AV_PIX_FMT_NV12; // CUVID outputs NV12 as software format - (*frames_ctx).width = (*ctx).coded_width; - (*frames_ctx).height = (*ctx).coded_height; - (*frames_ctx).initial_pool_size = 20; // Larger pool for B-frame reordering - - info!( - "get_format: Configuring CUDA hw_frames_ctx: {}x{}, sw_format=NV12, pool_size=20", - (*ctx).coded_width, - (*ctx).coded_height - ); - - // Initialize the frames context - let ret = av_hwframe_ctx_init(hw_frames_ref); - if ret < 0 { - warn!( - "get_format: Failed to initialize CUDA hw_frames_ctx (error {})", - ret - ); - av_buffer_unref(&mut (hw_frames_ref as *mut _)); - return AVPixelFormat::AV_PIX_FMT_NV12; - } - - // Attach to codec context - (*ctx).hw_frames_ctx = av_buffer_ref(hw_frames_ref); - av_buffer_unref(&mut (hw_frames_ref as *mut _)); - - info!("get_format: CUDA hw_frames_ctx initialized successfully!"); - AVPixelFormat::AV_PIX_FMT_CUDA - } - - // Note: VAAPI/Vulkan FFmpeg format callbacks removed - Linux now uses native Vulkan Video decoder - - /// Create decoder, trying hardware acceleration based on preference - /// (FFmpeg-based, not used on Linux) - #[cfg(target_os = "macos")] - fn create_decoder( - codec_id: ffmpeg::codec::Id, - backend: VideoDecoderBackend, - ) -> Result<(decoder::Video, bool)> { - info!( - "create_decoder: {:?} with backend preference {:?}", - codec_id, backend - ); - - // On macOS, try VideoToolbox hardware acceleration - #[cfg(target_os = "macos")] - { - if backend == VideoDecoderBackend::Auto || backend == VideoDecoderBackend::VideoToolbox - { - info!("macOS detected - attempting VideoToolbox hardware acceleration"); - - // First try to find specific VideoToolbox decoders - let vt_decoder_name = match codec_id { - ffmpeg::codec::Id::AV1 => Some("av1_videotoolbox"), - ffmpeg::codec::Id::HEVC => Some("hevc_videotoolbox"), - ffmpeg::codec::Id::H264 => Some("h264_videotoolbox"), - _ => None, - }; - - if let Some(name) = vt_decoder_name { - if let Some(codec) = ffmpeg::codec::decoder::find_by_name(name) { - info!("Found specific VideoToolbox decoder: {}", name); - - // Try to use explicit decoder with hardware context attached - // This helps ensure we get VIDEOTOOLBOX frames even without set_get_format - let res = unsafe { - use ffmpeg::ffi::*; - use std::ptr; - - let mut ctx = CodecContext::new_with_codec(codec); - - // Create HW device context - let mut hw_device_ctx: *mut AVBufferRef = ptr::null_mut(); - let ret = av_hwdevice_ctx_create( - &mut hw_device_ctx, - AVHWDeviceType::AV_HWDEVICE_TYPE_VIDEOTOOLBOX, - ptr::null(), - ptr::null_mut(), - 0, - ); - - if ret >= 0 && !hw_device_ctx.is_null() { - let raw_ctx = ctx.as_mut_ptr(); - (*raw_ctx).hw_device_ctx = av_buffer_ref(hw_device_ctx); - av_buffer_unref(&mut hw_device_ctx); - - // FORCE VIDEOTOOLBOX FORMAT via callback and simple hint - (*raw_ctx).get_format = Some(Self::get_videotoolbox_format); - (*raw_ctx).pix_fmt = AVPixelFormat::AV_PIX_FMT_VIDEOTOOLBOX; - } - - ctx.set_threading(ffmpeg::codec::threading::Config::count(4)); - ctx.decoder().video() - }; - - match res { - Ok(decoder) => { - info!( - "Specific VideoToolbox decoder ({}) opened successfully", - name - ); - return Ok((decoder, true)); - } - Err(e) => { - warn!( - "Failed to open specific VideoToolbox decoder {}: {:?}", - name, e - ); - } - } - } - } - - // Fallback: Generic decoder with manual hw_device_ctx attachment - // Try to set up VideoToolbox hwaccel using FFmpeg's device API - unsafe { - use ffmpeg::ffi::*; - use std::ptr; - - // Find the standard decoder - let codec = ffmpeg::codec::decoder::find(codec_id) - .ok_or_else(|| anyhow!("Decoder not found for {:?}", codec_id))?; - - let mut ctx = CodecContext::new_with_codec(codec); - - // Get raw pointer to AVCodecContext - let raw_ctx = ctx.as_mut_ptr(); - - // Create VideoToolbox hardware device context - let mut hw_device_ctx: *mut AVBufferRef = ptr::null_mut(); - let ret = av_hwdevice_ctx_create( - &mut hw_device_ctx, - AVHWDeviceType::AV_HWDEVICE_TYPE_VIDEOTOOLBOX, - ptr::null(), - ptr::null_mut(), - 0, - ); - - if ret >= 0 && !hw_device_ctx.is_null() { - // Attach hardware device context to codec context - (*raw_ctx).hw_device_ctx = av_buffer_ref(hw_device_ctx); - av_buffer_unref(&mut hw_device_ctx); - - // CRITICAL: Set get_format callback to request VideoToolbox pixel format - // Without this, the decoder will output software frames (YUV420P) - (*raw_ctx).get_format = Some(Self::get_videotoolbox_format); - - // Use single thread for lowest latency - multi-threading causes frame reordering delays - (*raw_ctx).thread_count = 1; - - // Low latency flags for streaming (same as Windows D3D11VA) - (*raw_ctx).flags |= AV_CODEC_FLAG_LOW_DELAY as i32; - (*raw_ctx).flags2 |= AV_CODEC_FLAG2_FAST as i32; - - match ctx.decoder().video() { - Ok(decoder) => { - info!("VideoToolbox hardware decoder created successfully (generic + hw_device + get_format)"); - return Ok((decoder, true)); - } - Err(e) => { - warn!("Failed to open VideoToolbox decoder: {:?}", e); - } - } - } else { - warn!( - "Failed to create VideoToolbox device context (error {})", - ret - ); - } - } - } else { - info!("VideoToolbox disabled by preference: {:?}", backend); - } - } - - // Platform-specific hardware decoders (Windows/Linux) - #[cfg(not(target_os = "macos"))] - { - // Windows hardware decoder selection - // Priority: D3D11VA (zero-copy, fastest) > CUVID (NVIDIA fallback) > QSV (Intel) - // Based on NVIDIA GeForce NOW client analysis: they use DXVADecoder (D3D11VA) on all GPUs - #[cfg(target_os = "windows")] - if backend != VideoDecoderBackend::Software { - let gpu_vendor = detect_gpu_vendor(); - - // Try D3D11VA first for all GPUs including NVIDIA - // NVIDIA's own GeForce NOW client uses DXVADecoder (D3D11 DXVA2), not CUVID - // D3D11VA provides zero-copy GPU textures which are faster than CUVID's GPU→CPU transfer - // If D3D11VA fails, we fall back to CUVID for NVIDIA or QSV for Intel - let try_d3d11va = - backend == VideoDecoderBackend::Dxva || backend == VideoDecoderBackend::Auto; - - // Try D3D11VA for AMD and Intel GPUs - provides zero-copy texture path - // SKIP D3D11VA on NVIDIA: FFmpeg's D3D11VA implementation has issues with NVIDIA drivers - // NVIDIA GPUs should use CUVID instead (more reliable, better tested) - // The issue is that D3D11VA "opens" successfully but then hw_frames_ctx creation fails - // during actual decoding with error 80070057, and by then it's too late to fall back - let skip_d3d11va_for_nvidia = matches!(gpu_vendor, GpuVendor::Nvidia); - - if try_d3d11va && !skip_d3d11va_for_nvidia { - info!( - "Attempting D3D11VA hardware acceleration (GPU: {:?}) - zero-copy mode", - gpu_vendor - ); - - let codec = ffmpeg::codec::decoder::find(codec_id) - .ok_or_else(|| anyhow!("Decoder not found for {:?}", codec_id)); - - if let Ok(codec) = codec { - // Try multiple D3D11VA initialization approaches - // Approach 1: Let FFmpeg create the D3D11VA device (most compatible) - // Approach 2: Create our own D3D11 device with VIDEO_SUPPORT (fallback) - - // Approach 1: FFmpeg-managed D3D11VA device - // This is more compatible with NVIDIA drivers as FFmpeg handles - // the device creation with proper flags internally - let result = unsafe { - use ffmpeg::ffi::*; - use std::ptr; - - let mut ctx = CodecContext::new_with_codec(codec); - let raw_ctx = ctx.as_mut_ptr(); - - // Let FFmpeg create the D3D11VA device context - // This is more compatible than creating our own device - let mut hw_device_ctx: *mut AVBufferRef = ptr::null_mut(); - let ret = av_hwdevice_ctx_create( - &mut hw_device_ctx, - AVHWDeviceType::AV_HWDEVICE_TYPE_D3D11VA, - ptr::null(), // Use default device - ptr::null_mut(), // No options - 0, - ); - - if ret >= 0 && !hw_device_ctx.is_null() { - info!("D3D11VA hw_device_ctx created by FFmpeg (automatic device)"); - - (*raw_ctx).hw_device_ctx = av_buffer_ref(hw_device_ctx); - av_buffer_unref(&mut hw_device_ctx); - - // Set format callback to select D3D11 pixel format and create hw_frames_ctx - (*raw_ctx).get_format = Some(Self::get_d3d11va_format); - - // Low latency flags for streaming - (*raw_ctx).flags |= AV_CODEC_FLAG_LOW_DELAY as i32; - (*raw_ctx).flags2 |= AV_CODEC_FLAG2_FAST as i32; - (*raw_ctx).thread_count = 1; // Single thread for lowest latency - - ctx.decoder().video() - } else { - warn!("FFmpeg failed to create D3D11VA device context (error {}), trying manual creation...", ret); - Err(ffmpeg::Error::Bug) - } - }; - - match result { - Ok(decoder) => { - info!("D3D11VA hardware decoder opened successfully (FFmpeg device) - zero-copy GPU decoding active!"); - return Ok((decoder, true)); - } - Err(_) => { - // Approach 2: Create our own D3D11 device with VIDEO_SUPPORT flag - // This may work better on some systems - info!("Trying D3D11VA with custom device creation..."); - - let result2 = unsafe { - use ffmpeg::ffi::*; - use windows::core::Interface; - use windows::Win32::Foundation::HMODULE; - use windows::Win32::Graphics::Direct3D::*; - use windows::Win32::Graphics::Direct3D11::*; - - // Create D3D11 device with VIDEO_SUPPORT flag - let mut device: Option = None; - let mut context: Option = None; - let mut feature_level = D3D_FEATURE_LEVEL_11_0; - - let flags = D3D11_CREATE_DEVICE_VIDEO_SUPPORT - | D3D11_CREATE_DEVICE_BGRA_SUPPORT; - - let hr = D3D11CreateDevice( - None, // Default adapter - D3D_DRIVER_TYPE_HARDWARE, - HMODULE::default(), - flags, - Some(&[D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0]), - D3D11_SDK_VERSION, - Some(&mut device), - Some(&mut feature_level), - Some(&mut context), - ); - - if hr.is_err() || device.is_none() { - warn!("Failed to create D3D11 device with video support: {:?}", hr); - return Err(anyhow!("Failed to create D3D11 device")); - } - - let device = device.unwrap(); - info!("Created custom D3D11 device with VIDEO_SUPPORT (feature level: {:?})", feature_level); - - // Enable multithread protection - if let Ok(mt) = device.cast::() { - mt.SetMultithreadProtected(true); - } - - // Allocate hw_device_ctx and configure with our device - let hw_device_ref = av_hwdevice_ctx_alloc( - AVHWDeviceType::AV_HWDEVICE_TYPE_D3D11VA, - ); - if hw_device_ref.is_null() { - warn!("Failed to allocate D3D11VA device context"); - return Err(anyhow!( - "Failed to allocate D3D11VA device context" - )); - } - - // Get the D3D11VA device context and set our device - let hw_device_ctx = - (*hw_device_ref).data as *mut AVHWDeviceContext; - let d3d11_device_hwctx = - (*hw_device_ctx).hwctx as *mut *mut std::ffi::c_void; - - // Set the device pointer (first field of AVD3D11VADeviceContext) - *d3d11_device_hwctx = std::mem::transmute_copy(&device); - std::mem::forget(device); // FFmpeg owns it now - - let ret = av_hwdevice_ctx_init(hw_device_ref); - if ret < 0 { - warn!("Failed to initialize D3D11VA device context (error {})", ret); - av_buffer_unref(&mut (hw_device_ref as *mut _)); - return Err(anyhow!( - "Failed to initialize D3D11VA device context" - )); - } - - info!("D3D11VA hw_device_ctx initialized with custom video device"); - - let mut ctx = CodecContext::new_with_codec(codec); - let raw_ctx = ctx.as_mut_ptr(); - - (*raw_ctx).hw_device_ctx = av_buffer_ref(hw_device_ref); - av_buffer_unref(&mut (hw_device_ref as *mut _)); - - (*raw_ctx).get_format = Some(Self::get_d3d11va_format); - (*raw_ctx).flags |= AV_CODEC_FLAG_LOW_DELAY as i32; - (*raw_ctx).flags2 |= AV_CODEC_FLAG2_FAST as i32; - (*raw_ctx).thread_count = 1; - - ctx.decoder().video() - }; - - match result2 { - Ok(decoder) => { - info!("D3D11VA hardware decoder opened successfully (custom device) - zero-copy GPU decoding active!"); - return Ok((decoder, true)); - } - Err(e) => { - warn!("D3D11VA decoder failed to open: {:?}, trying other backends...", e); - } - } - } - } - } - } - - // Try dedicated hardware decoders (CUVID/QSV) - // CUVID for NVIDIA, QSV for Intel - these are the most reliable options - // Always try these as fallback if D3D11VA failed above - let qsv_available = check_qsv_available(); - - // Don't try NVIDIA CUVID decoders on non-NVIDIA GPUs (causes libnvcuvid load errors) - let is_nvidia = matches!(gpu_vendor, GpuVendor::Nvidia); - let is_intel = matches!(gpu_vendor, GpuVendor::Intel); - - // If user selected DXVA but it failed, still try CUVID/QSV as fallback - let try_hw_fallback = backend != VideoDecoderBackend::Software; - - // Build prioritized list of hardware decoders to try - // Include CUVID/QSV as fallback even if user selected DXVA (since D3D11VA may fail) - let hw_decoders: Vec<&str> = match codec_id { - ffmpeg::codec::Id::H264 => { - let mut list = Vec::new(); - // NVIDIA CUVID (most reliable for NVIDIA, also fallback if DXVA failed) - if is_nvidia && try_hw_fallback { - list.push("h264_cuvid"); - } - // Intel QSV (with codec-specific capability check for older GPUs) - if (is_intel && is_qsv_supported_for_codec(codec_id) && try_hw_fallback) - || backend == VideoDecoderBackend::Qsv - { - list.push("h264_qsv"); - } - // AMD AMF (if available) - if gpu_vendor == GpuVendor::Amd && try_hw_fallback { - list.push("h264_amf"); - } - list - } - ffmpeg::codec::Id::HEVC => { - let mut list = Vec::new(); - // NVIDIA CUVID (most reliable for NVIDIA, also fallback if DXVA failed) - if is_nvidia && try_hw_fallback { - list.push("hevc_cuvid"); - } - // Intel QSV (with codec-specific capability check - HD 4000 doesn't support HEVC) - if (is_intel && is_qsv_supported_for_codec(codec_id) && try_hw_fallback) - || backend == VideoDecoderBackend::Qsv - { - list.push("hevc_qsv"); - } - // AMD AMF (if available) - if gpu_vendor == GpuVendor::Amd && try_hw_fallback { - list.push("hevc_amf"); - } - list - } - _ => vec![], - }; - - info!( - "Hardware decoders to try: {:?} (GPU: {:?}, backend: {:?})", - hw_decoders, gpu_vendor, backend - ); - - // Try each hardware decoder in order - for decoder_name in &hw_decoders { - if let Some(hw_codec) = ffmpeg::codec::decoder::find_by_name(decoder_name) { - info!( - "Found hardware decoder: {}, attempting to open...", - decoder_name - ); - - // For CUVID decoders, we need CUDA device context with proper hw_frames_ctx - if decoder_name.contains("cuvid") { - let result = unsafe { - use ffmpeg::ffi::*; - use std::ptr; - - let mut ctx = CodecContext::new_with_codec(hw_codec); - let raw_ctx = ctx.as_mut_ptr(); - - // Create CUDA device context for CUVID - let mut hw_device_ctx: *mut AVBufferRef = ptr::null_mut(); - let ret = av_hwdevice_ctx_create( - &mut hw_device_ctx, - AVHWDeviceType::AV_HWDEVICE_TYPE_CUDA, - ptr::null(), - ptr::null_mut(), - 0, - ); - - if ret >= 0 && !hw_device_ctx.is_null() { - (*raw_ctx).hw_device_ctx = av_buffer_ref(hw_device_ctx); - av_buffer_unref(&mut hw_device_ctx); - (*raw_ctx).get_format = Some(Self::get_cuda_format); - - // CRITICAL: Set thread_count=1 for CUVID to prevent frame reordering issues - // Multi-threaded decoding can cause frames to arrive out of order, - // leading to visual corruption when reference frames are missing - (*raw_ctx).thread_count = 1; - } else { - warn!("Failed to create CUDA device context (error {}), CUVID may not work", ret); - } - - // Set low latency flags for streaming - (*raw_ctx).flags |= AV_CODEC_FLAG_LOW_DELAY as i32; - (*raw_ctx).flags2 |= AV_CODEC_FLAG2_FAST as i32; - - ctx.decoder().video() - }; - - match result { - Ok(decoder) => { - info!("CUVID hardware decoder ({}) opened successfully - GPU decoding active!", decoder_name); - return Ok((decoder, true)); - } - Err(e) => { - warn!("Failed to open CUVID decoder {}: {:?}", decoder_name, e); - } - } - } else { - // For QSV and other decoders, just open directly - let mut ctx = CodecContext::new_with_codec(hw_codec); - - unsafe { - let raw_ctx = ctx.as_mut_ptr(); - // Set low latency flags - (*raw_ctx).flags |= ffmpeg::ffi::AV_CODEC_FLAG_LOW_DELAY as i32; - (*raw_ctx).flags2 |= ffmpeg::ffi::AV_CODEC_FLAG2_FAST as i32; - } - - match ctx.decoder().video() { - Ok(decoder) => { - info!("Hardware decoder ({}) opened successfully - GPU decoding active!", decoder_name); - return Ok((decoder, true)); - } - Err(e) => { - warn!( - "Failed to open hardware decoder {}: {:?}", - decoder_name, e - ); - } - } - } - } else { - debug!("Hardware decoder not found in FFmpeg: {}", decoder_name); - } - } - - warn!("All hardware decoders failed, will use software decoder"); - } - - // Note: Linux hardware decoder handling uses GStreamer via new_async() instead of FFmpeg. - // See gstreamer_decoder.rs for the implementation. - } - - // Fall back to software decoder - info!("Using software decoder for {:?}", codec_id); - let codec = ffmpeg::codec::decoder::find(codec_id) - .ok_or_else(|| anyhow!("Decoder not found for {:?}", codec_id))?; - info!("Found software decoder: {:?}", codec.name()); - - let mut ctx = CodecContext::new_with_codec(codec); - - // Use fewer threads on low-power devices to reduce memory usage - let gpu_vendor = detect_gpu_vendor(); - let thread_count = if matches!(gpu_vendor, GpuVendor::Broadcom) { - // Raspberry Pi: Use 2 threads to avoid memory overflow - // Pi 5 has 4 cores but limited RAM bandwidth - info!("Raspberry Pi detected: Using 2 decoder threads to conserve memory"); - 2 - } else { - // Desktop/laptop: Use 4 threads for better performance - 4 - }; - ctx.set_threading(ffmpeg::codec::threading::Config::count(thread_count)); - - let decoder = ctx.decoder().video()?; - info!( - "Software decoder opened successfully with {} threads", - thread_count - ); - Ok((decoder, false)) - } - - /// Check if a pixel format is a hardware format (FFmpeg, not used on Linux) - #[cfg(target_os = "macos")] - fn is_hw_pixel_format(format: Pixel) -> bool { - // Check common hardware formats - // Note: Some formats may not be available depending on FFmpeg build configuration - if matches!( - format, - Pixel::VIDEOTOOLBOX - | Pixel::CUDA - | Pixel::VDPAU - | Pixel::QSV - | Pixel::D3D11 - | Pixel::DXVA2_VLD - | Pixel::D3D11VA_VLD - | Pixel::VULKAN - ) { - return true; - } - - // VAAPI/Vulkan check by name since these may not exist in all ffmpeg-next builds - let format_name = format!("{:?}", format); - format_name.contains("VAAPI") || format_name.contains("VULKAN") - } - - /// Transfer hardware frame to system memory if needed (FFmpeg, not used on Linux) - #[cfg(target_os = "macos")] - fn transfer_hw_frame_if_needed(frame: &FfmpegFrame) -> Option { - let format = frame.format(); - - if !Self::is_hw_pixel_format(format) { - // Not a hardware frame, no transfer needed - return None; - } - - unsafe { - use ffmpeg::ffi::*; - - // Create a new frame for the software copy - let sw_frame_ptr = av_frame_alloc(); - if sw_frame_ptr.is_null() { - warn!("Failed to allocate software frame"); - return None; - } - - // Transfer data from hardware frame to software frame - // This is the main latency source - GPU to CPU copy - let ret = av_hwframe_transfer_data(sw_frame_ptr, frame.as_ptr(), 0); - if ret < 0 { - warn!( - "Failed to transfer hardware frame to software (error {})", - ret - ); - av_frame_free(&mut (sw_frame_ptr as *mut _)); - return None; - } - - // Copy frame properties - (*sw_frame_ptr).width = frame.width() as i32; - (*sw_frame_ptr).height = frame.height() as i32; - - // Wrap in FFmpeg frame type - Some(FfmpegFrame::wrap(sw_frame_ptr)) - } - } - - /// Calculate 256-byte aligned stride for GPU compatibility (wgpu/DX12 requirement) - #[cfg(target_os = "macos")] - fn get_aligned_stride(width: u32) -> u32 { - (width + 255) & !255 - } - - /// Decode a single frame (called in decoder thread) (FFmpeg, not used on Linux) - /// `in_recovery` suppresses repeated warnings when waiting for keyframe - #[cfg(target_os = "macos")] - fn decode_frame( - decoder: &mut decoder::Video, - scaler: &mut Option, - width: &mut u32, - height: &mut u32, - frames_decoded: &mut u64, - data: &[u8], - codec_id: ffmpeg::codec::Id, - in_recovery: bool, - ) -> Option { - // AV1 uses OBUs directly, no start codes needed - // H.264/H.265 need Annex B start codes (0x00 0x00 0x00 0x01) - let data = if codec_id == ffmpeg::codec::Id::AV1 { - // AV1 - use data as-is (OBU format) - data.to_vec() - } else if data.len() >= 4 && data[0..4] == [0, 0, 0, 1] { - data.to_vec() - } else if data.len() >= 3 && data[0..3] == [0, 0, 1] { - data.to_vec() - } else { - // Add start code for H.264/H.265 - let mut with_start = vec![0, 0, 0, 1]; - with_start.extend_from_slice(data); - with_start - }; - - // Create packet - let mut packet = Packet::new(data.len()); - if let Some(pkt_data) = packet.data_mut() { - pkt_data.copy_from_slice(&data); - } else { - warn!("Failed to allocate packet data"); - return None; - } - - // Send packet to decoder - if let Err(e) = decoder.send_packet(&packet) { - // EAGAIN means we need to receive frames first - match e { - ffmpeg::Error::Other { errno } if errno == libc::EAGAIN => {} - _ => { - // Suppress repeated warnings during keyframe recovery - if in_recovery { - debug!("Send packet error (waiting for keyframe): {:?}", e); - } else { - warn!("Send packet error: {:?}", e); - } - } - } - } - - // Try to receive decoded frame - let mut frame = FfmpegFrame::empty(); - match decoder.receive_frame(&mut frame) { - Ok(_) => { - *frames_decoded += 1; - - let w = frame.width(); - let h = frame.height(); - let format = frame.format(); - - // Extract color metadata from original frame - let color_range = match frame.color_range() { - ffmpeg::util::color::range::Range::JPEG => ColorRange::Full, - ffmpeg::util::color::range::Range::MPEG => ColorRange::Limited, - _ => ColorRange::Limited, - }; - - let color_space = match frame.color_space() { - ffmpeg::util::color::space::Space::BT709 => ColorSpace::BT709, - ffmpeg::util::color::space::Space::BT470BG => ColorSpace::BT601, - ffmpeg::util::color::space::Space::SMPTE170M => ColorSpace::BT601, - ffmpeg::util::color::space::Space::BT2020NCL => ColorSpace::BT2020, - _ => ColorSpace::BT709, - }; - - // Detect transfer function (SDR gamma vs HDR PQ/HLG) - let transfer_function = match frame.color_transfer_characteristic() { - ffmpeg::util::color::TransferCharacteristic::SMPTE2084 => TransferFunction::PQ, - ffmpeg::util::color::TransferCharacteristic::ARIB_STD_B67 => { - TransferFunction::HLG - } - _ => TransferFunction::SDR, - }; - - // Log color metadata on first frame - if *frames_decoded == 1 { - info!( - "First frame color info: space={:?}, range={:?}, transfer={:?} (raw: {:?})", - color_space, - color_range, - transfer_function, - frame.color_transfer_characteristic() - ); - } - - // ZERO-COPY PATH: For VideoToolbox, extract CVPixelBuffer directly - // This skips the expensive GPU->CPU->GPU copy entirely - #[cfg(target_os = "macos")] - if format == Pixel::VIDEOTOOLBOX { - use crate::media::videotoolbox; - use std::sync::Arc; - - // Extract CVPixelBuffer from frame.data[3] using raw FFmpeg pointer - // We use unsafe FFI because the safe wrapper does bounds checking - // that doesn't work for hardware frames - let cv_buffer = unsafe { - let raw_frame = frame.as_ptr(); - let data_ptr = (*raw_frame).data[3] as *mut u8; - if !data_ptr.is_null() { - videotoolbox::extract_cv_pixel_buffer_from_data(data_ptr) - } else { - None - } - }; - - if let Some(buffer) = cv_buffer { - if *frames_decoded == 1 { - info!( - "ZERO-COPY: First frame {}x{} via CVPixelBuffer (no CPU transfer!)", - w, h - ); - } - - *width = w; - *height = h; - - return Some(VideoFrame { - frame_id: super::next_frame_id(), - width: w, - height: h, - y_plane: Vec::new(), - u_plane: Vec::new(), - v_plane: Vec::new(), - y_stride: 0, - u_stride: 0, - v_stride: 0, - timestamp_us: 0, - format: PixelFormat::NV12, - color_range, - color_space, - transfer_function, - gpu_frame: Some(Arc::new(buffer)), - }); - } else { - warn!("Failed to extract CVPixelBuffer, falling back to CPU transfer"); - } - } - - // ZERO-COPY PATH: For D3D11VA, extract D3D11 texture directly - // This skips the expensive GPU->CPU->GPU copy entirely - #[cfg(target_os = "windows")] - if format == Pixel::D3D11 || format == Pixel::D3D11VA_VLD { - use crate::media::d3d11; - use std::sync::Arc; - - // Extract D3D11 texture from frame data - // FFmpeg D3D11VA frame layout: - // - data[0] = ID3D11Texture2D* - // - data[1] = texture array index (as intptr_t) - let d3d11_texture = unsafe { - let raw_frame = frame.as_ptr(); - let data0 = (*raw_frame).data[0] as *mut u8; - let data1 = (*raw_frame).data[1] as *mut u8; - d3d11::extract_d3d11_texture_from_frame(data0, data1) - }; - - if let Some(texture) = d3d11_texture { - if *frames_decoded == 1 { - info!( - "ZERO-COPY: First frame {}x{} via D3D11 texture (no CPU transfer!)", - w, h - ); - } - - *width = w; - *height = h; - - return Some(VideoFrame { - frame_id: super::next_frame_id(), - width: w, - height: h, - y_plane: Vec::new(), - u_plane: Vec::new(), - v_plane: Vec::new(), - y_stride: 0, - u_stride: 0, - v_stride: 0, - timestamp_us: 0, - format: PixelFormat::NV12, - color_range, - color_space, - transfer_function, - gpu_frame: Some(Arc::new(texture)), - }); - } else { - warn!("Failed to extract D3D11 texture, falling back to CPU transfer"); - } - } - - // ZERO-COPY PATH: For VAAPI, extract VA surface directly - // This skips the expensive GPU->CPU->GPU copy entirely - #[cfg(target_os = "linux")] - if format!("{:?}", format).contains("VAAPI") { - use crate::media::vaapi; - use std::sync::Arc; - - // Extract VAAPI surface from frame data - // FFmpeg VAAPI frame layout: - // - data[3] = VASurfaceID (as pointer-sized value) - // - hw_frames_ctx->device_ctx->hwctx = VADisplay - let vaapi_surface = unsafe { - let raw_frame = frame.as_ptr(); - let data3 = (*raw_frame).data[3] as *mut u8; - - // Get VADisplay from hw_frames_ctx - let hw_frames_ctx = (*raw_frame).hw_frames_ctx; - let va_display = if !hw_frames_ctx.is_null() { - let frames_ctx = - (*hw_frames_ctx).data as *mut ffmpeg::ffi::AVHWFramesContext; - if !frames_ctx.is_null() { - let device_ctx = (*frames_ctx).device_ctx; - if !device_ctx.is_null() { - (*device_ctx).hwctx as *mut std::ffi::c_void - } else { - std::ptr::null_mut() - } - } else { - std::ptr::null_mut() - } - } else { - std::ptr::null_mut() - }; - - vaapi::extract_vaapi_surface_from_frame(data3, va_display, w, h) - }; - - if let Some(surface) = vaapi_surface { - if *frames_decoded == 1 { - info!( - "ZERO-COPY: First frame {}x{} via VAAPI surface (no CPU transfer!)", - w, h - ); - } - - *width = w; - *height = h; - - return Some(VideoFrame { - frame_id: super::next_frame_id(), - width: w, - height: h, - y_plane: Vec::new(), - u_plane: Vec::new(), - v_plane: Vec::new(), - y_stride: 0, - u_stride: 0, - v_stride: 0, - timestamp_us: 0, - format: PixelFormat::NV12, - color_range, - color_space, - transfer_function, - gpu_frame: Some(Arc::new(surface)), - }); - } else { - warn!("Failed to extract VAAPI surface, falling back to CPU transfer"); - } - } - - // FALLBACK: Transfer hardware frame to CPU memory - let sw_frame = Self::transfer_hw_frame_if_needed(&frame); - let frame_to_use = sw_frame.as_ref().unwrap_or(&frame); - let actual_format = frame_to_use.format(); - - if *frames_decoded == 1 { - info!("First decoded frame: {}x{}, format: {:?} (hw: {:?}), range: {:?}, space: {:?}, transfer: {:?}", - w, h, actual_format, format, color_range, color_space, transfer_function); - } - - // Check if frame is NV12 - skip CPU scaler and pass directly to GPU - // NV12 has Y plane (full res) and UV plane (half res, interleaved) - // GPU shader will handle color conversion - much faster than CPU scaler - if actual_format == Pixel::NV12 { - *width = w; - *height = h; - - let y_stride = frame_to_use.stride(0) as u32; - let uv_stride = frame_to_use.stride(1) as u32; - let uv_height = h / 2; - - let y_data = frame_to_use.data(0); - let uv_data = frame_to_use.data(1); - - // Check if we actually have data - if y_data.is_empty() || uv_data.is_empty() || y_stride == 0 { - warn!( - "NV12 frame has empty data: y_len={}, uv_len={}, y_stride={}", - y_data.len(), - uv_data.len(), - y_stride - ); - // Fall through to scaler path - } else { - // GPU texture upload requires 256-byte aligned rows (wgpu restriction) - let aligned_y_stride = Self::get_aligned_stride(w); - let aligned_uv_stride = Self::get_aligned_stride(w); - - if *frames_decoded == 1 { - info!("NV12 direct path: {}x{}, y_stride={}, uv_stride={}, y_len={}, uv_len={}", - w, h, y_stride, uv_stride, y_data.len(), uv_data.len()); - } - - // Optimized copy - fast path when strides match - let copy_plane_fast = |src: &[u8], - src_stride: u32, - dst_stride: u32, - copy_width: u32, - height: u32| - -> Vec { - let total_size = (dst_stride * height) as usize; - if src_stride == dst_stride && src.len() >= total_size { - // Fast path: single memcpy - src[..total_size].to_vec() - } else { - // Slow path: row-by-row - let mut dst = vec![0u8; total_size]; - for row in 0..height as usize { - let src_start = row * src_stride as usize; - let src_end = src_start + copy_width as usize; - let dst_start = row * dst_stride as usize; - if src_end <= src.len() { - dst[dst_start..dst_start + copy_width as usize] - .copy_from_slice(&src[src_start..src_end]); - } - } - dst - } - }; - - let y_plane = copy_plane_fast(y_data, y_stride, aligned_y_stride, w, h); - let uv_plane = - copy_plane_fast(uv_data, uv_stride, aligned_uv_stride, w, uv_height); - - if *frames_decoded == 1 { - info!("NV12 direct GPU path: {}x{} - bypassing CPU scaler (y={} bytes, uv={} bytes)", - w, h, y_plane.len(), uv_plane.len()); - } - - return Some(VideoFrame { - frame_id: super::next_frame_id(), - width: w, - height: h, - y_plane, - u_plane: uv_plane, - v_plane: Vec::new(), - y_stride: aligned_y_stride, - u_stride: aligned_uv_stride, - v_stride: 0, - timestamp_us: 0, - format: PixelFormat::NV12, - color_range, - color_space, - transfer_function, - #[cfg(target_os = "macos")] - gpu_frame: None, - #[cfg(target_os = "windows")] - gpu_frame: None, - #[cfg(target_os = "linux")] - gpu_frame: None, - }); - } - } - - // For other formats, use scaler to convert to NV12 - // NV12 is more efficient for GPU upload and hardware decoders at high bitrates - // Use POINT (nearest neighbor) since we're not resizing - just color format conversion - // This is much faster than BILINEAR for same-size conversion - if scaler.is_none() || *width != w || *height != h { - *width = w; - *height = h; - - info!( - "Creating scaler: {:?} {}x{} -> NV12 {}x{} (POINT mode)", - actual_format, w, h, w, h - ); - - match ScalerContext::get( - actual_format, - w, - h, - Pixel::NV12, - w, - h, - ScalerFlags::POINT, // Fastest - no interpolation needed for same-size conversion - ) { - Ok(s) => *scaler = Some(s), - Err(e) => { - warn!("Failed to create scaler: {:?}", e); - return None; - } - } - } - - // Convert to NV12 - // We must allocate the destination frame first! - let mut nv12_frame = FfmpegFrame::new(Pixel::NV12, w, h); - - if let Some(ref mut s) = scaler { - if let Err(e) = s.run(frame_to_use, &mut nv12_frame) { - warn!("Scaler run failed: {:?}", e); - return None; - } - } else { - return None; - } - - // Extract NV12 planes with alignment - // NV12: Y plane (full res) + UV plane (half height, interleaved) - let y_stride = nv12_frame.stride(0) as u32; - let uv_stride = nv12_frame.stride(1) as u32; - - let aligned_y_stride = Self::get_aligned_stride(w); - let aligned_uv_stride = Self::get_aligned_stride(w); - - let uv_height = h / 2; - - // Optimized plane copy - use bulk copy when strides match, row-by-row otherwise - let copy_plane_optimized = |src: &[u8], - src_stride: u32, - dst_stride: u32, - width: u32, - height: u32| - -> Vec { - let total_size = (dst_stride * height) as usize; - - // Fast path: if source stride equals destination stride AND covers the data we need, - // we can do a single memcpy - if src_stride == dst_stride && src.len() >= total_size { - src[..total_size].to_vec() - } else { - // Slow path: row-by-row copy with stride conversion - let mut dst = vec![0u8; total_size]; - let width = width as usize; - let src_stride = src_stride as usize; - let dst_stride = dst_stride as usize; - - for row in 0..height as usize { - let src_start = row * src_stride; - let src_end = src_start + width; - let dst_start = row * dst_stride; - if src_end <= src.len() { - dst[dst_start..dst_start + width] - .copy_from_slice(&src[src_start..src_end]); - } - } - dst - } - }; - - Some(VideoFrame { - frame_id: super::next_frame_id(), - width: w, - height: h, - y_plane: copy_plane_optimized( - nv12_frame.data(0), - y_stride, - aligned_y_stride, - w, - h, - ), - u_plane: copy_plane_optimized( - nv12_frame.data(1), - uv_stride, - aligned_uv_stride, - w, - uv_height, - ), - v_plane: Vec::new(), // NV12 has no separate V plane - y_stride: aligned_y_stride, - u_stride: aligned_uv_stride, - v_stride: 0, - timestamp_us: 0, - format: PixelFormat::NV12, - color_range, - color_space, - transfer_function, - #[cfg(target_os = "macos")] - gpu_frame: None, - #[cfg(target_os = "windows")] - gpu_frame: None, - #[cfg(target_os = "linux")] - gpu_frame: None, - }) - } - Err(ffmpeg::Error::Other { errno }) if errno == libc::EAGAIN => None, - Err(e) => { - debug!("Receive frame error: {:?}", e); - None - } - } - } - - /// Decode a NAL unit - sends to decoder thread and receives result - /// WARNING: This is BLOCKING and will stall the calling thread! - /// For low-latency streaming, use `decode_async()` instead. - pub fn decode(&mut self, data: &[u8]) -> Result> { - // Send decode command - self.cmd_tx - .send(DecoderCommand::Decode(data.to_vec())) - .map_err(|_| anyhow!("Decoder thread closed"))?; - - // Receive result (blocking) - match self.frame_rx.recv() { - Ok(frame) => { - if frame.is_some() { - self.frames_decoded += 1; - } - Ok(frame) - } - Err(_) => Err(anyhow!("Decoder thread closed")), - } - } - - /// Decode a NAL unit asynchronously - fire and forget - /// The decoded frame will be written directly to the SharedFrame. - /// Stats are sent via the stats channel returned from `new_async()`. - /// - /// This method NEVER blocks the calling thread, making it ideal for - /// the main streaming loop where input responsiveness is critical. - pub fn decode_async(&mut self, data: &[u8], receive_time: std::time::Instant) -> Result<()> { - self.cmd_tx - .send(DecoderCommand::DecodeAsync { - data: data.to_vec(), - receive_time, - }) - .map_err(|_| anyhow!("Decoder thread closed"))?; - - self.frames_decoded += 1; // Optimistic count - Ok(()) - } - - /// Check if using hardware acceleration - pub fn is_hw_accelerated(&self) -> bool { - self.hw_accel - } - - /// Get number of frames decoded - pub fn frames_decoded(&self) -> u64 { - self.frames_decoded - } -} - -impl Drop for VideoDecoder { - fn drop(&mut self) { - // Signal decoder thread to stop - let _ = self.cmd_tx.send(DecoderCommand::Stop); - } -} - -// ============================================================================ -// Unified Video Decoder - Wraps FFmpeg or Native DXVA decoder -// ============================================================================ - -/// Unified video decoder that can use either FFmpeg or native DXVA backend -/// -/// This enum provides a common interface for decoder types, allowing -/// the streaming code to use the appropriate backend transparently. -/// - Windows x64: GStreamer D3D11 for all codecs, Native DXVA for HEVC (experimental) -/// - Windows ARM64: Native DXVA only (GStreamer not available) -/// - macOS: FFmpeg with VideoToolbox -/// - Linux: Handled separately via Vulkan Video or GStreamer -#[cfg(all(windows, target_arch = "x86_64"))] -pub enum UnifiedVideoDecoder { - /// Native D3D11 Video decoder (HEVC only, NVIDIA-style) - Native(super::native_video::NativeVideoDecoder), - /// GStreamer D3D11 decoder (H.264/H.265/AV1, with hardware acceleration) - GStreamer(GStreamerDecoderWrapper), -} - -/// Windows ARM64: Only native DXVA available (no GStreamer) -#[cfg(all(windows, target_arch = "aarch64"))] -pub enum UnifiedVideoDecoder { - /// Native D3D11 Video decoder (HEVC only) - Native(super::native_video::NativeVideoDecoder), -} - -/// Wrapper for GStreamer decoder with async interface (Windows x64 only) -#[cfg(all(windows, target_arch = "x86_64"))] -pub struct GStreamerDecoderWrapper { - decoder: super::gstreamer_decoder::GStreamerDecoder, - shared_frame: Arc, - stats_tx: tokio_mpsc::Sender, - frames_decoded: u64, - /// Track consecutive failures for keyframe request - consecutive_failures: u32, -} - -#[cfg(target_os = "macos")] -pub enum UnifiedVideoDecoder { - /// FFmpeg-based decoder with VideoToolbox - Ffmpeg(VideoDecoder), -} - -#[cfg(target_os = "linux")] -pub enum UnifiedVideoDecoder { - /// Linux uses Vulkan Video or GStreamer (placeholder for unified interface) - Ffmpeg(VideoDecoder), -} - -impl UnifiedVideoDecoder { - /// Create a new unified decoder with the specified backend - pub fn new_async( - codec: VideoCodec, - backend: VideoDecoderBackend, - shared_frame: Arc, - ) -> Result<(Self, tokio_mpsc::Receiver)> { - // Windows x64: Use GStreamer D3D11 by default, Native DXVA only for HEVC when explicitly selected - #[cfg(all(windows, target_arch = "x86_64"))] - { - // Determine if we should use native DXVA decoder - // Native DXVA only supports HEVC and must be explicitly selected - let use_native = - backend == VideoDecoderBackend::NativeDxva && codec == VideoCodec::H265; - - if use_native { - // Native D3D11 Video decoder (HEVC only) - EXPERIMENTAL - // Only used when explicitly selected by user - info!("Creating native DXVA decoder for HEVC (experimental)"); - - let (native_decoder, native_stats_rx) = - super::native_video::NativeVideoDecoder::new_async( - codec, - shared_frame.clone(), - )?; - - info!("Native DXVA HEVC decoder created successfully"); - - // Convert NativeDecodeStats to DecodeStats via a bridge channel - let (stats_tx, stats_rx) = tokio_mpsc::channel::(64); - - // Spawn a task to convert stats - tokio::spawn(async move { - let mut native_rx = native_stats_rx; - while let Some(native_stats) = native_rx.recv().await { - let stats = DecodeStats { - decode_time_ms: native_stats.decode_time_ms, - frame_produced: native_stats.frame_produced, - needs_keyframe: native_stats.needs_keyframe, - }; - if stats_tx.send(stats).await.is_err() { - break; - } - } - }); - - return Ok((UnifiedVideoDecoder::Native(native_decoder), stats_rx)); - } - - // Default: Use GStreamer D3D11 decoder for all codecs - // This is stable and supports H.264, H.265, and AV1 - let gst_codec = match codec { - VideoCodec::H264 => { - info!("Creating GStreamer D3D11 decoder for H.264"); - super::gstreamer_decoder::GstCodec::H264 - } - VideoCodec::H265 => { - info!("Creating GStreamer D3D11 decoder for H.265"); - super::gstreamer_decoder::GstCodec::H265 - } - VideoCodec::AV1 => { - info!("Creating GStreamer D3D11 decoder for AV1"); - super::gstreamer_decoder::GstCodec::AV1 - } - }; - - let gst_config = super::gstreamer_decoder::GstDecoderConfig { - codec: gst_codec, - width: 1920, - height: 1080, - low_latency: true, - }; - - let gst_decoder = super::gstreamer_decoder::GStreamerDecoder::new(gst_config) - .map_err(|e| anyhow!("Failed to create GStreamer {:?} decoder: {}", codec, e))?; - - info!("GStreamer D3D11 {:?} decoder created successfully", codec); - - let (stats_tx, stats_rx) = tokio_mpsc::channel::(64); - - let wrapper = GStreamerDecoderWrapper { - decoder: gst_decoder, - shared_frame: shared_frame.clone(), - stats_tx, - frames_decoded: 0, - consecutive_failures: 0, - }; - - return Ok((UnifiedVideoDecoder::GStreamer(wrapper), stats_rx)); - } - - // Windows ARM64: Only native DXVA available (no GStreamer binaries for ARM64) - // Native DXVA only supports HEVC, so H.264/AV1 will fail - #[cfg(all(windows, target_arch = "aarch64"))] - { - if codec != VideoCodec::H265 { - return Err(anyhow!( - "Windows ARM64 only supports H.265/HEVC decoding. \ - H.264 and AV1 are not supported because GStreamer ARM64 binaries are not available. \ - Please use H.265 codec in settings." - )); - } - - info!("Creating native DXVA decoder for HEVC (Windows ARM64)"); - - let (native_decoder, native_stats_rx) = - super::native_video::NativeVideoDecoder::new_async(codec, shared_frame.clone())?; - - info!("Native DXVA HEVC decoder created successfully (ARM64)"); - - // Convert NativeDecodeStats to DecodeStats via a bridge channel - let (stats_tx, stats_rx) = tokio_mpsc::channel::(64); - - // Spawn a task to convert stats - tokio::spawn(async move { - let mut native_rx = native_stats_rx; - while let Some(native_stats) = native_rx.recv().await { - let stats = DecodeStats { - decode_time_ms: native_stats.decode_time_ms, - frame_produced: native_stats.frame_produced, - needs_keyframe: native_stats.needs_keyframe, - }; - if stats_tx.send(stats).await.is_err() { - break; - } - } - }); - - return Ok((UnifiedVideoDecoder::Native(native_decoder), stats_rx)); - } - - // macOS/Linux: Use FFmpeg decoder - #[cfg(not(windows))] - { - let (ffmpeg_decoder, stats_rx) = VideoDecoder::new_async(codec, backend, shared_frame)?; - Ok((UnifiedVideoDecoder::Ffmpeg(ffmpeg_decoder), stats_rx)) - } - } - - /// Decode a frame asynchronously - pub fn decode_async(&mut self, data: &[u8], receive_time: std::time::Instant) -> Result<()> { - match self { - #[cfg(not(windows))] - UnifiedVideoDecoder::Ffmpeg(decoder) => decoder.decode_async(data, receive_time), - #[cfg(windows)] - UnifiedVideoDecoder::Native(decoder) => { - decoder.decode_async(data.to_vec(), receive_time); - Ok(()) - } - #[cfg(all(windows, target_arch = "x86_64"))] - UnifiedVideoDecoder::GStreamer(wrapper) => { - wrapper.decode_async(data, receive_time); - Ok(()) - } - } - } - - /// Check if using hardware acceleration - pub fn is_hw_accelerated(&self) -> bool { - match self { - #[cfg(not(windows))] - UnifiedVideoDecoder::Ffmpeg(decoder) => decoder.is_hw_accelerated(), - #[cfg(windows)] - UnifiedVideoDecoder::Native(decoder) => decoder.is_hw_accel(), - #[cfg(all(windows, target_arch = "x86_64"))] - UnifiedVideoDecoder::GStreamer(_) => true, // GStreamer uses D3D11 hardware acceleration - } - } - - /// Get number of frames decoded - pub fn frames_decoded(&self) -> u64 { - match self { - #[cfg(not(windows))] - UnifiedVideoDecoder::Ffmpeg(decoder) => decoder.frames_decoded(), - #[cfg(windows)] - UnifiedVideoDecoder::Native(decoder) => decoder.frames_decoded(), - #[cfg(all(windows, target_arch = "x86_64"))] - UnifiedVideoDecoder::GStreamer(wrapper) => wrapper.frames_decoded, - } - } -} - -#[cfg(all(windows, target_arch = "x86_64"))] -impl GStreamerDecoderWrapper { - /// Threshold for requesting a keyframe after consecutive failures (lowered for faster recovery) - const KEYFRAME_REQUEST_THRESHOLD: u32 = 3; - - /// Decode a frame asynchronously and write to SharedFrame - pub fn decode_async(&mut self, data: &[u8], receive_time: std::time::Instant) { - let decode_start = std::time::Instant::now(); - - match self.decoder.decode(data) { - Ok(Some(frame)) => { - self.frames_decoded += 1; - self.consecutive_failures = 0; - self.shared_frame.write(frame); - - // Measure decode time from when we started pushing data - let decode_time_ms = decode_start.elapsed().as_secs_f32() * 1000.0; - - // Log first frame - if self.frames_decoded == 1 { - info!( - "GStreamer: First frame decoded in {:.1}ms (pipeline latency: {:.1}ms)", - decode_time_ms, - receive_time.elapsed().as_secs_f32() * 1000.0 - ); - } - - let _ = self.stats_tx.try_send(DecodeStats { - decode_time_ms, - frame_produced: true, - needs_keyframe: false, - }); - } - Ok(None) => { - // No frame produced yet (buffering or B-frame reordering) - self.consecutive_failures += 1; - - let needs_keyframe = - if self.consecutive_failures == Self::KEYFRAME_REQUEST_THRESHOLD { - warn!( - "GStreamer: {} consecutive packets without frame - requesting keyframe", - self.consecutive_failures - ); - true - } else if self.consecutive_failures > Self::KEYFRAME_REQUEST_THRESHOLD - && self.consecutive_failures % 20 == 0 - { - warn!( - "GStreamer: Still failing after {} packets - requesting keyframe again", - self.consecutive_failures - ); - true - } else { - false - }; - - let decode_time_ms = decode_start.elapsed().as_secs_f32() * 1000.0; - let _ = self.stats_tx.try_send(DecodeStats { - decode_time_ms, - frame_produced: false, - needs_keyframe, - }); - } - Err(e) => { - warn!("GStreamer decode error: {}", e); - self.consecutive_failures += 1; - - let decode_time_ms = decode_start.elapsed().as_secs_f32() * 1000.0; - let _ = self.stats_tx.try_send(DecodeStats { - decode_time_ms, - frame_produced: false, - needs_keyframe: self.consecutive_failures >= Self::KEYFRAME_REQUEST_THRESHOLD, - }); - } - } - } -} diff --git a/opennow-streamer/src/media/videotoolbox.rs b/opennow-streamer/src/media/videotoolbox.rs deleted file mode 100644 index dc524c6..0000000 --- a/opennow-streamer/src/media/videotoolbox.rs +++ /dev/null @@ -1,1005 +0,0 @@ -//! VideoToolbox Zero-Copy Support (macOS only) -//! -//! Provides zero-copy video frame handling by keeping decoded frames on GPU. -//! Instead of copying pixel data from VideoToolbox to CPU memory, we: -//! 1. Extract the CVPixelBuffer from the decoded AVFrame -//! 2. Retain it and pass to the renderer -//! 3. Create Metal textures directly from the IOSurface -//! 4. Use those textures in wgpu for rendering -//! -//! This eliminates ~360MB/sec of memory copies at 1080p@120fps. - -#![cfg(target_os = "macos")] - -use foreign_types::ForeignType; -use log::{debug, info, warn}; -use objc::runtime::{Object, YES}; -use objc::{class, msg_send, sel, sel_impl}; -use std::ffi::c_void; -use std::sync::Arc; - -// Core Video FFI -#[link(name = "CoreVideo", kind = "framework")] -extern "C" { - fn CVPixelBufferRetain(buffer: *mut c_void) -> *mut c_void; - fn CVPixelBufferRelease(buffer: *mut c_void); - fn CVPixelBufferGetWidth(buffer: *mut c_void) -> usize; - fn CVPixelBufferGetHeight(buffer: *mut c_void) -> usize; - fn CVPixelBufferGetPixelFormatType(buffer: *mut c_void) -> u32; - fn CVPixelBufferGetIOSurface(buffer: *mut c_void) -> *mut c_void; - fn CVPixelBufferLockBaseAddress(buffer: *mut c_void, flags: u64) -> i32; - fn CVPixelBufferUnlockBaseAddress(buffer: *mut c_void, flags: u64) -> i32; - fn CVPixelBufferGetPlaneCount(buffer: *mut c_void) -> usize; - fn CVPixelBufferGetBaseAddressOfPlane(buffer: *mut c_void, plane: usize) -> *mut u8; - fn CVPixelBufferGetBytesPerRowOfPlane(buffer: *mut c_void, plane: usize) -> usize; - fn CVPixelBufferGetHeightOfPlane(buffer: *mut c_void, plane: usize) -> usize; - fn CVPixelBufferGetWidthOfPlane(buffer: *mut c_void, plane: usize) -> usize; -} - -// Metal FFI for getting the system default device -#[link(name = "Metal", kind = "framework")] -extern "C" { - fn MTLCreateSystemDefaultDevice() -> *mut Object; -} - -// CoreVideo Metal texture cache FFI - TRUE zero-copy -#[link(name = "CoreVideo", kind = "framework")] -extern "C" { - fn CVMetalTextureCacheCreate( - allocator: *const c_void, - cache_attributes: *const c_void, - metal_device: *mut Object, - texture_attributes: *const c_void, - cache_out: *mut *mut c_void, - ) -> i32; - - fn CVMetalTextureCacheCreateTextureFromImage( - allocator: *const c_void, - texture_cache: *mut c_void, - source_image: *mut c_void, // CVPixelBufferRef - texture_attributes: *const c_void, - pixel_format: u64, // MTLPixelFormat - width: usize, - height: usize, - plane_index: usize, - texture_out: *mut *mut c_void, - ) -> i32; - - fn CVMetalTextureGetTexture(texture: *mut c_void) -> *mut Object; // Returns MTLTexture - fn CVMetalTextureCacheFlush(texture_cache: *mut c_void, options: u64); -} - -// kCVReturn success -const K_CV_RETURN_SUCCESS: i32 = 0; - -// MTLPixelFormat values -const MTL_PIXEL_FORMAT_R8_UNORM: u64 = 10; // For Y plane -const MTL_PIXEL_FORMAT_RG8_UNORM: u64 = 30; // For UV plane (interleaved) - -// Lock flags -const K_CV_PIXEL_BUFFER_LOCK_READ_ONLY: u64 = 0x00000001; - -// IOSurface FFI -#[link(name = "IOSurface", kind = "framework")] -extern "C" { - fn IOSurfaceGetWidth(surface: *mut c_void) -> usize; - fn IOSurfaceGetHeight(surface: *mut c_void) -> usize; - fn IOSurfaceIncrementUseCount(surface: *mut c_void); - fn IOSurfaceDecrementUseCount(surface: *mut c_void); -} - -// NV12 format constants -const K_CV_PIXEL_FORMAT_TYPE_420_YP_CB_CR_8_BI_PLANAR_VIDEO_RANGE: u32 = 0x34323076; // '420v' -const K_CV_PIXEL_FORMAT_TYPE_420_YP_CB_CR_8_BI_PLANAR_FULL_RANGE: u32 = 0x34323066; // '420f' - -/// Wrapper around CVPixelBuffer that handles retain/release -/// This allows passing the GPU buffer between threads safely -pub struct CVPixelBufferWrapper { - buffer: *mut c_void, - width: u32, - height: u32, - is_nv12: bool, -} - -impl std::fmt::Debug for CVPixelBufferWrapper { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("CVPixelBufferWrapper") - .field("width", &self.width) - .field("height", &self.height) - .field("is_nv12", &self.is_nv12) - .finish() - } -} - -// CVPixelBuffer is reference-counted and thread-safe -unsafe impl Send for CVPixelBufferWrapper {} -unsafe impl Sync for CVPixelBufferWrapper {} - -impl CVPixelBufferWrapper { - /// Create a new wrapper, retaining the CVPixelBuffer - /// - /// # Safety - /// The provided pointer must be a valid CVPixelBufferRef - pub unsafe fn new(buffer: *mut c_void) -> Option { - if buffer.is_null() { - return None; - } - - // Retain the buffer so it stays valid - CVPixelBufferRetain(buffer); - - let width = CVPixelBufferGetWidth(buffer) as u32; - let height = CVPixelBufferGetHeight(buffer) as u32; - let format = CVPixelBufferGetPixelFormatType(buffer); - - // Check if it's NV12 format (what VideoToolbox typically outputs) - let is_nv12 = format == K_CV_PIXEL_FORMAT_TYPE_420_YP_CB_CR_8_BI_PLANAR_VIDEO_RANGE - || format == K_CV_PIXEL_FORMAT_TYPE_420_YP_CB_CR_8_BI_PLANAR_FULL_RANGE; - - if !is_nv12 { - debug!("CVPixelBuffer format is not NV12: {:#x}", format); - } - - Some(Self { - buffer, - width, - height, - is_nv12, - }) - } - - pub fn width(&self) -> u32 { - self.width - } - - pub fn height(&self) -> u32 { - self.height - } - - pub fn is_nv12(&self) -> bool { - self.is_nv12 - } - - /// Get the IOSurface backing this pixel buffer - /// Returns None if the buffer is not backed by an IOSurface - pub fn io_surface(&self) -> Option<*mut c_void> { - unsafe { - let surface = CVPixelBufferGetIOSurface(self.buffer); - if surface.is_null() { - None - } else { - Some(surface) - } - } - } - - /// Get the raw CVPixelBufferRef (for FFI) - pub fn as_raw(&self) -> *mut c_void { - self.buffer - } - - /// Lock the pixel buffer and get direct access to plane data - /// This maps GPU memory to CPU address space WITHOUT copying - /// Returns (y_data, y_stride, uv_data, uv_stride) for NV12 format - /// IMPORTANT: Call unlock() when done to release the mapping - pub fn lock_and_get_planes(&self) -> Option { - unsafe { - // Lock for read-only access (faster) - let result = - CVPixelBufferLockBaseAddress(self.buffer, K_CV_PIXEL_BUFFER_LOCK_READ_ONLY); - if result != 0 { - warn!("Failed to lock CVPixelBuffer: {}", result); - return None; - } - - let plane_count = CVPixelBufferGetPlaneCount(self.buffer); - if plane_count < 2 { - warn!( - "CVPixelBuffer has {} planes, expected 2 for NV12", - plane_count - ); - CVPixelBufferUnlockBaseAddress(self.buffer, K_CV_PIXEL_BUFFER_LOCK_READ_ONLY); - return None; - } - - // Y plane (plane 0) - let y_ptr = CVPixelBufferGetBaseAddressOfPlane(self.buffer, 0); - let y_stride = CVPixelBufferGetBytesPerRowOfPlane(self.buffer, 0); - let y_height = CVPixelBufferGetHeightOfPlane(self.buffer, 0); - - // UV plane (plane 1) - interleaved for NV12 - let uv_ptr = CVPixelBufferGetBaseAddressOfPlane(self.buffer, 1); - let uv_stride = CVPixelBufferGetBytesPerRowOfPlane(self.buffer, 1); - let uv_height = CVPixelBufferGetHeightOfPlane(self.buffer, 1); - - if y_ptr.is_null() || uv_ptr.is_null() { - warn!("CVPixelBuffer plane pointers are null"); - CVPixelBufferUnlockBaseAddress(self.buffer, K_CV_PIXEL_BUFFER_LOCK_READ_ONLY); - return None; - } - - Some(LockedPlanes { - buffer: self.buffer, - y_data: std::slice::from_raw_parts(y_ptr, y_stride * y_height), - y_stride: y_stride as u32, - y_height: y_height as u32, - uv_data: std::slice::from_raw_parts(uv_ptr, uv_stride * uv_height), - uv_stride: uv_stride as u32, - uv_height: uv_height as u32, - }) - } - } -} - -/// Locked plane data from CVPixelBuffer -/// Automatically unlocks on drop -pub struct LockedPlanes<'a> { - buffer: *mut c_void, - pub y_data: &'a [u8], - pub y_stride: u32, - pub y_height: u32, - pub uv_data: &'a [u8], - pub uv_stride: u32, - pub uv_height: u32, -} - -impl<'a> Drop for LockedPlanes<'a> { - fn drop(&mut self) { - unsafe { - CVPixelBufferUnlockBaseAddress(self.buffer, K_CV_PIXEL_BUFFER_LOCK_READ_ONLY); - } - } -} - -impl Drop for CVPixelBufferWrapper { - fn drop(&mut self) { - unsafe { - CVPixelBufferRelease(self.buffer); - } - } -} - -impl Clone for CVPixelBufferWrapper { - fn clone(&self) -> Self { - unsafe { - CVPixelBufferRetain(self.buffer); - } - Self { - buffer: self.buffer, - width: self.width, - height: self.height, - is_nv12: self.is_nv12, - } - } -} - -/// Extract CVPixelBuffer from an FFmpeg hardware frame -/// -/// # Safety -/// The AVFrame must be a VideoToolbox hardware frame (format = AV_PIX_FMT_VIDEOTOOLBOX) -/// The frame_data_3 parameter should be frame.data[3] from the AVFrame -pub unsafe fn extract_cv_pixel_buffer_from_data( - frame_data_3: *mut u8, -) -> Option { - // For VideoToolbox frames, data[3] contains the CVPixelBufferRef - // This is FFmpeg's convention for VideoToolbox hardware frames - let cv_buffer = frame_data_3 as *mut c_void; - CVPixelBufferWrapper::new(cv_buffer) -} - -/// Zero-copy video frame that holds GPU buffer reference -#[derive(Clone)] -pub struct ZeroCopyFrame { - pub buffer: Arc, -} - -impl ZeroCopyFrame { - pub fn new(buffer: CVPixelBufferWrapper) -> Self { - Self { - buffer: Arc::new(buffer), - } - } - - pub fn width(&self) -> u32 { - self.buffer.width() - } - - pub fn height(&self) -> u32 { - self.buffer.height() - } -} - -impl std::fmt::Debug for ZeroCopyFrame { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ZeroCopyFrame") - .field("width", &self.buffer.width()) - .field("height", &self.buffer.height()) - .field("is_nv12", &self.buffer.is_nv12()) - .finish() - } -} - -/// IOSurface wrapper for safe handling -pub struct IOSurfaceWrapper { - surface: *mut c_void, -} - -unsafe impl Send for IOSurfaceWrapper {} -unsafe impl Sync for IOSurfaceWrapper {} - -impl IOSurfaceWrapper { - pub unsafe fn new(surface: *mut c_void) -> Option { - if surface.is_null() { - return None; - } - IOSurfaceIncrementUseCount(surface); - Some(Self { surface }) - } - - pub fn as_ptr(&self) -> *mut c_void { - self.surface - } - - pub fn width(&self) -> u32 { - unsafe { IOSurfaceGetWidth(self.surface) as u32 } - } - - pub fn height(&self) -> u32 { - unsafe { IOSurfaceGetHeight(self.surface) as u32 } - } -} - -impl Drop for IOSurfaceWrapper { - fn drop(&mut self) { - unsafe { - IOSurfaceDecrementUseCount(self.surface); - } - } -} - -/// Metal texture pair created from IOSurface (Y and UV planes) -/// These textures share memory with the CVPixelBuffer - true zero-copy! -pub struct MetalTexturesFromIOSurface { - pub y_texture: *mut Object, // MTLTexture for Y plane - pub uv_texture: *mut Object, // MTLTexture for UV plane (interleaved) - pub width: u32, - pub height: u32, - // Keep the CVPixelBuffer alive while textures are in use - _cv_buffer: Arc, -} - -unsafe impl Send for MetalTexturesFromIOSurface {} -unsafe impl Sync for MetalTexturesFromIOSurface {} - -impl Drop for MetalTexturesFromIOSurface { - fn drop(&mut self) { - unsafe { - if !self.y_texture.is_null() { - let _: () = msg_send![self.y_texture, release]; - } - if !self.uv_texture.is_null() { - let _: () = msg_send![self.uv_texture, release]; - } - } - } -} - -impl MetalTexturesFromIOSurface { - /// Create Metal textures directly from CVPixelBuffer's IOSurface - /// This is TRUE zero-copy - the textures share GPU memory with the decoded frame - pub fn from_cv_buffer( - cv_buffer: Arc, - metal_device: *mut Object, - ) -> Option { - if metal_device.is_null() { - warn!("Metal device is null"); - return None; - } - - let io_surface = cv_buffer.io_surface()?; - let width = cv_buffer.width(); - let height = cv_buffer.height(); - - unsafe { - // Create Y texture (plane 0) - R8Unorm format - let y_texture = Self::create_texture_from_iosurface( - metal_device, - io_surface, - 0, // plane 0 = Y - width, - height, - 8, // MTLPixelFormatR8Unorm - )?; - - // Create UV texture (plane 1) - RG8Unorm format (interleaved UV) - let uv_texture = Self::create_texture_from_iosurface( - metal_device, - io_surface, - 1, // plane 1 = UV - width / 2, - height / 2, - 30, // MTLPixelFormatRG8Unorm - )?; - - info!( - "Created Metal textures from IOSurface: {}x{} (zero-copy)", - width, height - ); - - Some(Self { - y_texture, - uv_texture, - width, - height, - _cv_buffer: cv_buffer, - }) - } - } - - /// Create a single Metal texture from an IOSurface plane - unsafe fn create_texture_from_iosurface( - device: *mut Object, - io_surface: *mut c_void, - plane: usize, - width: u32, - height: u32, - pixel_format: u64, - ) -> Option<*mut Object> { - // Create MTLTextureDescriptor - let descriptor: *mut Object = msg_send![class!(MTLTextureDescriptor), new]; - if descriptor.is_null() { - return None; - } - - // Configure descriptor - let _: () = msg_send![descriptor, setTextureType: 2u64]; // MTLTextureType2D - let _: () = msg_send![descriptor, setPixelFormat: pixel_format]; - let _: () = msg_send![descriptor, setWidth: width as u64]; - let _: () = msg_send![descriptor, setHeight: height as u64]; - let _: () = msg_send![descriptor, setStorageMode: 1u64]; // MTLStorageModeManaged (shared on Apple Silicon) - let _: () = msg_send![descriptor, setUsage: 1u64]; // MTLTextureUsageShaderRead - - // Create texture from IOSurface - let texture: *mut Object = - msg_send![device, newTextureWithDescriptor:descriptor iosurface:io_surface plane:plane]; - - // Release descriptor - let _: () = msg_send![descriptor, release]; - - if texture.is_null() { - warn!( - "Failed to create Metal texture from IOSurface plane {}", - plane - ); - return None; - } - - Some(texture) - } -} - -/// Manager for zero-copy GPU textures using CVMetalTextureCache -/// This creates Metal textures that share GPU memory with CVPixelBuffer - NO CPU COPY! -pub struct ZeroCopyTextureManager { - metal_device: *mut Object, - texture_cache: *mut c_void, - command_queue: *mut Object, // Cached command queue for GPU blits -} - -unsafe impl Send for ZeroCopyTextureManager {} -unsafe impl Sync for ZeroCopyTextureManager {} - -impl ZeroCopyTextureManager { - /// Create a new texture manager with CVMetalTextureCache - /// Returns None on legacy Macs (when legacy-macos feature is enabled) - pub fn new() -> Option { - // Legacy macOS mode: disable zero-copy to use CPU fallback - // This is needed for older Intel Macs (2015 and earlier) that have - // limited Metal support (Metal 1.0/1.1 instead of Metal 1.2+) - #[cfg(feature = "legacy-macos")] - { - info!("Legacy macOS mode: Zero-copy rendering disabled, using CPU fallback"); - return None; - } - - #[cfg(not(feature = "legacy-macos"))] - unsafe { - // Get system default Metal device - let metal_device = MTLCreateSystemDefaultDevice(); - if metal_device.is_null() { - warn!("Could not get Metal device"); - return None; - } - - // Create CVMetalTextureCache - let mut texture_cache: *mut c_void = std::ptr::null_mut(); - let result = CVMetalTextureCacheCreate( - std::ptr::null(), // default allocator - std::ptr::null(), // no cache attributes - metal_device, - std::ptr::null(), // no texture attributes - &mut texture_cache, - ); - - if result != K_CV_RETURN_SUCCESS || texture_cache.is_null() { - warn!("Failed to create CVMetalTextureCache: {}", result); - let _: () = msg_send![metal_device, release]; - return None; - } - - // Create a persistent command queue for GPU blits - let command_queue: *mut Object = msg_send![metal_device, newCommandQueue]; - if command_queue.is_null() { - warn!("Failed to create Metal command queue"); - CFRelease(texture_cache); - let _: () = msg_send![metal_device, release]; - return None; - } - - info!("ZeroCopyTextureManager: Created with CVMetalTextureCache and command queue (TRUE zero-copy)"); - Some(Self { - metal_device, - texture_cache, - command_queue, - }) - } - } - - /// Create Metal textures from CVPixelBuffer - TRUE ZERO-COPY - /// Returns (y_texture, uv_texture) as raw MTLTexture pointers - pub fn create_textures_from_cv_buffer( - &self, - cv_buffer: &CVPixelBufferWrapper, - ) -> Option<(CVMetalTexture, CVMetalTexture)> { - let width = cv_buffer.width() as usize; - let height = cv_buffer.height() as usize; - - unsafe { - // Create Y plane texture (plane 0) - let mut y_cv_texture: *mut c_void = std::ptr::null_mut(); - let result = CVMetalTextureCacheCreateTextureFromImage( - std::ptr::null(), - self.texture_cache, - cv_buffer.as_raw(), - std::ptr::null(), - MTL_PIXEL_FORMAT_R8_UNORM, - width, - height, - 0, // plane 0 = Y - &mut y_cv_texture, - ); - - if result != K_CV_RETURN_SUCCESS || y_cv_texture.is_null() { - warn!("Failed to create Y texture from CVPixelBuffer: {}", result); - return None; - } - - // Create UV plane texture (plane 1) - let mut uv_cv_texture: *mut c_void = std::ptr::null_mut(); - let result = CVMetalTextureCacheCreateTextureFromImage( - std::ptr::null(), - self.texture_cache, - cv_buffer.as_raw(), - std::ptr::null(), - MTL_PIXEL_FORMAT_RG8_UNORM, - width / 2, - height / 2, - 1, // plane 1 = UV - &mut uv_cv_texture, - ); - - if result != K_CV_RETURN_SUCCESS || uv_cv_texture.is_null() { - warn!("Failed to create UV texture from CVPixelBuffer: {}", result); - // Clean up Y texture - CFRelease(y_cv_texture); - return None; - } - - Some(( - CVMetalTexture::new( - y_cv_texture, - width as u32, - height as u32, - MTL_PIXEL_FORMAT_R8_UNORM, - ), - CVMetalTexture::new( - uv_cv_texture, - (width / 2) as u32, - (height / 2) as u32, - MTL_PIXEL_FORMAT_RG8_UNORM, - ), - )) - } - } - - /// Flush the texture cache (call periodically to free unused textures) - pub fn flush(&self) { - unsafe { - CVMetalTextureCacheFlush(self.texture_cache, 0); - } - } - - /// Get the Metal device pointer - pub fn metal_device(&self) -> *mut Object { - self.metal_device - } - - /// Get the cached command queue for GPU blits - pub fn command_queue(&self) -> *mut Object { - self.command_queue - } -} - -impl Default for ZeroCopyTextureManager { - fn default() -> Self { - Self::new().expect("Failed to create CVMetalTextureCache") - } -} - -impl Drop for ZeroCopyTextureManager { - fn drop(&mut self) { - unsafe { - if !self.command_queue.is_null() { - let _: () = msg_send![self.command_queue, release]; - } - if !self.texture_cache.is_null() { - CFRelease(self.texture_cache); - } - if !self.metal_device.is_null() { - let _: () = msg_send![self.metal_device, release]; - } - } - } -} - -// CFRelease for CoreFoundation objects -#[link(name = "CoreFoundation", kind = "framework")] -extern "C" { - fn CFRelease(cf: *mut c_void); - fn CFRetain(cf: *mut c_void) -> *mut c_void; -} - -/// Wrapper around CVMetalTexture that handles release -pub struct CVMetalTexture { - cv_texture: *mut c_void, - width: u32, - height: u32, - format: u64, // MTLPixelFormat -} - -unsafe impl Send for CVMetalTexture {} -unsafe impl Sync for CVMetalTexture {} - -impl CVMetalTexture { - fn new(cv_texture: *mut c_void, width: u32, height: u32, format: u64) -> Self { - Self { - cv_texture, - width, - height, - format, - } - } - - /// Get the underlying MTLTexture pointer - this shares GPU memory with CVPixelBuffer! - pub fn metal_texture_ptr(&self) -> *mut Object { - unsafe { CVMetalTextureGetTexture(self.cv_texture) } - } - - /// Get as metal-rs Texture type (for wgpu-hal integration) - /// The returned texture shares GPU memory with the CVPixelBuffer - TRUE ZERO-COPY! - pub fn as_metal_texture(&self) -> metal::Texture { - unsafe { - let ptr = self.metal_texture_ptr(); - // Retain because metal::Texture will release on drop - let _: () = msg_send![ptr, retain]; - metal::Texture::from_ptr(ptr as *mut _) - } - } - - pub fn width(&self) -> u32 { - self.width - } - pub fn height(&self) -> u32 { - self.height - } - pub fn pixel_format(&self) -> u64 { - self.format - } - - /// Convert MTLPixelFormat to wgpu TextureFormat - pub fn wgpu_format(&self) -> wgpu::TextureFormat { - match self.format { - 10 => wgpu::TextureFormat::R8Unorm, // MTLPixelFormatR8Unorm (Y plane) - 30 => wgpu::TextureFormat::Rg8Unorm, // MTLPixelFormatRG8Unorm (UV plane) - _ => wgpu::TextureFormat::R8Unorm, - } - } -} - -impl Drop for CVMetalTexture { - fn drop(&mut self) { - if !self.cv_texture.is_null() { - unsafe { - CFRelease(self.cv_texture); - } - } - } -} - -/// Metal-based video renderer for TRUE zero-copy rendering -/// Renders NV12 video directly from CVMetalTexture to the screen -pub struct MetalVideoRenderer { - device: *mut Object, - command_queue: *mut Object, - pipeline_state: *mut Object, - sampler_state: *mut Object, -} - -unsafe impl Send for MetalVideoRenderer {} -unsafe impl Sync for MetalVideoRenderer {} - -impl MetalVideoRenderer { - /// Create a new Metal video renderer - pub fn new(device: *mut Object) -> Option { - unsafe { - if device.is_null() { - return None; - } - - // Retain device - let _: () = msg_send![device, retain]; - - // Create command queue - let command_queue: *mut Object = msg_send![device, newCommandQueue]; - if command_queue.is_null() { - warn!("Failed to create Metal command queue"); - let _: () = msg_send![device, release]; - return None; - } - - // Create sampler state - let sampler_descriptor: *mut Object = msg_send![class!(MTLSamplerDescriptor), new]; - let _: () = msg_send![sampler_descriptor, setMinFilter: 1u64]; // Linear - let _: () = msg_send![sampler_descriptor, setMagFilter: 1u64]; // Linear - let _: () = msg_send![sampler_descriptor, setSAddressMode: 0u64]; // ClampToEdge - let _: () = msg_send![sampler_descriptor, setTAddressMode: 0u64]; // ClampToEdge - - let sampler_state: *mut Object = - msg_send![device, newSamplerStateWithDescriptor: sampler_descriptor]; - let _: () = msg_send![sampler_descriptor, release]; - - if sampler_state.is_null() { - warn!("Failed to create Metal sampler state"); - let _: () = msg_send![command_queue, release]; - let _: () = msg_send![device, release]; - return None; - } - - // Create render pipeline for NV12 to RGB conversion - let pipeline_state = Self::create_nv12_pipeline(device)?; - - info!("MetalVideoRenderer: Initialized for zero-copy video rendering"); - - Some(Self { - device, - command_queue, - pipeline_state, - sampler_state, - }) - } - } - - /// Create the NV12 to RGB render pipeline - unsafe fn create_nv12_pipeline(device: *mut Object) -> Option<*mut Object> { - // NV12 to RGB shader source (Metal Shading Language) - let shader_source = r#" - #include - using namespace metal; - - struct VertexOut { - float4 position [[position]]; - float2 texCoord; - }; - - // Full-screen triangle vertex shader - vertex VertexOut nv12_vertex(uint vertexID [[vertex_id]]) { - VertexOut out; - // Generate full-screen triangle - float2 positions[3] = { - float2(-1.0, -1.0), - float2(3.0, -1.0), - float2(-1.0, 3.0) - }; - float2 texCoords[3] = { - float2(0.0, 1.0), - float2(2.0, 1.0), - float2(0.0, -1.0) - }; - out.position = float4(positions[vertexID], 0.0, 1.0); - out.texCoord = texCoords[vertexID]; - return out; - } - - // NV12 to RGB fragment shader (BT.709) - fragment float4 nv12_fragment( - VertexOut in [[stage_in]], - texture2d yTexture [[texture(0)]], - texture2d uvTexture [[texture(1)]], - sampler texSampler [[sampler(0)]] - ) { - float y = yTexture.sample(texSampler, in.texCoord).r; - float2 uv = uvTexture.sample(texSampler, in.texCoord).rg; - - // BT.709 YUV to RGB conversion (video range) - float u = uv.r - 0.5; - float v = uv.g - 0.5; - - // BT.709 matrix - float r = y + 1.5748 * v; - float g = y - 0.1873 * u - 0.4681 * v; - float b = y + 1.8556 * u; - - return float4(saturate(float3(r, g, b)), 1.0); - } - "#; - - // Create shader library - let source_nsstring = Self::create_nsstring(shader_source); - if source_nsstring.is_null() { - return None; - } - - let mut error: *mut Object = std::ptr::null_mut(); - let library: *mut Object = msg_send![device, newLibraryWithSource: source_nsstring options: std::ptr::null::() error: &mut error]; - let _: () = msg_send![source_nsstring, release]; - - if library.is_null() { - if !error.is_null() { - let desc: *mut Object = msg_send![error, localizedDescription]; - let cstr: *const i8 = msg_send![desc, UTF8String]; - if !cstr.is_null() { - let err_str = std::ffi::CStr::from_ptr(cstr).to_string_lossy(); - warn!("Metal shader compilation error: {}", err_str); - } - } - return None; - } - - // Get vertex and fragment functions - let vertex_name = Self::create_nsstring("nv12_vertex"); - let fragment_name = Self::create_nsstring("nv12_fragment"); - - let vertex_fn: *mut Object = msg_send![library, newFunctionWithName: vertex_name]; - let fragment_fn: *mut Object = msg_send![library, newFunctionWithName: fragment_name]; - - let _: () = msg_send![vertex_name, release]; - let _: () = msg_send![fragment_name, release]; - let _: () = msg_send![library, release]; - - if vertex_fn.is_null() || fragment_fn.is_null() { - warn!("Failed to get shader functions"); - return None; - } - - // Create pipeline descriptor - let pipeline_desc: *mut Object = msg_send![class!(MTLRenderPipelineDescriptor), new]; - let _: () = msg_send![pipeline_desc, setVertexFunction: vertex_fn]; - let _: () = msg_send![pipeline_desc, setFragmentFunction: fragment_fn]; - - // Set color attachment format (BGRA8Unorm for Metal drawable) - let color_attachments: *mut Object = msg_send![pipeline_desc, colorAttachments]; - let attachment0: *mut Object = - msg_send![color_attachments, objectAtIndexedSubscript: 0usize]; - let _: () = msg_send![attachment0, setPixelFormat: 80u64]; // MTLPixelFormatBGRA8Unorm - - // Create pipeline state - let pipeline_state: *mut Object = msg_send![device, newRenderPipelineStateWithDescriptor: pipeline_desc error: &mut error]; - - let _: () = msg_send![pipeline_desc, release]; - let _: () = msg_send![vertex_fn, release]; - let _: () = msg_send![fragment_fn, release]; - - if pipeline_state.is_null() { - warn!("Failed to create render pipeline state"); - return None; - } - - Some(pipeline_state) - } - - /// Helper to create NSString - unsafe fn create_nsstring(s: &str) -> *mut Object { - let nsstring_class = class!(NSString); - let bytes = s.as_ptr() as *const i8; - let len = s.len(); - msg_send![nsstring_class, stringWithUTF8String: bytes] - } - - /// Render video frame using Metal - TRUE ZERO-COPY - /// Takes CVMetalTextures and renders directly to the provided drawable - pub fn render( - &self, - y_texture: &CVMetalTexture, - uv_texture: &CVMetalTexture, - drawable: *mut Object, // CAMetalDrawable - ) -> bool { - unsafe { - let y_mtl = y_texture.metal_texture_ptr(); - let uv_mtl = uv_texture.metal_texture_ptr(); - - if y_mtl.is_null() || uv_mtl.is_null() || drawable.is_null() { - return false; - } - - // Get drawable texture - let target_texture: *mut Object = msg_send![drawable, texture]; - if target_texture.is_null() { - return false; - } - - // Create command buffer - let command_buffer: *mut Object = msg_send![self.command_queue, commandBuffer]; - if command_buffer.is_null() { - return false; - } - - // Create render pass descriptor - let pass_desc: *mut Object = - msg_send![class!(MTLRenderPassDescriptor), renderPassDescriptor]; - let color_attachments: *mut Object = msg_send![pass_desc, colorAttachments]; - let attachment0: *mut Object = - msg_send![color_attachments, objectAtIndexedSubscript: 0usize]; - let _: () = msg_send![attachment0, setTexture: target_texture]; - let _: () = msg_send![attachment0, setLoadAction: 2u64]; // Clear - let _: () = msg_send![attachment0, setStoreAction: 1u64]; // Store - - // Create render encoder - let encoder: *mut Object = - msg_send![command_buffer, renderCommandEncoderWithDescriptor: pass_desc]; - if encoder.is_null() { - return false; - } - - // Set pipeline state - let _: () = msg_send![encoder, setRenderPipelineState: self.pipeline_state]; - - // Set textures (Y = 0, UV = 1) - let _: () = msg_send![encoder, setFragmentTexture: y_mtl atIndex: 0usize]; - let _: () = msg_send![encoder, setFragmentTexture: uv_mtl atIndex: 1usize]; - let _: () = - msg_send![encoder, setFragmentSamplerState: self.sampler_state atIndex: 0usize]; - - // Draw full-screen triangle - let _: () = - msg_send![encoder, drawPrimitives: 3u64 vertexStart: 0usize vertexCount: 3usize]; // Triangle - - // End encoding - let _: () = msg_send![encoder, endEncoding]; - - // Present and commit - let _: () = msg_send![command_buffer, presentDrawable: drawable]; - let _: () = msg_send![command_buffer, commit]; - - true - } - } -} - -impl Drop for MetalVideoRenderer { - fn drop(&mut self) { - unsafe { - if !self.sampler_state.is_null() { - let _: () = msg_send![self.sampler_state, release]; - } - if !self.pipeline_state.is_null() { - let _: () = msg_send![self.pipeline_state, release]; - } - if !self.command_queue.is_null() { - let _: () = msg_send![self.command_queue, release]; - } - if !self.device.is_null() { - let _: () = msg_send![self.device, release]; - } - } - } -} diff --git a/opennow-streamer/src/profiling.rs b/opennow-streamer/src/profiling.rs deleted file mode 100644 index cf8d9ba..0000000 --- a/opennow-streamer/src/profiling.rs +++ /dev/null @@ -1,70 +0,0 @@ -//! Profiling Module -//! -//! Optional Tracy profiler integration for performance analysis. -//! Enable with: cargo build --release --features tracy -//! -//! Usage: -//! 1. Build with tracy feature: cargo build --release --features tracy -//! 2. Download Tracy profiler from https://github.com/wolfpld/tracy/releases -//! 3. Run Tracy profiler and click "Connect" -//! 4. Run the application - Tracy will capture real-time profiling data - -/// Initialize the profiling system -/// Returns true if logging was initialized (caller should NOT call env_logger::init) -/// Returns false if caller should initialize logging themselves -pub fn init() -> bool { - #[cfg(feature = "tracy")] - { - use tracing_subscriber::layer::SubscriberExt; - use tracing_subscriber::util::SubscriberInitExt; - - // Create Tracy layer for profiling - let tracy_layer = tracing_tracy::TracyLayer::default(); - - // Create fmt layer for console logging (replaces env_logger when tracy is enabled) - let fmt_layer = tracing_subscriber::fmt::layer() - .with_target(true) - .with_level(true); - - // Create env filter (reads RUST_LOG, defaults to info) - let filter = tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")); - - // Set up tracing subscriber with both Tracy and console output - tracing_subscriber::registry() - .with(filter) - .with(fmt_layer) - .with(tracy_layer) - .init(); - - // Also set up log -> tracing bridge so log macros (info!, warn!, etc.) work - tracing_log::LogTracer::init().ok(); - - return true; // Logging initialized, caller should NOT call env_logger - } - - #[cfg(not(feature = "tracy"))] - { - return false; // Caller should initialize logging - } -} - -/// Profile a scope with a given name -/// This macro creates a tracing span that Tracy can visualize -#[macro_export] -macro_rules! profile_scope { - ($name:expr) => { - #[cfg(feature = "tracy")] - let _span = tracing::info_span!($name).entered(); - }; -} - -/// Mark a frame boundary for Tracy's frame view -/// Call this once per frame in the main render loop -#[inline] -pub fn frame_mark() { - #[cfg(feature = "tracy")] - { - tracing_tracy::client::frame_mark(); - } -} diff --git a/opennow-streamer/src/utils/logging.rs b/opennow-streamer/src/utils/logging.rs deleted file mode 100644 index e26b9e3..0000000 --- a/opennow-streamer/src/utils/logging.rs +++ /dev/null @@ -1,157 +0,0 @@ -//! Logging Utilities -//! -//! File-based and console logging. - -use std::fs::{File, OpenOptions}; -use std::io::Write; -use std::path::PathBuf; -use std::sync::Mutex; -use log::{Log, Metadata, Record, Level, LevelFilter}; - -/// Get the log file path -pub fn get_log_file_path() -> PathBuf { - super::get_app_data_dir().join("streamer.log") -} - -/// Simple file logger -pub struct FileLogger { - file: Mutex>, - console: bool, -} - -impl FileLogger { - pub fn new(console: bool) -> Self { - let file = Self::open_log_file(); - Self { - file: Mutex::new(file), - console, - } - } - - fn open_log_file() -> Option { - let path = get_log_file_path(); - - // Ensure directory exists - if let Some(parent) = path.parent() { - let _ = std::fs::create_dir_all(parent); - } - - OpenOptions::new() - .create(true) - .append(true) - .open(&path) - .ok() - } -} - -impl Log for FileLogger { - fn enabled(&self, metadata: &Metadata) -> bool { - let target = metadata.target(); - let level = metadata.level(); - - // STRICT filtering to prevent log spam from external crates - // This is CRITICAL for performance - even file I/O has overhead - - // Our crate: allow INFO and above (DEBUG only if explicitly needed) - if target.starts_with("opennow_streamer") { - level <= Level::Info - } else { - // External crates: WARN and ERROR only - // This silences: webrtc_sctp, webrtc_ice, webrtc, wgpu, wgpu_hal, etc. - level <= Level::Warn - } - } - - fn log(&self, record: &Record) { - let target = record.target(); - let level = record.level(); - - // Double-check filtering (belt and suspenders) - // External crates are restricted to WARN level - if !target.starts_with("opennow_streamer") && level > Level::Warn { - return; - } - - // Our crate allows DEBUG - if target.starts_with("opennow_streamer") && level > Level::Debug { - return; - } - - let timestamp = chrono::Local::now().format("%Y-%m-%d %H:%M:%S%.3f"); - let message = record.args(); - - let line = format!("[{}] {} {} - {}\n", timestamp, level, target, message); - - // Write to file - if let Ok(mut guard) = self.file.lock() { - if let Some(ref mut file) = *guard { - let _ = file.write_all(line.as_bytes()); - } - } - - // Write to console if enabled - if self.console { - print!("{}", line); - } - } - - fn flush(&self) { - if let Ok(mut guard) = self.file.lock() { - if let Some(ref mut file) = *guard { - let _ = file.flush(); - } - } - } -} - -/// Initialize the logging system -/// -/// Console logging is DISABLED by default for performance. -/// Windows console I/O is blocking and causes severe frame drops when -/// external crates (webrtc_sctp, wgpu, etc.) spam debug messages. -/// All logs are still written to the log file for debugging. -pub fn init_logging() -> Result<(), log::SetLoggerError> { - // CRITICAL: Console logging disabled for performance - // External crates spam DEBUG logs on every mouse movement - // Console I/O on Windows is blocking, causing "20 fps feel" - let logger = Box::new(FileLogger::new(false)); - log::set_boxed_logger(logger)?; - // Set global max to Info - we don't need DEBUG from external crates - // Our crate can still log at any level via the logger's enabled() check - log::set_max_level(LevelFilter::Info); - Ok(()) -} - -/// Initialize logging with console output (for debugging only) -/// WARNING: This will cause performance issues during streaming! -pub fn init_logging_with_console() -> Result<(), log::SetLoggerError> { - let logger = Box::new(FileLogger::new(true)); - log::set_boxed_logger(logger)?; - log::set_max_level(LevelFilter::Info); - Ok(()) -} - -/// Clear log file -pub fn clear_logs() -> std::io::Result<()> { - let path = get_log_file_path(); - if path.exists() { - std::fs::write(&path, "")?; - } - Ok(()) -} - -/// Export logs to a specific path -pub fn export_logs(dest: &PathBuf) -> std::io::Result<()> { - let src = get_log_file_path(); - if src.exists() { - std::fs::copy(&src, dest)?; - } - Ok(()) -} - -/// Print a message directly to console (bypasses logger) -/// Use sparingly - only for critical startup info -#[inline] -pub fn console_print(msg: &str) { - println!("{}", msg); -} diff --git a/opennow-streamer/src/utils/mod.rs b/opennow-streamer/src/utils/mod.rs deleted file mode 100644 index 21dfed7..0000000 --- a/opennow-streamer/src/utils/mod.rs +++ /dev/null @@ -1,44 +0,0 @@ -//! Utility Functions -//! -//! Common utilities used throughout the application. - -mod logging; -mod time; - -pub use logging::*; -pub use time::*; - -use std::path::PathBuf; - -/// Get the application data directory -pub fn get_app_data_dir() -> PathBuf { - dirs::config_dir() - .unwrap_or_else(|| PathBuf::from(".")) - .join("opennow-streamer") -} - -/// Get the cache directory -pub fn get_cache_dir() -> PathBuf { - dirs::cache_dir() - .unwrap_or_else(|| PathBuf::from(".")) - .join("opennow-streamer") -} - -/// Ensure a directory exists -pub fn ensure_dir(path: &PathBuf) -> std::io::Result<()> { - if !path.exists() { - std::fs::create_dir_all(path)?; - } - Ok(()) -} - -/// Generate a random peer ID for signaling -pub fn generate_peer_id() -> String { - let random: u64 = rand::random::() % 10_000_000_000; - format!("peer-{}", random) -} - -/// Generate a UUID string -pub fn generate_uuid() -> String { - uuid::Uuid::new_v4().to_string() -} diff --git a/opennow-streamer/src/utils/time.rs b/opennow-streamer/src/utils/time.rs deleted file mode 100644 index b3eb119..0000000 --- a/opennow-streamer/src/utils/time.rs +++ /dev/null @@ -1,128 +0,0 @@ -//! Time Utilities -//! -//! High-precision timing for input and frame synchronization. - -use std::time::{Duration, Instant}; - -/// High-precision timer for measuring frame times -pub struct FrameTimer { - start: Instant, - last_frame: Instant, - frame_count: u64, - frame_times: Vec, -} - -impl FrameTimer { - pub fn new() -> Self { - let now = Instant::now(); - Self { - start: now, - last_frame: now, - frame_count: 0, - frame_times: Vec::with_capacity(120), - } - } - - /// Mark a new frame and return delta time - pub fn tick(&mut self) -> Duration { - let now = Instant::now(); - let delta = now - self.last_frame; - self.last_frame = now; - self.frame_count += 1; - - // Keep last 120 frame times for FPS calculation - self.frame_times.push(delta); - if self.frame_times.len() > 120 { - self.frame_times.remove(0); - } - - delta - } - - /// Get current FPS based on recent frame times - pub fn fps(&self) -> f32 { - if self.frame_times.is_empty() { - return 0.0; - } - - let total: Duration = self.frame_times.iter().sum(); - let avg = total.as_secs_f32() / self.frame_times.len() as f32; - - if avg > 0.0 { - 1.0 / avg - } else { - 0.0 - } - } - - /// Get total elapsed time since start - pub fn elapsed(&self) -> Duration { - self.start.elapsed() - } - - /// Get total frame count - pub fn frame_count(&self) -> u64 { - self.frame_count - } - - /// Get average frame time in milliseconds - pub fn avg_frame_time_ms(&self) -> f32 { - if self.frame_times.is_empty() { - return 0.0; - } - - let total: Duration = self.frame_times.iter().sum(); - total.as_secs_f32() * 1000.0 / self.frame_times.len() as f32 - } -} - -impl Default for FrameTimer { - fn default() -> Self { - Self::new() - } -} - -/// Get current timestamp in microseconds (for input events) -pub fn timestamp_us() -> u64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_micros() as u64) - .unwrap_or(0) -} - -/// Get current timestamp in milliseconds -pub fn timestamp_ms() -> u64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0) -} - -/// Relative timestamp from a start time (in microseconds) -pub struct RelativeTimer { - start: Instant, -} - -impl RelativeTimer { - pub fn new() -> Self { - Self { - start: Instant::now(), - } - } - - /// Get microseconds since start - pub fn elapsed_us(&self) -> u64 { - self.start.elapsed().as_micros() as u64 - } - - /// Get milliseconds since start - pub fn elapsed_ms(&self) -> u64 { - self.start.elapsed().as_millis() as u64 - } -} - -impl Default for RelativeTimer { - fn default() -> Self { - Self::new() - } -} diff --git a/opennow-streamer/src/webrtc/datachannel.rs b/opennow-streamer/src/webrtc/datachannel.rs deleted file mode 100644 index eafe676..0000000 --- a/opennow-streamer/src/webrtc/datachannel.rs +++ /dev/null @@ -1,661 +0,0 @@ -//! GFN Input Protocol Encoder/Decoder -//! -//! Binary protocol for sending input events and receiving output events -//! (force feedback, rumble) over WebRTC data channel. - -use bytes::{Buf, BufMut, BytesMut}; -use log::debug; -use std::time::Instant; - -/// Input event type constants (Client → Server) -pub const INPUT_HEARTBEAT: u32 = 2; -pub const INPUT_KEY_DOWN: u32 = 3; // Type 3 = Key pressed -pub const INPUT_KEY_UP: u32 = 4; // Type 4 = Key released -pub const INPUT_MOUSE_ABS: u32 = 5; -pub const INPUT_MOUSE_REL: u32 = 7; -pub const INPUT_MOUSE_BUTTON_DOWN: u32 = 8; -pub const INPUT_MOUSE_BUTTON_UP: u32 = 9; -pub const INPUT_MOUSE_WHEEL: u32 = 10; -pub const INPUT_GAMEPAD: u32 = 12; // Type 12 = Gamepad state (NOT 6!) - -/// Output event type constants (Server → Client) -/// These are for force feedback / haptics from the game server -pub const OUTPUT_RUMBLE: u32 = 13; // Controller rumble/vibration -pub const OUTPUT_FORCE_FEEDBACK: u32 = 14; // Racing wheel force feedback - -/// Mouse buttons -pub const MOUSE_BUTTON_LEFT: u8 = 0; -pub const MOUSE_BUTTON_RIGHT: u8 = 1; -pub const MOUSE_BUTTON_MIDDLE: u8 = 2; - -/// Maximum clipboard paste buffer size (64KB, matches official GFN client) -pub const MAX_CLIPBOARD_PASTE_SIZE: usize = 65536; - -/// Input events that can be sent to the server -/// Each event carries its own timestamp_us (microseconds since app start) -/// for accurate timing even when events are queued. -#[derive(Debug, Clone)] -pub enum InputEvent { - /// Keyboard key pressed - KeyDown { - keycode: u16, - scancode: u16, - modifiers: u16, - timestamp_us: u64, - }, - /// Keyboard key released - KeyUp { - keycode: u16, - scancode: u16, - modifiers: u16, - timestamp_us: u64, - }, - /// Mouse moved (relative) - MouseMove { dx: i16, dy: i16, timestamp_us: u64 }, - /// Mouse button pressed - MouseButtonDown { button: u8, timestamp_us: u64 }, - /// Mouse button released - MouseButtonUp { button: u8, timestamp_us: u64 }, - /// Mouse wheel scrolled - MouseWheel { delta: i16, timestamp_us: u64 }, - /// Heartbeat (keep-alive) - Heartbeat, - /// Gamepad state update - Gamepad { - controller_id: u8, - button_flags: u16, - left_trigger: u8, - right_trigger: u8, - left_stick_x: i16, - left_stick_y: i16, - right_stick_x: i16, - right_stick_y: i16, - flags: u16, - timestamp_us: u64, - }, - /// Clipboard paste - text to be typed into the remote session - /// The text is sent character by character as keyboard input - /// (matches official GFN client behavior with clipboardHintStringType: "keyboard") - ClipboardPaste { text: String }, -} - -/// Encoder for GFN input protocol -pub struct InputEncoder { - buffer: BytesMut, - start_time: Instant, - protocol_version: u8, -} - -impl InputEncoder { - pub fn new() -> Self { - Self { - buffer: BytesMut::with_capacity(256), - start_time: Instant::now(), - protocol_version: 2, - } - } - - /// Set protocol version (received from handshake) - pub fn set_protocol_version(&mut self, version: u8) { - self.protocol_version = version; - } - - /// Get timestamp in microseconds since start - fn timestamp_us(&self) -> u64 { - self.start_time.elapsed().as_micros() as u64 - } - - /// Encode an input event to binary format - /// Uses the timestamp embedded in each event (captured at creation time) - pub fn encode(&mut self, event: &InputEvent) -> Vec { - self.buffer.clear(); - - match event { - InputEvent::KeyDown { - keycode, - scancode, - modifiers, - timestamp_us, - } => { - // Type 3 (Key Down): 18 bytes - // [type 4B LE][keycode 2B BE][modifiers 2B BE][scancode 2B BE][timestamp 8B BE] - self.buffer.put_u32_le(INPUT_KEY_DOWN); - self.buffer.put_u16(*keycode); - self.buffer.put_u16(*modifiers); - self.buffer.put_u16(*scancode); - self.buffer.put_u64(*timestamp_us); - } - - InputEvent::KeyUp { - keycode, - scancode, - modifiers, - timestamp_us, - } => { - self.buffer.put_u32_le(INPUT_KEY_UP); - self.buffer.put_u16(*keycode); - self.buffer.put_u16(*modifiers); - self.buffer.put_u16(*scancode); - self.buffer.put_u64(*timestamp_us); - } - - InputEvent::MouseMove { - dx, - dy, - timestamp_us, - } => { - // Type 7 (Mouse Relative): 22 bytes - // [type 4B LE][dx 2B BE][dy 2B BE][reserved 6B][timestamp 8B BE] - self.buffer.put_u32_le(INPUT_MOUSE_REL); - self.buffer.put_i16(*dx); - self.buffer.put_i16(*dy); - self.buffer.put_u16(0); // Reserved - self.buffer.put_u32(0); // Reserved - self.buffer.put_u64(*timestamp_us); - } - - InputEvent::MouseButtonDown { - button, - timestamp_us, - } => { - // Type 8 (Mouse Button Down): 18 bytes - // [type 4B LE][button 1B][pad 1B][reserved 4B][timestamp 8B BE] - self.buffer.put_u32_le(INPUT_MOUSE_BUTTON_DOWN); - self.buffer.put_u8(*button); - self.buffer.put_u8(0); // Padding - self.buffer.put_u32(0); // Reserved - self.buffer.put_u64(*timestamp_us); - } - - InputEvent::MouseButtonUp { - button, - timestamp_us, - } => { - self.buffer.put_u32_le(INPUT_MOUSE_BUTTON_UP); - self.buffer.put_u8(*button); - self.buffer.put_u8(0); - self.buffer.put_u32(0); - self.buffer.put_u64(*timestamp_us); - } - - InputEvent::MouseWheel { - delta, - timestamp_us, - } => { - // Type 10 (Mouse Wheel): 22 bytes - // [type 4B LE][horiz 2B BE][vert 2B BE][reserved 6B][timestamp 8B BE] - self.buffer.put_u32_le(INPUT_MOUSE_WHEEL); - self.buffer.put_i16(0); // Horizontal (unused) - self.buffer.put_i16(*delta); // Vertical (positive = scroll up) - self.buffer.put_u16(0); // Reserved - self.buffer.put_u32(0); // Reserved - self.buffer.put_u64(*timestamp_us); - } - - InputEvent::Heartbeat => { - // Type 2 (Heartbeat): 4 bytes - self.buffer.put_u32_le(INPUT_HEARTBEAT); - } - - InputEvent::ClipboardPaste { .. } => { - // ClipboardPaste is handled specially - it expands to multiple key events - // This should not be called directly; use encode_clipboard_paste() instead - // Return empty buffer as fallback - } - - InputEvent::Gamepad { - controller_id, - button_flags, - left_trigger, - right_trigger, - left_stick_x, - left_stick_y, - right_stick_x, - right_stick_y, - flags, - timestamp_us, - } => { - // Type 12 (Gamepad): 38 bytes total - from web client analysis - // Web client uses ALL LITTLE ENDIAN (DataView getUint16(true) = LE) - // - // Structure (from vendor_beautified.js fd() decoder): - // [0x00] Type: 4B LE (event type = 12) - // [0x04] Padding: 2B LE (reserved) - // [0x06] Index: 2B LE (gamepad index 0-3) - // [0x08] Bitmap: 2B LE (device type bitmap / flags) - // [0x0A] Padding: 2B LE (reserved) - // [0x0C] Buttons: 2B LE (button state bitmask) - // [0x0E] Trigger: 2B LE (packed: low=LT, high=RT, 0-255 each) - // [0x10] Axes[0]: 2B LE signed (Left X) - // [0x12] Axes[1]: 2B LE signed (Left Y) - // [0x14] Axes[2]: 2B LE signed (Right X) - // [0x16] Axes[3]: 2B LE signed (Right Y) - // [0x18] Padding: 2B LE (reserved) - // [0x1A] Padding: 2B LE (reserved) - // [0x1C] Padding: 2B LE (reserved) - // [0x1E] Timestamp: 8B LE (capture timestamp in microseconds) - // Total: 38 bytes - - self.buffer.put_u32_le(INPUT_GAMEPAD); // 0x00: Type = 12 (LE) - self.buffer.put_u16_le(0); // 0x04: Padding - self.buffer.put_u16_le(*controller_id as u16); // 0x06: Index (LE) - self.buffer.put_u16_le(*flags); // 0x08: Bitmap/flags (LE) - self.buffer.put_u16_le(0); // 0x0A: Padding - self.buffer.put_u16_le(*button_flags); // 0x0C: Buttons (LE) - // Pack triggers: low byte = LT, high byte = RT - let packed_triggers = (*left_trigger as u16) | ((*right_trigger as u16) << 8); - self.buffer.put_u16_le(packed_triggers); // 0x0E: Triggers packed (LE) - self.buffer.put_i16_le(*left_stick_x); // 0x10: Left X (LE) - self.buffer.put_i16_le(*left_stick_y); // 0x12: Left Y (LE) - self.buffer.put_i16_le(*right_stick_x); // 0x14: Right X (LE) - self.buffer.put_i16_le(*right_stick_y); // 0x16: Right Y (LE) - self.buffer.put_u16_le(0); // 0x18: Padding - self.buffer.put_u16_le(0); // 0x1A: Padding - self.buffer.put_u16_le(0); // 0x1C: Padding - self.buffer.put_u64_le(*timestamp_us); // 0x1E: Timestamp (LE) - } - } - - // Protocol v3+ requires single event wrapper - // Official client uses: [0x22][payload] for single events - if self.protocol_version > 2 { - let payload = self.buffer.to_vec(); - let mut final_buf = BytesMut::with_capacity(1 + payload.len()); - - // Single event wrapper marker (34 = 0x22) - final_buf.put_u8(0x22); - // Payload (already contains timestamp) - final_buf.extend_from_slice(&payload); - - final_buf.to_vec() - } else { - self.buffer.to_vec() - } - } - - /// Encode handshake response - pub fn encode_handshake_response(major: u8, minor: u8, flags: u8) -> Vec { - vec![0x0e, major, minor, flags] - } -} - -impl Default for InputEncoder { - fn default() -> Self { - Self::new() - } -} - -/// Convert a character to Windows Virtual Key code and shift state -/// Returns (vk_code, needs_shift) -pub fn char_to_vk(c: char) -> Option<(u16, bool)> { - match c { - // Lowercase letters -> VK_A to VK_Z (0x41-0x5A) - 'a'..='z' => Some((c.to_ascii_uppercase() as u16, false)), - // Uppercase letters -> VK_A to VK_Z with shift - 'A'..='Z' => Some((c as u16, true)), - // Numbers -> VK_0 to VK_9 (0x30-0x39) - '0'..='9' => Some((c as u16, false)), - // Shifted number symbols - '!' => Some((0x31, true)), // Shift+1 - '@' => Some((0x32, true)), // Shift+2 - '#' => Some((0x33, true)), // Shift+3 - '$' => Some((0x34, true)), // Shift+4 - '%' => Some((0x35, true)), // Shift+5 - '^' => Some((0x36, true)), // Shift+6 - '&' => Some((0x37, true)), // Shift+7 - '*' => Some((0x38, true)), // Shift+8 - '(' => Some((0x39, true)), // Shift+9 - ')' => Some((0x30, true)), // Shift+0 - // Common punctuation - ' ' => Some((0x20, false)), // VK_SPACE - '\t' => Some((0x09, false)), // VK_TAB - '\n' => None, // Skip newline (Enter) - could trigger unwanted form submissions - '\r' => None, // Skip carriage return - // OEM keys (US keyboard layout) - '-' => Some((0xBD, false)), // VK_OEM_MINUS - '_' => Some((0xBD, true)), // Shift+minus - '=' => Some((0xBB, false)), // VK_OEM_PLUS (equals key) - '+' => Some((0xBB, true)), // Shift+equals - '[' => Some((0xDB, false)), // VK_OEM_4 - '{' => Some((0xDB, true)), // Shift+[ - ']' => Some((0xDD, false)), // VK_OEM_6 - '}' => Some((0xDD, true)), // Shift+] - '\\' => Some((0xDC, false)), // VK_OEM_5 - '|' => Some((0xDC, true)), // Shift+backslash - ';' => Some((0xBA, false)), // VK_OEM_1 - ':' => Some((0xBA, true)), // Shift+semicolon - '\'' => Some((0xDE, false)), // VK_OEM_7 - '"' => Some((0xDE, true)), // Shift+quote - ',' => Some((0xBC, false)), // VK_OEM_COMMA - '<' => Some((0xBC, true)), // Shift+comma - '.' => Some((0xBE, false)), // VK_OEM_PERIOD - '>' => Some((0xBE, true)), // Shift+period - '/' => Some((0xBF, false)), // VK_OEM_2 - '?' => Some((0xBF, true)), // Shift+slash - '`' => Some((0xC0, false)), // VK_OEM_3 - '~' => Some((0xC0, true)), // Shift+backtick - _ => None, // Unsupported character - } -} - -/// Generate key events for clipboard paste text -/// Returns a vector of encoded key event packets ready to send -pub fn encode_clipboard_paste(encoder: &mut InputEncoder, text: &str) -> Vec> { - let mut packets = Vec::new(); - let base_timestamp = encoder.timestamp_us(); - let mut time_offset: u64 = 0; - - // VK_SHIFT = 0x10 - const VK_SHIFT: u16 = 0x10; - - for c in text.chars() { - if let Some((vk_code, needs_shift)) = char_to_vk(c) { - let timestamp = base_timestamp + time_offset; - - // Press shift if needed - if needs_shift { - let shift_down = InputEvent::KeyDown { - keycode: VK_SHIFT, - scancode: 0, - modifiers: 0x01, // Shift modifier - timestamp_us: timestamp, - }; - packets.push(encoder.encode(&shift_down)); - time_offset += 1; // 1 microsecond between events - } - - // Key down - let key_down = InputEvent::KeyDown { - keycode: vk_code, - scancode: 0, - modifiers: if needs_shift { 0x01 } else { 0 }, - timestamp_us: base_timestamp + time_offset, - }; - packets.push(encoder.encode(&key_down)); - time_offset += 1; - - // Key up - let key_up = InputEvent::KeyUp { - keycode: vk_code, - scancode: 0, - modifiers: if needs_shift { 0x01 } else { 0 }, - timestamp_us: base_timestamp + time_offset, - }; - packets.push(encoder.encode(&key_up)); - time_offset += 1; - - // Release shift if it was pressed - if needs_shift { - let shift_up = InputEvent::KeyUp { - keycode: VK_SHIFT, - scancode: 0, - modifiers: 0, - timestamp_us: base_timestamp + time_offset, - }; - packets.push(encoder.encode(&shift_up)); - time_offset += 1; - } - - // Small delay between characters (10 microseconds) - time_offset += 10; - } - // Skip unsupported characters silently - } - - packets -} - -/// Output events received from the server (force feedback / haptics) -#[derive(Debug, Clone)] -pub enum OutputEvent { - /// Controller rumble/vibration - /// Sent by server when game triggers haptic feedback - Rumble { - /// Controller index (0-3) - controller_id: u8, - /// Left motor intensity (0-255, low frequency / strong) - left_motor: u8, - /// Right motor intensity (0-255, high frequency / weak) - right_motor: u8, - /// Duration in milliseconds (0 = stop, 65535 = indefinite) - duration_ms: u16, - }, - /// Racing wheel force feedback - /// Sent by server for steering wheel effects - ForceFeedback { - /// Wheel index (usually 0) - wheel_id: u8, - /// Effect type (0=constant, 1=spring, 2=damper, 3=friction) - effect_type: u8, - /// Force magnitude (-1.0 to 1.0 mapped to -32768 to 32767) - magnitude: i16, - /// Duration in milliseconds - duration_ms: u16, - /// Additional parameters based on effect type - param1: i16, - param2: i16, - }, - /// Unknown output event - Unknown { event_type: u32, data: Vec }, -} - -/// Decoder for output events from server -pub struct OutputDecoder { - protocol_version: u8, -} - -impl OutputDecoder { - pub fn new() -> Self { - Self { - protocol_version: 2, - } - } - - /// Set protocol version (received from handshake) - pub fn set_protocol_version(&mut self, version: u8) { - self.protocol_version = version; - } - - /// Decode an output event from binary data - /// Returns None if data is not a recognized output event - pub fn decode(&self, data: &[u8]) -> Option { - if data.is_empty() { - return None; - } - - let mut buf = data; - - // Protocol v3+ has wrapper byte - if self.protocol_version > 2 && !buf.is_empty() && buf[0] == 0x22 { - buf = &buf[1..]; - } - - // Need at least 4 bytes for event type - if buf.len() < 4 { - return None; - } - - // Read event type (4 bytes LE) - let event_type = u32::from_le_bytes([buf[0], buf[1], buf[2], buf[3]]); - let payload = &buf[4..]; - - match event_type { - OUTPUT_RUMBLE => self.decode_rumble(payload), - OUTPUT_FORCE_FEEDBACK => self.decode_force_feedback(payload), - _ => { - // Only return Some for actual output events (rumble/FFB) - // Return None for everything else to allow other handlers to process - // (e.g., handshake messages which start with 0x0e or 0x020e) - None - } - } - } - - /// Decode rumble event - /// Expected format (after type): - /// [0x00] Controller ID: 1B - /// [0x01] Left motor: 1B (0-255) - /// [0x02] Right motor: 1B (0-255) - /// [0x03] Padding: 1B - /// [0x04] Duration: 2B LE (milliseconds) - fn decode_rumble(&self, payload: &[u8]) -> Option { - if payload.len() < 6 { - debug!("Rumble payload too short: {} bytes", payload.len()); - return None; - } - - let controller_id = payload[0]; - let left_motor = payload[1]; - let right_motor = payload[2]; - // payload[3] is padding - let duration_ms = u16::from_le_bytes([payload[4], payload[5]]); - - debug!( - "Decoded rumble: controller={}, left={}, right={}, duration={}ms", - controller_id, left_motor, right_motor, duration_ms - ); - - Some(OutputEvent::Rumble { - controller_id, - left_motor, - right_motor, - duration_ms, - }) - } - - /// Decode force feedback event - /// Expected format (after type): - /// [0x00] Wheel ID: 1B - /// [0x01] Effect type: 1B (0=constant, 1=spring, 2=damper, 3=friction) - /// [0x02] Magnitude: 2B LE signed (-32768 to 32767) - /// [0x04] Duration: 2B LE (milliseconds) - /// [0x06] Param1: 2B LE signed (effect-specific) - /// [0x08] Param2: 2B LE signed (effect-specific) - fn decode_force_feedback(&self, payload: &[u8]) -> Option { - if payload.len() < 10 { - debug!("FFB payload too short: {} bytes", payload.len()); - return None; - } - - let wheel_id = payload[0]; - let effect_type = payload[1]; - let magnitude = i16::from_le_bytes([payload[2], payload[3]]); - let duration_ms = u16::from_le_bytes([payload[4], payload[5]]); - let param1 = i16::from_le_bytes([payload[6], payload[7]]); - let param2 = i16::from_le_bytes([payload[8], payload[9]]); - - debug!( - "Decoded FFB: wheel={}, type={}, magnitude={}, duration={}ms, p1={}, p2={}", - wheel_id, effect_type, magnitude, duration_ms, param1, param2 - ); - - Some(OutputEvent::ForceFeedback { - wheel_id, - effect_type, - magnitude, - duration_ms, - param1, - param2, - }) - } -} - -impl Default for OutputDecoder { - fn default() -> Self { - Self::new() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_mouse_move_encoding() { - let mut encoder = InputEncoder::new(); - let event = InputEvent::MouseMove { - dx: -1, - dy: 5, - timestamp_us: 12345, - }; - let encoded = encoder.encode(&event); - - assert_eq!(encoded.len(), 22); - // Type 7 in LE - assert_eq!(&encoded[0..4], &[0x07, 0x00, 0x00, 0x00]); - } - - #[test] - fn test_heartbeat_encoding() { - let mut encoder = InputEncoder::new(); - let event = InputEvent::Heartbeat; - let encoded = encoder.encode(&event); - - assert_eq!(encoded.len(), 4); - assert_eq!(&encoded[0..4], &[0x02, 0x00, 0x00, 0x00]); - } - - #[test] - fn test_rumble_decoding() { - let decoder = OutputDecoder::new(); - - // Type 13 (rumble) + payload - let data: Vec = vec![ - 0x0D, 0x00, 0x00, 0x00, // Type 13 LE - 0x00, // Controller ID - 0xFF, // Left motor (max) - 0x80, // Right motor (half) - 0x00, // Padding - 0xE8, 0x03, // Duration 1000ms LE - ]; - - let event = decoder.decode(&data).unwrap(); - match event { - OutputEvent::Rumble { - controller_id, - left_motor, - right_motor, - duration_ms, - } => { - assert_eq!(controller_id, 0); - assert_eq!(left_motor, 255); - assert_eq!(right_motor, 128); - assert_eq!(duration_ms, 1000); - } - _ => panic!("Expected Rumble event"), - } - } - - #[test] - fn test_ffb_decoding() { - let decoder = OutputDecoder::new(); - - // Type 14 (FFB) + payload - let data: Vec = vec![ - 0x0E, 0x00, 0x00, 0x00, // Type 14 LE - 0x00, // Wheel ID - 0x00, // Effect type (constant) - 0x00, 0x40, // Magnitude 16384 (0.5 force) - 0xF4, 0x01, // Duration 500ms LE - 0x00, 0x00, // Param1 - 0x00, 0x00, // Param2 - ]; - - let event = decoder.decode(&data).unwrap(); - match event { - OutputEvent::ForceFeedback { - wheel_id, - effect_type, - magnitude, - duration_ms, - .. - } => { - assert_eq!(wheel_id, 0); - assert_eq!(effect_type, 0); - assert_eq!(magnitude, 16384); - assert_eq!(duration_ms, 500); - } - _ => panic!("Expected ForceFeedback event"), - } - } -} diff --git a/opennow-streamer/src/webrtc/mod.rs b/opennow-streamer/src/webrtc/mod.rs deleted file mode 100644 index ad1c24f..0000000 --- a/opennow-streamer/src/webrtc/mod.rs +++ /dev/null @@ -1,1099 +0,0 @@ -//! WebRTC Module -//! -//! WebRTC peer connection, signaling, and data channels for GFN streaming. - -mod datachannel; -mod peer; -mod sdp; -mod signaling; - -pub use datachannel::*; -pub use peer::{request_keyframe, NetworkStats, WebRtcEvent, WebRtcPeer}; -pub use sdp::*; -pub use signaling::{GfnSignaling, IceCandidate, SignalingEvent}; -// StreamingResult is defined in this module and exported automatically -use log::{debug, error, info, warn}; -use std::sync::Arc; -use tokio::sync::mpsc; -use webrtc::ice_transport::ice_server::RTCIceServer; - -use crate::app::{SessionInfo, Settings, SharedFrame, VideoCodec}; - -/// Result of a streaming session - indicates why the stream ended -#[derive(Debug, Clone)] -pub enum StreamingResult { - /// Stream ended normally (user stopped or server disconnected gracefully) - Normal, - /// Stream failed due to an error - Error(String), - /// Stream was interrupted by SSRC change (resolution change on server) - /// Contains the stall duration in milliseconds before detection - SsrcChangeDetected { stall_duration_ms: u64 }, -} -use crate::input::{ControllerManager, FfbEffectType, G29FfbManager, InputHandler, WheelManager}; -use crate::media::{ - AudioDecoder, AudioPlayer, DepacketizerCodec, RtpDepacketizer, StreamStats, UnifiedVideoDecoder, -}; - -/// Active streaming session -pub struct StreamingSession { - pub signaling: Option, - pub peer: Option, - pub connected: bool, - pub stats: StreamStats, - pub input_ready: bool, -} - -impl StreamingSession { - pub fn new() -> Self { - Self { - signaling: None, - peer: None, - connected: false, - stats: StreamStats::default(), - input_ready: false, - } - } -} - -impl Default for StreamingSession { - fn default() -> Self { - Self::new() - } -} - -/// Build nvstSdp string with streaming parameters -/// Based on official GFN browser client format -fn build_nvst_sdp( - ice_ufrag: &str, - ice_pwd: &str, - fingerprint: &str, - width: u32, - height: u32, - fps: u32, - max_bitrate_kbps: u32, -) -> String { - let min_bitrate_kbps = std::cmp::min(10000, max_bitrate_kbps / 10); - let initial_bitrate_kbps = max_bitrate_kbps / 2; - - let is_high_fps = fps >= 120; - let is_120_fps = fps == 120; - let is_240_fps = fps >= 240; - - let mut lines = vec![ - "v=0".to_string(), - "o=SdpTest test_id_13 14 IN IPv4 127.0.0.1".to_string(), - "s=-".to_string(), - "t=0 0".to_string(), - format!("a=general.icePassword:{}", ice_pwd), - format!("a=general.iceUserNameFragment:{}", ice_ufrag), - format!("a=general.dtlsFingerprint:{}", fingerprint), - "m=video 0 RTP/AVP".to_string(), - "a=msid:fbc-video-0".to_string(), - // FEC settings - "a=vqos.fec.rateDropWindow:10".to_string(), - "a=vqos.fec.minRequiredFecPackets:2".to_string(), - "a=vqos.fec.repairMinPercent:5".to_string(), - "a=vqos.fec.repairPercent:5".to_string(), - "a=vqos.fec.repairMaxPercent:35".to_string(), - ]; - - // DRC/DFC settings based on FPS - // Always disable DRC to allow full bitrate - lines.push("a=vqos.drc.enable:0".to_string()); - if is_high_fps { - lines.push("a=vqos.dfc.enable:1".to_string()); - lines.push("a=vqos.dfc.decodeFpsAdjPercent:85".to_string()); - lines.push("a=vqos.dfc.targetDownCooldownMs:250".to_string()); - lines.push("a=vqos.dfc.dfcAlgoVersion:2".to_string()); - lines.push(format!( - "a=vqos.dfc.minTargetFps:{}", - if is_120_fps { 100 } else { 60 } - )); - } - - // Video encoder settings - lines.extend(vec![ - "a=video.dx9EnableNv12:1".to_string(), - "a=video.dx9EnableHdr:1".to_string(), - "a=vqos.qpg.enable:1".to_string(), - "a=vqos.resControl.qp.qpg.featureSetting:7".to_string(), - "a=bwe.useOwdCongestionControl:1".to_string(), - "a=video.enableRtpNack:1".to_string(), - "a=vqos.bw.txRxLag.minFeedbackTxDeltaMs:200".to_string(), - "a=vqos.drc.bitrateIirFilterFactor:18".to_string(), - "a=video.packetSize:1140".to_string(), - "a=packetPacing.minNumPacketsPerGroup:15".to_string(), - ]); - - // High FPS optimizations - if is_high_fps { - lines.extend(vec![ - "a=bwe.iirFilterFactor:8".to_string(), - "a=video.encoderFeatureSetting:47".to_string(), - "a=video.encoderPreset:6".to_string(), - "a=vqos.resControl.cpmRtc.badNwSkipFramesCount:600".to_string(), - "a=vqos.resControl.cpmRtc.decodeTimeThresholdMs:9".to_string(), - format!( - "a=video.fbcDynamicFpsGrabTimeoutMs:{}", - if is_120_fps { 6 } else { 18 } - ), - format!( - "a=vqos.resControl.cpmRtc.serverResolutionUpdateCoolDownCount:{}", - if is_120_fps { 6000 } else { 12000 } - ), - ]); - } - - // 240+ FPS optimizations - if is_240_fps { - lines.extend(vec![ - "a=video.enableNextCaptureMode:1".to_string(), - "a=vqos.maxStreamFpsEstimate:240".to_string(), - "a=video.videoSplitEncodeStripsPerFrame:3".to_string(), - "a=video.updateSplitEncodeStateDynamically:1".to_string(), - ]); - } - - // Out of focus and additional settings - lines.extend(vec![ - "a=vqos.adjustStreamingFpsDuringOutOfFocus:1".to_string(), - "a=vqos.resControl.cpmRtc.ignoreOutOfFocusWindowState:1".to_string(), - "a=vqos.resControl.perfHistory.rtcIgnoreOutOfFocusWindowState:1".to_string(), - // Disable CPM-based resolution changes (prevents SSRC switches) - // featureMask: 0 = disable all CPM features, 3 = enable some - "a=vqos.resControl.cpmRtc.featureMask:0".to_string(), - // Disable resolution scaling entirely - "a=vqos.resControl.cpmRtc.enable:0".to_string(), - // Never scale down resolution - "a=vqos.resControl.cpmRtc.minResolutionPercent:100".to_string(), - // Infinite cooldown to prevent resolution changes - "a=vqos.resControl.cpmRtc.resolutionChangeHoldonMs:999999".to_string(), - format!( - "a=packetPacing.numGroups:{}", - if is_120_fps { 3 } else { 5 } - ), - "a=packetPacing.maxDelayUs:1000".to_string(), - "a=packetPacing.minNumPacketsFrame:10".to_string(), - // NACK settings - "a=video.rtpNackQueueLength:1024".to_string(), - "a=video.rtpNackQueueMaxPackets:512".to_string(), - "a=video.rtpNackMaxPacketCount:25".to_string(), - // Resolution/quality - "a=vqos.drc.qpMaxResThresholdAdj:4".to_string(), - "a=vqos.grc.qpMaxResThresholdAdj:4".to_string(), - "a=vqos.drc.iirFilterFactor:100".to_string(), - // Viewport and FPS - format!("a=video.clientViewportWd:{}", width), - format!("a=video.clientViewportHt:{}", height), - format!("a=video.maxFPS:{}", fps), - // Bitrate - critical for achieving high bitrates - // Initial bitrate should be high to avoid slow ramp-up - format!("a=video.initialBitrateKbps:{}", max_bitrate_kbps * 3 / 4), // Start at 75% of max - format!("a=video.initialPeakBitrateKbps:{}", max_bitrate_kbps), - format!("a=vqos.bw.maximumBitrateKbps:{}", max_bitrate_kbps), - format!("a=vqos.bw.minimumBitrateKbps:{}", min_bitrate_kbps), - // Peak bitrate settings - these are critical for allowing bitrate above 100Mbps - format!("a=vqos.bw.peakBitrateKbps:{}", max_bitrate_kbps), - format!("a=vqos.bw.serverPeakBitrateKbps:{}", max_bitrate_kbps), - // Bandwidth estimation settings - disable conservative limiting - "a=vqos.bw.enableBandwidthEstimation:1".to_string(), - "a=vqos.bw.disableBitrateLimit:1".to_string(), - // GRC (Global Rate Control) settings - allow full bitrate - format!("a=vqos.grc.maximumBitrateKbps:{}", max_bitrate_kbps), - "a=vqos.grc.enable:0".to_string(), // Disable GRC limiting - // Encoder settings - "a=video.maxNumReferenceFrames:4".to_string(), - "a=video.mapRtpTimestampsToFrames:1".to_string(), - "a=video.encoderCscMode:3".to_string(), - "a=video.scalingFeature1:0".to_string(), - "a=video.prefilterParams.prefilterModel:0".to_string(), - // Audio track - "m=audio 0 RTP/AVP".to_string(), - "a=msid:audio".to_string(), - // Mic track - "m=mic 0 RTP/AVP".to_string(), - "a=msid:mic".to_string(), - // Input/application track - "m=application 0 RTP/AVP".to_string(), - "a=msid:input_1".to_string(), - "a=ri.partialReliableThresholdMs:300".to_string(), - "".to_string(), - ]); - - lines.join("\n") -} - -/// Extract ICE credentials from SDP -fn extract_ice_credentials(sdp: &str) -> (String, String, String) { - let ufrag = sdp - .lines() - .find(|l| l.starts_with("a=ice-ufrag:")) - .map(|l| l.trim_start_matches("a=ice-ufrag:").to_string()) - .unwrap_or_default(); - - let pwd = sdp - .lines() - .find(|l| l.starts_with("a=ice-pwd:")) - .map(|l| l.trim_start_matches("a=ice-pwd:").to_string()) - .unwrap_or_default(); - - let fingerprint = sdp - .lines() - .find(|l| l.starts_with("a=fingerprint:sha-256 ")) - .map(|l| l.trim_start_matches("a=fingerprint:sha-256 ").to_string()) - .unwrap_or_default(); - - (ufrag, pwd, fingerprint) -} - -/// Extract public IP from server hostname (e.g., "95-178-87-234.zai..." -> "95.178.87.234") -/// Also handles direct IP strings or IPs embedded in signaling URLs -fn extract_public_ip(input: &str) -> Option { - // Check for standard IP-like patterns with dashes (e.g. 80-250-97-38) - let re_dash = regex::Regex::new(r"(\d{1,3})-(\d{1,3})-(\d{1,3})-(\d{1,3})").ok()?; - if let Some(captures) = re_dash.captures(input) { - let ip = format!( - "{}.{}.{}.{}", - &captures[1], &captures[2], &captures[3], &captures[4] - ); - return Some(ip); - } - - // Check for standard IP patterns (e.g. 80.250.97.38) - let re_dot = regex::Regex::new(r"(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})").ok()?; - if let Some(captures) = re_dot.captures(input) { - return Some(captures[0].to_string()); - } - - None -} - -/// Run the streaming session -/// Returns a `StreamingResult` indicating how/why the session ended -pub async fn run_streaming( - session_info: SessionInfo, - settings: Settings, - shared_frame: Arc, - stats_tx: mpsc::Sender, - input_handler: Arc, -) -> StreamingResult { - info!( - "Starting streaming to {} with session {}", - session_info.server_ip, session_info.session_id - ); - - let (width, height) = settings.resolution_tuple(); - let fps = settings.fps; - let max_bitrate = settings.max_bitrate_kbps(); - let codec = settings.codec; - let _codec_str = codec.as_str().to_string(); - - // Create signaling client - let (sig_event_tx, mut sig_event_rx) = mpsc::channel::(64); - let server_ip = session_info - .signaling_url - .as_ref() - .and_then(|url| url.split("://").nth(1).and_then(|s| s.split('/').next())) - .unwrap_or(&session_info.server_ip) - .to_string(); - - let mut signaling = GfnSignaling::new( - server_ip.clone(), - session_info.session_id.clone(), - sig_event_tx, - ); - - // Connect to signaling - if let Err(e) = signaling.connect().await { - return StreamingResult::Error(format!("Failed to connect signaling: {}", e)); - } - info!("Signaling connected"); - - // Create WebRTC peer - let (peer_event_tx, mut peer_event_rx) = mpsc::channel(64); - let mut peer = WebRtcPeer::new(peer_event_tx); - - // Video decoder - use async mode for non-blocking decode - // Decoded frames are written directly to SharedFrame by the decoder thread - // Uses UnifiedVideoDecoder to support both FFmpeg and native DXVA backends - let (mut video_decoder, mut decode_stats_rx) = - match UnifiedVideoDecoder::new_async(codec, settings.decoder_backend, shared_frame.clone()) - { - Ok(decoder) => decoder, - Err(e) => { - return StreamingResult::Error(format!("Failed to create video decoder: {}", e)) - } - }; - - // Create RTP depacketizer with correct codec - let depacketizer_codec = match codec { - VideoCodec::H264 => DepacketizerCodec::H264, - VideoCodec::H265 => DepacketizerCodec::H265, - VideoCodec::AV1 => DepacketizerCodec::AV1, - }; - let mut rtp_depacketizer = RtpDepacketizer::with_codec(depacketizer_codec); - info!("RTP depacketizer using {:?} mode", depacketizer_codec); - - let mut audio_decoder = match AudioDecoder::new(48000, 2) { - Ok(decoder) => decoder, - Err(e) => return StreamingResult::Error(format!("Failed to create audio decoder: {}", e)), - }; - - // Get the sample receiver from the decoder for async operation - let audio_sample_rx = audio_decoder.take_sample_receiver(); - - // Audio player thread - receives decoded samples and plays them - // Uses larger jitter buffer (150ms) to handle network timing variations - std::thread::spawn(move || { - if let Ok(audio_player) = AudioPlayer::new(48000, 2) { - info!("Audio player thread started (async mode with jitter buffer)"); - if let Some(mut rx) = audio_sample_rx { - let mut total_samples: u64 = 0; - let mut log_interval = std::time::Instant::now(); - while let Some(samples) = rx.blocking_recv() { - if total_samples == 0 { - info!( - "First audio samples received by player: {} samples", - samples.len() - ); - } - total_samples += samples.len() as u64; - audio_player.push_samples(&samples); - - // Log buffer status periodically - if log_interval.elapsed().as_secs() >= 5 { - info!( - "Audio: {} total samples played, buffer level: {} samples", - total_samples, - audio_player.buffer_available() - ); - log_interval = std::time::Instant::now(); - } - } - } - } else { - warn!("Failed to create audio player - audio disabled"); - } - }); - - // Stats tracking - let mut stats = StreamStats::default(); - let mut last_stats_time = std::time::Instant::now(); - let mut frames_received: u64 = 0; - let mut frames_decoded: u64 = 0; - let frames_dropped: u64 = 0; - let mut bytes_received: u64 = 0; - let mut last_frames_decoded: u64 = 0; // For actual FPS calculation - - // Pipeline latency tracking (receive to decode complete) - let mut pipeline_latency_sum: f64 = 0.0; - let mut pipeline_latency_count: u64 = 0; - - // Input latency tracking (event creation to transmission) - let mut input_latency_sum: f64 = 0.0; - let mut input_latency_count: u64 = 0; - - // Input rate tracking (events per second) - let mut input_events_this_period: u64 = 0; - - // Input state - use atomic for cross-task communication - // input_ready_flag and input_protocol_version_shared are created later with the input task - - // Input channel - connect InputHandler to the streaming loop - // Reduced buffer (32) to prevent latency buildup (buffer bloat) - // If consumer is slow, we want to conflate events, not buffer them - let (input_event_tx, input_event_rx) = mpsc::channel::(32); - input_handler.set_event_sender(input_event_tx.clone()); - - // Also set raw input sender for direct mouse events (Windows/macOS) - #[cfg(any(target_os = "windows", target_os = "macos"))] - crate::input::set_raw_input_sender(input_event_tx.clone()); - - info!("Input handler connected to streaming loop"); - - // Initialize and start ControllerManager (gamepads via gilrs) - let controller_manager = Arc::new(ControllerManager::new()); - controller_manager.set_event_sender(input_event_tx.clone()); - controller_manager.start(); - info!("Controller manager started"); - - // Initialize and start WheelManager (racing wheels via Windows.Gaming.Input) - // WheelManager detects dedicated racing wheels and provides proper axis separation - // (wheel rotation, throttle, brake, clutch, handbrake) mapped to gamepad format - let wheel_manager = Arc::new(WheelManager::new()); - wheel_manager.set_event_sender(input_event_tx.clone()); - wheel_manager.start(); - if wheel_manager.has_wheels() { - info!("Racing wheel manager started - wheel input active"); - // Initialize force feedback for detected wheels - for i in 0..wheel_manager.wheel_count() { - if wheel_manager.init_force_feedback(i) { - info!("Force feedback initialized for wheel {}", i); - } - } - } else { - info!("No racing wheels detected - wheels will be handled as gamepads via gilrs"); - } - - // Initialize G29 force feedback manager (HID-based, works in PS3 mode) - // This provides FFB support for Logitech G29 wheels that aren't detected by Windows.Gaming.Input - let g29_ffb = Arc::new(G29FfbManager::new()); - let g29_connected = g29_ffb.init(); - if g29_connected { - info!("G29 force feedback initialized via HID"); - } - - // Output decoder for force feedback / rumble messages from server - let mut output_decoder = OutputDecoder::new(); - - // Channel for input task to send encoded packets to the WebRTC peer - // This decouples input processing from video decoding completely - // Tuple: (encoded_data, is_mouse, is_controller, latency_us) - let (input_packet_tx, mut input_packet_rx) = mpsc::channel::<(Vec, bool, bool, u64)>(1024); - - // Stats interval timer (must be created OUTSIDE the loop to persist across iterations) - let mut stats_interval = tokio::time::interval(std::time::Duration::from_secs(1)); - - // Spawn dedicated input processing task - completely decoupled from video/signaling - // This ensures mouse/keyboard events are processed immediately without being blocked - // by video decoding or network operations - let input_packet_tx_clone = input_packet_tx.clone(); - let input_ready_flag = Arc::new(std::sync::atomic::AtomicBool::new(false)); - let input_ready_flag_clone = input_ready_flag.clone(); - let input_protocol_version_shared = Arc::new(std::sync::atomic::AtomicU8::new(0)); - let input_protocol_version_clone = input_protocol_version_shared.clone(); - - tokio::spawn(async move { - let mut input_encoder = InputEncoder::new(); - let mut input_event_rx = input_event_rx; - - loop { - match input_event_rx.recv().await { - Some(event) => { - // Only process if input is ready (handshake complete) - if !input_ready_flag_clone.load(std::sync::atomic::Ordering::Acquire) { - continue; - } - - // Update encoder protocol version if changed - let version = - input_protocol_version_clone.load(std::sync::atomic::Ordering::Relaxed); - input_encoder.set_protocol_version(version); - - // Handle ClipboardPaste specially - expand into multiple key events - if let InputEvent::ClipboardPaste { ref text } = event { - info!("Processing clipboard paste: {} chars", text.chars().count()); - let packets = datachannel::encode_clipboard_paste(&mut input_encoder, text); - for encoded in packets { - // Send each key event packet - // Clipboard paste uses keyboard channel (reliable) - if input_packet_tx_clone - .try_send((encoded, false, false, 0)) - .is_err() - { - warn!("Input channel full during clipboard paste"); - } - // Small delay between packets to avoid overwhelming the server - // This is handled by timestamps in the packets themselves - } - continue; // Don't process as normal event - } - - // Extract event timestamp for latency calculation - let event_timestamp_us = match &event { - InputEvent::KeyDown { timestamp_us, .. } - | InputEvent::KeyUp { timestamp_us, .. } - | InputEvent::MouseMove { timestamp_us, .. } - | InputEvent::MouseButtonDown { timestamp_us, .. } - | InputEvent::MouseButtonUp { timestamp_us, .. } - | InputEvent::MouseWheel { timestamp_us, .. } - | InputEvent::Gamepad { timestamp_us, .. } => *timestamp_us, - InputEvent::Heartbeat | InputEvent::ClipboardPaste { .. } => 0, - }; - - // Calculate input latency (time from event creation to now) - let now_us = crate::input::get_timestamp_us(); - let latency_us = now_us.saturating_sub(event_timestamp_us); - - // Encode the event - let encoded = input_encoder.encode(&event); - - // Determine if this is a mouse event - let is_mouse = matches!( - &event, - InputEvent::MouseMove { .. } - | InputEvent::MouseButtonDown { .. } - | InputEvent::MouseButtonUp { .. } - | InputEvent::MouseWheel { .. } - ); - - // Determine if this is a gamepad/controller event - let is_controller = matches!(&event, InputEvent::Gamepad { .. }); - - // Send to main loop for WebRTC transmission - // Use try_send to never block the input thread - if input_packet_tx_clone - .try_send((encoded, is_mouse, is_controller, latency_us)) - .is_err() - { - // Channel full - this is fine, old packets can be dropped for mouse - } - } - None => { - // Channel closed, exit task - break; - } - } - } - debug!("Input processing task ended"); - }); - - // Main event loop - no longer processes input directly - loop { - tokio::select! { - // Process encoded input packets from the input task (high priority) - biased; - - Some((encoded, is_mouse, is_controller, latency_us)) = input_packet_rx.recv() => { - // Track input latency and count for stats - input_events_this_period += 1; - if latency_us > 0 { - input_latency_sum += latency_us as f64; - input_latency_count += 1; - } - - // Log first few mouse events to verify flow - static MOUSE_LOG_COUNT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); - if is_mouse { - let count = MOUSE_LOG_COUNT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - if count < 10 { - info!("Sending mouse #{}: {} bytes, latency {}us", count, encoded.len(), latency_us); - } - } - - if is_controller { - // Controller input (Input Channel V1) - // "input_channel_v1 needs to be only controller" - let _ = peer.send_controller_input(&encoded).await; - } else if is_mouse { - // Log if mouse channel is ready - static MOUSE_READY_LOGGED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); - if !MOUSE_READY_LOGGED.load(std::sync::atomic::Ordering::Relaxed) { - info!("Mouse events will use PARTIALLY RELIABLE channel (lower latency)"); - MOUSE_READY_LOGGED.store(true, std::sync::atomic::Ordering::Relaxed); - } - // Send mouse on PARTIALLY RELIABLE channel (unordered, low latency) - let _ = peer.send_mouse_input(&encoded).await; - } else { - // Keyboard events - // Currently uses send_input (V1) - // If user requires V1 to be *strictly* controller, we might need to route keyboard elsewhere? - // But usually keyboard shares reliable channel. For now, keep it here. - let _ = peer.send_input(&encoded).await; - } - } - Some(event) = sig_event_rx.recv() => { - match event { - SignalingEvent::SdpOffer(sdp) => { - info!("Received SDP offer, length: {}", sdp.len()); - - // Detect codec to use - let codec = match settings.codec { - VideoCodec::H264 => "H264", - VideoCodec::H265 => "H265", - VideoCodec::AV1 => "AV1", - }; - - info!("Preferred codec: {}", codec); - - // Use media_connection_info IP first, then server_ip - let public_ip = session_info.media_connection_info.as_ref() - .and_then(|mci| extract_public_ip(&mci.ip)) - .or_else(|| extract_public_ip(&session_info.server_ip)); - - // Modify SDP with extracted IP - let modified_sdp = if let Some(ref ip) = public_ip { - fix_server_ip(&sdp, ip) - } else { - sdp.clone() - }; - - // CRITICAL: Inject provisional SSRCs (2, 3, 4) for video - // GFN server uses sequential SSRCs when resolution changes, but - // webrtc-rs can't handle undeclared SSRCs without MID extensions. - // This is based on reverse-engineering of official GFN client (Bifrost2.dll). - let modified_sdp = inject_provisional_ssrcs(&modified_sdp); - - // Prefer codec - let modified_sdp = prefer_codec(&modified_sdp, &settings.codec); - - // CRITICAL: Create input channel BEFORE SDP negotiation (per GFN protocol) - info!("Creating input channel BEFORE SDP negotiation..."); - - // Align with official client: Use ICE servers from SessionInfo (TURN/STUN) - // This corresponds to `iceServerConfiguration` in the CloudMatch response. - let mut ice_servers = Vec::new(); - - // Convert SessionInfo ICE servers to RTCIceServer - for server in &session_info.ice_servers { - let mut s = RTCIceServer { - urls: server.urls.clone(), - ..Default::default() - }; - if let Some(user) = &server.username { - s.username = user.clone(); - } - if let Some(cred) = &server.credential { - s.credential = cred.clone(); - } - ice_servers.push(s); - } - - // Always add default STUN servers as fallback (Alliance robustness) - ice_servers.push(RTCIceServer { - urls: vec!["stun:s1.stun.gamestream.nvidia.com:19308".to_string()], - ..Default::default() - }); - ice_servers.push(RTCIceServer { - urls: vec![ - "stun:stun.l.google.com:19302".to_string(), - "stun:stun1.l.google.com:19302".to_string() - ], - ..Default::default() - }); - - if ice_servers.len() <= 2 { - info!("Using default/fallback ICE servers only"); - } else { - info!("Using {} ICE servers (session + fallback)", ice_servers.len()); - } - - // Handle offer and create answer - match peer.handle_offer(&modified_sdp, ice_servers).await { - Ok(answer_sdp) => { - // Create input channel - if let Err(e) = peer.create_input_channel().await { - warn!("Failed to create input channel: {}", e); - } - - // Extract ICE credentials from our answer - let (ufrag, pwd, fingerprint) = extract_ice_credentials(&answer_sdp); - - // Build rich GFN-specific SDP (nvstSdp) - let nvst_sdp_content = build_nvst_sdp( - &ufrag, &pwd, &fingerprint, - width, height, fps, max_bitrate - ); - info!("Generated nvstSdp, length: {}", nvst_sdp_content.len()); - - // Use raw nvstSdp string (no wrapper object) - if let Err(e) = signaling.send_answer(&answer_sdp, Some(&nvst_sdp_content)).await { - error!("Failed to send SDP answer: {}", e); - } - - // For resume flow or Alliance partners (manual candidate needed) - if let Some(ref mci) = session_info.media_connection_info { - info!("Using media port {} from session API", mci.port); - - // EXTRACT RAW IP from hostname (needed for valid ICE candidate) - // Use extract_public_ip which handles "x-x-x-x" format or direct IP - let raw_ip = extract_public_ip(&mci.ip) - .or_else(|| { - // Fallback: try to resolve hostname - use std::net::ToSocketAddrs; - format!("{}:{}", mci.ip, mci.port) - .to_socket_addrs() - .ok()? - .next() - .map(|addr| addr.ip().to_string()) - }) - .unwrap_or_else(|| mci.ip.clone()); - - let candidate = format!( - "candidate:1 1 udp 2130706431 {} {} typ host", - raw_ip, mci.port - ); - info!("Adding manual ICE candidate: {}", candidate); - - // Extract server ufrag from offer (needed for ice-lite) - let (server_ufrag, _, _) = extract_ice_credentials(&sdp); - - if let Err(e) = peer.add_ice_candidate(&candidate, Some("0"), Some(0), Some(server_ufrag.clone())).await { - warn!("Failed to add manual ICE candidate: {}", e); - // Try other mids just in case - for mid in ["1", "2", "3"] { - if peer.add_ice_candidate(&candidate, Some(mid), Some(mid.parse().unwrap_or(0)), Some(server_ufrag.clone())).await.is_ok() { - info!("Added ICE candidate with sdpMid={}", mid); - break; - } - } - } - } else { - info!("No media_connection_info - waiting for ICE negotiation"); - } - - // Update stats with codec info - stats.codec = codec.to_string(); - stats.resolution = format!("{}x{}", width, height); - stats.target_fps = fps; - } - - Err(e) => { - error!("Failed to handle offer: {}", e); - } - } - } - SignalingEvent::IceCandidate(candidate) => { - info!("Received trickle ICE candidate"); - if let Err(e) = peer.add_ice_candidate( - &candidate.candidate, - candidate.sdp_mid.as_deref(), - candidate.sdp_mline_index.map(|i| i as u16), - None - ).await { - warn!("Failed to add ICE candidate: {}", e); - } - } - SignalingEvent::Connected => { - info!("Signaling connected event"); - } - SignalingEvent::Disconnected(reason) => { - info!("Signaling disconnected: {}", reason); - break; - } - SignalingEvent::Error(e) => { - error!("Signaling error: {}", e); - break; - } - } - } - Some(event) = peer_event_rx.recv() => { - match event { - WebRtcEvent::Connected => { - info!("=== WebRTC CONNECTED ==="); - stats.gpu_type = session_info.gpu_type.clone().unwrap_or_default(); - } - WebRtcEvent::Disconnected => { - warn!("WebRTC disconnected"); - break; - } - WebRtcEvent::VideoFrame { payload, rtp_timestamp: _, marker } => { - frames_received += 1; - bytes_received += payload.len() as u64; - let packet_receive_time = std::time::Instant::now(); - - // Only log first packet - if frames_received == 1 { - info!("First video RTP packet received: {} bytes", payload.len()); - } - - // Handle codec-specific depacketization - match depacketizer_codec { - DepacketizerCodec::AV1 => { - // AV1: Use specialized OBU accumulation - rtp_depacketizer.process_av1_raw(&payload); - - // On marker bit, flush pending OBU and get complete frame - if marker { - rtp_depacketizer.flush_pending_obu(); - if let Some(frame_data) = rtp_depacketizer.take_accumulated_frame() { - if let Err(e) = video_decoder.decode_async(&frame_data, packet_receive_time) { - warn!("AV1 decode async failed: {}", e); - } - } - } - } - DepacketizerCodec::H264 | DepacketizerCodec::H265 => { - // H.264/H.265: depacketize RTP and accumulate NAL units - let nal_units = rtp_depacketizer.process(&payload); - - // Accumulate NAL units until marker bit (end of frame) - // Each frame consists of multiple NAL units that must be sent together - for nal_unit in nal_units { - rtp_depacketizer.accumulate_nal(nal_unit); - } - - // On marker bit, we have a complete Access Unit - send to decoder - if marker { - if let Some(frame_data) = rtp_depacketizer.take_nal_frame() { - if let Err(e) = video_decoder.decode_async(&frame_data, packet_receive_time) { - warn!("Decode async failed: {}", e); - } - } - } - } - } - } - WebRtcEvent::AudioFrame(rtp_data) => { - // Async decode - non-blocking, samples go directly to audio player - static AUDIO_PACKET_COUNT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); - let count = AUDIO_PACKET_COUNT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - if count == 0 { - info!("First audio packet received: {} bytes", rtp_data.len()); - } else if count % 500 == 0 { - debug!("Audio packets received: {}", count); - } - audio_decoder.decode_async(&rtp_data); - } - WebRtcEvent::DataChannelOpen(label) => { - info!("Data channel opened: {}", label); - if label.contains("input") { - info!("Input channel ready, waiting for handshake..."); - } - } - WebRtcEvent::DataChannelMessage(label, data) => { - debug!("Data channel '{}' message: {} bytes", label, data.len()); - - // Try to decode as output event (force feedback / rumble) - if let Some(output_event) = output_decoder.decode(&data) { - match output_event { - OutputEvent::Rumble { - controller_id, - left_motor, - right_motor, - duration_ms, - } => { - debug!( - "Rumble event: controller={}, left={}, right={}, duration={}ms", - controller_id, left_motor, right_motor, duration_ms - ); - // Queue rumble effect on the controller - controller_manager.queue_rumble( - controller_id, - left_motor, - right_motor, - duration_ms, - ); - } - OutputEvent::ForceFeedback { - wheel_id, - effect_type, - magnitude, - duration_ms, - .. - } => { - debug!( - "FFB event: wheel={}, type={}, magnitude={}, duration={}ms", - wheel_id, effect_type, magnitude, duration_ms - ); - // Convert magnitude from i16 (-32768 to 32767) to f64 (-1.0 to 1.0) - let mag_normalized = magnitude as f64 / 32767.0; - - // Try Windows.Gaming.Input first (for wheels that support it) - if wheel_manager.has_wheels() { - wheel_manager.apply_force_feedback( - wheel_id as usize, - FfbEffectType::from(effect_type), - mag_normalized, - duration_ms, - ); - } else if g29_ffb.is_connected() { - // Fallback to G29 HID-based FFB - g29_ffb.apply_constant_force(mag_normalized); - } - } - OutputEvent::Unknown { .. } => { - // Should not happen - decoder only returns Rumble/ForceFeedback - } - } - } - - // Handle input handshake - if data.len() >= 2 { - let first_word = u16::from_le_bytes([data[0], data.get(1).copied().unwrap_or(0)]); - let mut protocol_version: u16 = 0; - - if first_word == 526 { - // New format: 0x020E (526 LE) - protocol_version = data.get(2..4) - .map(|b| u16::from_le_bytes([b[0], b[1]])) - .unwrap_or(0); - info!("Input handshake (new format), version={}", protocol_version); - } else if data[0] == 0x0e { - // Old format - protocol_version = first_word; - info!("Input handshake (old format), version={}", protocol_version); - } - - // Echo handshake response - let is_ready = input_ready_flag.load(std::sync::atomic::Ordering::Acquire); - if !is_ready && (first_word == 526 || data[0] == 0x0e) { - if let Err(e) = peer.send_input(&data).await { - error!("Failed to send handshake response: {}", e); - } else { - info!("Sent handshake response, input is ready! Protocol version: {}", protocol_version); - - // Update shared protocol version for input task - input_protocol_version_shared.store(protocol_version as u8, std::sync::atomic::Ordering::Release); - - // Update output decoder protocol version too - output_decoder.set_protocol_version(protocol_version as u8); - - // Signal input task that handshake is complete - input_ready_flag.store(true, std::sync::atomic::Ordering::Release); - - info!("Input encoder protocol version set to {}", protocol_version); - } - } - } - } - WebRtcEvent::IceCandidate(candidate, sdp_mid, sdp_mline_index) => { - // Send our ICE candidate to server - if let Err(e) = signaling.send_ice_candidate( - &candidate, - sdp_mid.as_deref(), - sdp_mline_index.map(|i| i as u32), - ).await { - warn!("Failed to send ICE candidate: {}", e); - } - } - WebRtcEvent::Error(e) => { - error!("WebRTC error: {}", e); - } - WebRtcEvent::SsrcChangeDetected { stall_duration_ms } => { - // SSRC change detected - the server switched video streams - // This is a known limitation of webrtc-rs when handling mid-stream SSRC changes - // without MID header extensions (which GFN doesn't send). - error!( - "SSRC change detected after {}ms stall. Initiating auto-reconnect...", - stall_duration_ms - ); - - // Stop input managers before returning - controller_manager.stop(); - wheel_manager.stop(); - g29_ffb.stop(); - - // Clear raw input sender - #[cfg(any(target_os = "windows", target_os = "macos"))] - crate::input::clear_raw_input_sender(); - - // Return SSRC change result to trigger reconnection - return StreamingResult::SsrcChangeDetected { stall_duration_ms }; - } - } - } - // Receive decode stats from the decoder thread (non-blocking) - Some(decode_stat) = decode_stats_rx.recv() => { - if decode_stat.frame_produced { - frames_decoded += 1; - - // Track decode latency - stats.decode_time_ms = decode_stat.decode_time_ms; - pipeline_latency_sum += decode_stat.decode_time_ms as f64; - pipeline_latency_count += 1; - stats.latency_ms = (pipeline_latency_sum / pipeline_latency_count as f64) as f32; - - // Log first decoded frame - if frames_decoded == 1 { - info!("First frame decoded (async) in {:.1}ms", decode_stat.decode_time_ms); - } - } - - // Request keyframe if decoder is failing - if decode_stat.needs_keyframe { - // Reset depacketizer state to clear any corrupted fragment state - // This is critical for recovering from packet loss/corruption - rtp_depacketizer.reset_state(); - request_keyframe().await; - } - } - // Update stats periodically (interval persists across loop iterations) - _ = stats_interval.tick() => { - let now = std::time::Instant::now(); - let elapsed = now.duration_since(last_stats_time).as_secs_f64(); - - // Calculate actual FPS from decoded frames - let frames_this_period = frames_decoded - last_frames_decoded; - stats.fps = (frames_this_period as f64 / elapsed) as f32; - last_frames_decoded = frames_decoded; - - // Calculate bitrate - stats.bitrate_mbps = ((bytes_received as f64 * 8.0) / (elapsed * 1_000_000.0)) as f32; - stats.frames_received = frames_received; - stats.frames_decoded = frames_decoded; - stats.frames_dropped = frames_dropped; - - // Calculate average input latency (microseconds to milliseconds) - if input_latency_count > 0 { - stats.input_latency_ms = (input_latency_sum / input_latency_count as f64 / 1000.0) as f32; - // Reset for next period - input_latency_sum = 0.0; - input_latency_count = 0; - } - - // Calculate input rate (events per second) - stats.input_rate = (input_events_this_period as f64 / elapsed) as f32; - input_events_this_period = 0; - - // Calculate frame delivery latency (pipeline latency) - if pipeline_latency_count > 0 { - stats.frame_delivery_ms = (pipeline_latency_sum / pipeline_latency_count as f64) as f32; - pipeline_latency_sum = 0.0; - pipeline_latency_count = 0; - } - - // Get network stats from WebRTC (RTT from ICE candidate pair) - let net_stats = peer.get_network_stats().await; - if net_stats.rtt_ms > 0.0 { - stats.rtt_ms = net_stats.rtt_ms; - } - - // Estimate end-to-end latency: - // E2E = network_rtt/2 (input to server) + server_processing (~16ms at 60fps) - // + network_rtt/2 (video back) + decode_time + render_time - // If RTT is 0 (ice-lite), estimate based on typical values - let (estimated_network_oneway, rtt_source) = if stats.rtt_ms > 0.0 { - (stats.rtt_ms / 2.0, "measured") - } else { - // Estimate ~10ms one-way (20ms RTT) for fiber/good internet connection - // This prevents alarming ~80ms E2E reports on good connections where RTT is unmeasured - (10.0, "estimated") - }; - let server_frame_time = 1000.0 / stats.target_fps.max(1) as f32; // ~16.7ms at 60fps - let server_encode_time = 8.0; // Estimated server encode latency ~8ms - stats.estimated_e2e_ms = estimated_network_oneway * 2.0 // network round trip - + server_frame_time // server processing (1 frame) - + server_encode_time // server encode - + stats.decode_time_ms - + stats.render_time_ms; - - // Log latency breakdown once - static LOGGED_LATENCY: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); - if !LOGGED_LATENCY.swap(true, std::sync::atomic::Ordering::Relaxed) && stats.decode_time_ms > 0.0 { - info!("Latency breakdown ({}): network={:.0}ms x2, server_frame={:.0}ms, encode=8ms, decode={:.1}ms, render={:.1}ms = ~{:.0}ms E2E", - rtt_source, estimated_network_oneway, server_frame_time, stats.decode_time_ms, stats.render_time_ms, stats.estimated_e2e_ms); - info!("Note: If actual latency is higher, check server distance or try a closer region"); - } - - // Log if FPS is significantly below target (more than 20% drop) - if stats.fps > 0.0 && stats.fps < (fps as f32 * 0.8) { - debug!("FPS below target: {:.1} / {} (dropped: {})", stats.fps, fps, frames_dropped); - } - - // Update racing wheel count for UI notification - stats.wheel_count = wheel_manager.wheel_count(); - - // Reset counters - bytes_received = 0; - last_stats_time = now; - - // Send stats update - let _ = stats_tx.try_send(stats.clone()); - } - } - } - - // Stop input managers - controller_manager.stop(); - wheel_manager.stop(); - g29_ffb.stop(); - - // Clean up raw input sender - #[cfg(any(target_os = "windows", target_os = "macos"))] - crate::input::clear_raw_input_sender(); - - info!("Streaming session ended"); - StreamingResult::Normal -} diff --git a/opennow-streamer/src/webrtc/peer.rs b/opennow-streamer/src/webrtc/peer.rs deleted file mode 100644 index d6cdc0e..0000000 --- a/opennow-streamer/src/webrtc/peer.rs +++ /dev/null @@ -1,733 +0,0 @@ -//! WebRTC Peer Connection -//! -//! Handles WebRTC peer connection, media streams, and data channels. - -use std::sync::Arc; -use tokio::sync::mpsc; -use parking_lot::Mutex; -use webrtc::api::media_engine::MediaEngine; -use webrtc::api::setting_engine::SettingEngine; -use webrtc::api::APIBuilder; -use webrtc::api::interceptor_registry::register_default_interceptors; -use webrtc::data_channel::RTCDataChannel; -use webrtc::dtls_transport::dtls_role::DTLSRole; -use webrtc::ice_transport::ice_server::RTCIceServer; -use webrtc::ice_transport::ice_gatherer_state::RTCIceGathererState; -use webrtc::interceptor::registry::Registry; -use webrtc::peer_connection::RTCPeerConnection; -use webrtc::peer_connection::configuration::RTCConfiguration; -use webrtc::peer_connection::sdp::session_description::RTCSessionDescription; -use webrtc::rtp_transceiver::rtp_codec::{RTCRtpCodecCapability, RTCRtpCodecParameters, RTPCodecType}; -use webrtc::rtp_transceiver::rtp_transceiver_direction::RTCRtpTransceiverDirection; -use webrtc::rtp_transceiver::rtp_codec::RTCRtpHeaderExtensionCapability; -use webrtc::rtcp::payload_feedbacks::picture_loss_indication::PictureLossIndication; -use anyhow::{Result, Context}; -use log::{info, debug, warn, error}; -use bytes::Bytes; - -/// MIME type for H265/HEVC video codec -const MIME_TYPE_H265: &str = "video/H265"; -/// MIME type for AV1 video codec -const MIME_TYPE_AV1: &str = "video/AV1"; - -use super::InputEncoder; -use super::sdp::is_ice_lite; - -/// Events from WebRTC connection -#[derive(Debug)] -pub enum WebRtcEvent { - Connected, - Disconnected, - /// Video frame with RTP timestamp (90kHz clock) and marker bit - VideoFrame { payload: Vec, rtp_timestamp: u32, marker: bool }, - AudioFrame(Vec), - DataChannelOpen(String), - DataChannelMessage(String, Vec), - IceCandidate(String, Option, Option), - Error(String), - /// Stream stalled due to SSRC change - reconnection recommended - /// This happens when GFN server changes resolution and switches to a new SSRC - /// that webrtc-rs can't handle without MID header extensions. - SsrcChangeDetected { stall_duration_ms: u64 }, -} - -/// Shared peer connection for PLI requests (static to allow access from decoder) -static PEER_CONNECTION: Mutex>> = Mutex::new(None); -/// Track SSRC for PLI -static VIDEO_SSRC: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0); - -/// WebRTC peer for GFN streaming -pub struct WebRtcPeer { - peer_connection: Option>, - input_channel: Option>, - /// Partially reliable channel for mouse (lower latency, unordered) - mouse_channel: Option>, - event_tx: mpsc::Sender, - input_encoder: InputEncoder, - handshake_complete: bool, -} - -/// Request a keyframe (PLI - Picture Loss Indication) -/// Call this when decode errors occur to recover the stream -pub async fn request_keyframe() { - let pc = PEER_CONNECTION.lock().clone(); - let ssrc = VIDEO_SSRC.load(std::sync::atomic::Ordering::Relaxed); - - if let Some(pc) = pc { - if ssrc != 0 { - let pli = PictureLossIndication { - sender_ssrc: 0, - media_ssrc: ssrc, - }; - - match pc.write_rtcp(&[Box::new(pli)]).await { - Ok(_) => info!("Sent PLI (keyframe request) for SSRC {}", ssrc), - Err(e) => warn!("Failed to send PLI: {:?}", e), - } - } else { - debug!("Cannot send PLI: no video SSRC yet"); - } - } else { - debug!("Cannot send PLI: no peer connection"); - } -} - -impl WebRtcPeer { - pub fn new(event_tx: mpsc::Sender) -> Self { - Self { - peer_connection: None, - input_channel: None, - mouse_channel: None, - event_tx, - input_encoder: InputEncoder::new(), - handshake_complete: false, - } - } - - /// Create peer connection and set remote SDP offer - pub async fn handle_offer(&mut self, sdp_offer: &str, ice_servers: Vec) -> Result { - info!("Setting up WebRTC peer connection"); - - // Detect ice-lite BEFORE creating peer connection - this affects DTLS role - let offer_is_ice_lite = is_ice_lite(sdp_offer); - if offer_is_ice_lite { - info!("Server is ice-lite - will configure active DTLS role (Client)"); - } - - // Create media engine with all required codecs - let mut media_engine = MediaEngine::default(); - - // Register default codecs (H264, VP8, VP9, Opus, etc.) - media_engine.register_default_codecs()?; - - // Register H265/HEVC codec (not in default codecs!) - // Use payload_type 0 for dynamic payload type negotiation from SDP - media_engine.register_codec( - RTCRtpCodecParameters { - capability: RTCRtpCodecCapability { - mime_type: MIME_TYPE_H265.to_string(), - clock_rate: 90000, - channels: 0, - sdp_fmtp_line: "".to_string(), - rtcp_feedback: vec![], - }, - payload_type: 0, // Dynamic - will be negotiated from SDP - ..Default::default() - }, - RTPCodecType::Video, - )?; - info!("Registered H265/HEVC codec"); - - // Register AV1 codec (for future use) - media_engine.register_codec( - RTCRtpCodecParameters { - capability: RTCRtpCodecCapability { - mime_type: MIME_TYPE_AV1.to_string(), - clock_rate: 90000, - channels: 0, - sdp_fmtp_line: "".to_string(), - rtcp_feedback: vec![], - }, - payload_type: 0, // Dynamic - will be negotiated from SDP - ..Default::default() - }, - RTPCodecType::Video, - )?; - info!("Registered AV1 codec"); - - // Register RTP header extensions for SSRC demuxing - // These are required to handle mid-stream SSRC changes and simulcast - // MID extension - identifies which media section an RTP packet belongs to - media_engine.register_header_extension( - RTCRtpHeaderExtensionCapability { - uri: "urn:ietf:params:rtp-hdrext:sdes:mid".to_string(), - }, - RTPCodecType::Video, - Some(RTCRtpTransceiverDirection::Recvonly), - )?; - media_engine.register_header_extension( - RTCRtpHeaderExtensionCapability { - uri: "urn:ietf:params:rtp-hdrext:sdes:mid".to_string(), - }, - RTPCodecType::Audio, - Some(RTCRtpTransceiverDirection::Recvonly), - )?; - info!("Registered SDES MID header extension for video and audio"); - - // RTP Stream ID extension - identifies specific streams in simulcast - media_engine.register_header_extension( - RTCRtpHeaderExtensionCapability { - uri: "urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id".to_string(), - }, - RTPCodecType::Video, - Some(RTCRtpTransceiverDirection::Recvonly), - )?; - info!("Registered SDES RTP-Stream-ID header extension"); - - // Repaired RTP Stream ID extension - required for SSRC changes during stream - // This allows webrtc-rs to handle mid-stream SSRC switches (e.g., HDR mode changes) - media_engine.register_header_extension( - RTCRtpHeaderExtensionCapability { - uri: "urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id".to_string(), - }, - RTPCodecType::Video, - Some(RTCRtpTransceiverDirection::Recvonly), - )?; - info!("Registered SDES Repaired-RTP-Stream-ID header extension"); - - // Create interceptor registry - let mut registry = Registry::new(); - registry = register_default_interceptors(registry, &mut media_engine)?; - - // Create setting engine - configure DTLS role for ice-lite - let mut setting_engine = SettingEngine::default(); - if offer_is_ice_lite { - // When server is ice-lite, we MUST be DTLS Client (active/initiator) - // This makes us send the DTLS ClientHello to start the handshake - setting_engine.set_answering_dtls_role(DTLSRole::Client)?; - info!("Configured DTLS role to Client (active) for ice-lite server"); - } - - // Enable provisional SSRC support for GFN resolution changes - // GFN server uses new SSRCs when changing resolution, and doesn't send MID extensions - // This allows undeclared SSRCs to be routed to existing video transceivers - setting_engine.set_allow_provisional_ssrc(true); - info!("Enabled provisional SSRC support for GFN compatibility"); - - // Create API with setting engine - let api = APIBuilder::new() - .with_media_engine(media_engine) - .with_interceptor_registry(registry) - .with_setting_engine(setting_engine) - .build(); - - // Create RTCConfiguration - let config = RTCConfiguration { - ice_servers, - ..Default::default() - }; - - // Create peer connection - let peer_connection = Arc::new(api.new_peer_connection(config).await?); - info!("Peer connection created"); - - // Set up event handlers - let event_tx = self.event_tx.clone(); - - // On ICE candidate - let event_tx_ice = event_tx.clone(); - peer_connection.on_ice_candidate(Box::new(move |candidate| { - let tx = event_tx_ice.clone(); - Box::pin(async move { - if let Some(c) = candidate { - let candidate_str = c.to_json().map(|j| j.candidate).unwrap_or_default(); - info!("Gathered local ICE candidate: {}", candidate_str); - let sdp_mid = c.to_json().ok().and_then(|j| j.sdp_mid); - let sdp_mline_index = c.to_json().ok().and_then(|j| j.sdp_mline_index); - let _ = tx.send(WebRtcEvent::IceCandidate( - candidate_str, - sdp_mid, - sdp_mline_index, - )).await; - } - }) - })); - - // On ICE connection state change - let event_tx_state = event_tx.clone(); - peer_connection.on_ice_connection_state_change(Box::new(move |state| { - let tx = event_tx_state.clone(); - info!("ICE connection state: {:?}", state); - Box::pin(async move { - match state { - webrtc::ice_transport::ice_connection_state::RTCIceConnectionState::Connected => { - let _ = tx.send(WebRtcEvent::Connected).await; - } - webrtc::ice_transport::ice_connection_state::RTCIceConnectionState::Disconnected | - webrtc::ice_transport::ice_connection_state::RTCIceConnectionState::Failed => { - let _ = tx.send(WebRtcEvent::Disconnected).await; - } - _ => {} - } - }) - })); - - // On peer connection state change (includes DTLS state) - let _pc_for_state = peer_connection.clone(); - peer_connection.on_peer_connection_state_change(Box::new(move |state| { - info!("Peer connection state: {:?}", state); - Box::pin(async move { - match state { - webrtc::peer_connection::peer_connection_state::RTCPeerConnectionState::Connected => { - info!("=== DTLS HANDSHAKE COMPLETE - FULLY CONNECTED ==="); - } - webrtc::peer_connection::peer_connection_state::RTCPeerConnectionState::Failed => { - warn!("Peer connection FAILED (likely DTLS handshake failure)"); - } - _ => {} - } - }) - })); - - // On track (video/audio) - let event_tx_track = event_tx.clone(); - peer_connection.on_track(Box::new(move |track, _receiver, _transceiver| { - let tx = event_tx_track.clone(); - let track = track.clone(); - let track_kind = track.kind(); - let track_id = track.id().to_string(); - info!("Track received: kind={:?}, id={}, codec={:?}", track_kind, track_id, track.codec()); - - // IMPORTANT: Spawn a separate tokio task for reading from the track - // The Future returned from on_track callback may not be properly spawned by webrtc-rs - let tx_clone = tx.clone(); - let track_clone = track.clone(); - let track_id_clone = track_id.clone(); - tokio::spawn(async move { - let mut buffer = vec![0u8; 1500]; - let mut packet_count: u64 = 0; - let mut last_packet_time = std::time::Instant::now(); - let mut stall_warning_sent = false; - const STALL_TIMEOUT_MS: u64 = 2000; // 2 seconds without packets = stall - - info!("=== Starting track read loop for {} ({}) ===", - track_id_clone, - if track_kind == webrtc::rtp_transceiver::rtp_codec::RTPCodecType::Video { "VIDEO" } else { "AUDIO" }); - - loop { - // Use timeout on read to detect stalls (e.g., SSRC change that we can't handle) - let read_timeout = tokio::time::Duration::from_millis(500); - match tokio::time::timeout(read_timeout, track_clone.read(&mut buffer)).await { - Ok(Ok((rtp_packet, _))) => { - packet_count += 1; - last_packet_time = std::time::Instant::now(); - stall_warning_sent = false; - - // Store SSRC for PLI on first video packet - if packet_count == 1 { - info!("[{}] First RTP packet: {} bytes payload, SSRC: {}", - track_id_clone, rtp_packet.payload.len(), rtp_packet.header.ssrc); - - if track_kind == webrtc::rtp_transceiver::rtp_codec::RTPCodecType::Video { - VIDEO_SSRC.store(rtp_packet.header.ssrc, std::sync::atomic::Ordering::Relaxed); - - // Request keyframe immediately when video track starts - // This ensures we get an IDR frame to begin decoding - info!("Video track started - requesting initial keyframe"); - let pc_clone = PEER_CONNECTION.lock().clone(); - if let Some(pc) = pc_clone { - let pli = PictureLossIndication { - sender_ssrc: 0, - media_ssrc: rtp_packet.header.ssrc, - }; - if let Err(e) = pc.write_rtcp(&[Box::new(pli)]).await { - warn!("Failed to send initial PLI: {:?}", e); - } else { - info!("Sent initial PLI for SSRC {}", rtp_packet.header.ssrc); - } - } - } - } - - if track_kind == webrtc::rtp_transceiver::rtp_codec::RTPCodecType::Video { - if let Err(e) = tx_clone.send(WebRtcEvent::VideoFrame { - payload: rtp_packet.payload.to_vec(), - rtp_timestamp: rtp_packet.header.timestamp, - marker: rtp_packet.header.marker, - }).await { - warn!("Failed to send video frame event: {:?}", e); - break; - } - } else { - if let Err(e) = tx_clone.send(WebRtcEvent::AudioFrame(rtp_packet.payload.to_vec())).await { - warn!("Failed to send audio frame event: {:?}", e); - break; - } - } - } - Ok(Err(e)) => { - // Check if this is a transient error or connection closed - let error_str = format!("{:?}", e); - if error_str.contains("EOF") || error_str.contains("closed") { - info!("Track {} connection closed: {}", track_id_clone, e); - break; - } else { - error!("Track {} read error (will retry): {}", track_id_clone, e); - // Don't break on transient errors - continue reading - tokio::time::sleep(tokio::time::Duration::from_millis(10)).await; - continue; - } - } - Err(_timeout) => { - // Read timed out - check for stall - let elapsed_ms = last_packet_time.elapsed().as_millis() as u64; - - if elapsed_ms > STALL_TIMEOUT_MS && !stall_warning_sent { - // Video track has stalled - likely SSRC changed - if track_kind == webrtc::rtp_transceiver::rtp_codec::RTPCodecType::Video { - warn!("Video track stalled for {}ms - SSRC change detected (server switched stream ID). This is a webrtc-rs limitation.", elapsed_ms); - stall_warning_sent = true; - - // Send SSRC change event to trigger auto-reconnect - let _ = tx_clone.send(WebRtcEvent::SsrcChangeDetected { - stall_duration_ms: elapsed_ms, - }).await; - - // Also send user-visible error - let _ = tx_clone.send(WebRtcEvent::Error( - "Stream interrupted: Server changed video stream (SSRC change). Auto-reconnect recommended.".to_string() - )).await; - - // Try to request keyframe in case it helps (unlikely but worth trying) - let ssrc = VIDEO_SSRC.load(std::sync::atomic::Ordering::Relaxed); - if ssrc != 0 { - let pc_clone = PEER_CONNECTION.lock().clone(); - if let Some(pc) = pc_clone { - let pli = PictureLossIndication { - sender_ssrc: 0, - media_ssrc: ssrc, - }; - let _ = pc.write_rtcp(&[Box::new(pli)]).await; - } - } - } - } - // Continue loop to try reading again - continue; - } - } - } - error!("Track {} read loop ended after {} packets - THIS SHOULD NOT HAPPEN DURING STREAM", - track_id_clone, packet_count); - }); - - // Return empty future since we spawned the actual work - Box::pin(async {}) - })); - - // On data channel - let event_tx_dc = event_tx.clone(); - peer_connection.on_data_channel(Box::new(move |dc| { - let tx = event_tx_dc.clone(); - let dc_label = dc.label().to_string(); - info!("Data channel received: {}", dc_label); - - Box::pin(async move { - let label = dc_label.clone(); - - let tx_open = tx.clone(); - let label_open = label.clone(); - dc.on_open(Box::new(move || { - let tx = tx_open.clone(); - let label = label_open.clone(); - Box::pin(async move { - info!("Data channel '{}' opened", label); - let _ = tx.send(WebRtcEvent::DataChannelOpen(label)).await; - }) - })); - - let tx_msg = tx.clone(); - let label_msg = label.clone(); - dc.on_message(Box::new(move |msg| { - let tx = tx_msg.clone(); - let label = label_msg.clone(); - Box::pin(async move { - debug!("Data channel '{}' message: {} bytes", label, msg.data.len()); - let _ = tx.send(WebRtcEvent::DataChannelMessage(label, msg.data.to_vec())).await; - }) - })); - }) - })); - - // Log offer SDP for debugging - debug!("=== OFFER SDP (from server) ==="); - for line in sdp_offer.lines() { - debug!("OFFER: {}", line); - } - debug!("=== END OFFER SDP ==="); - - // Set remote description (offer) - let offer = RTCSessionDescription::offer(sdp_offer.to_string())?; - peer_connection.set_remote_description(offer).await?; - info!("Remote description set"); - - // Wait for ICE gathering - let (gather_tx, gather_rx) = tokio::sync::oneshot::channel::<()>(); - let gather_tx = Arc::new(std::sync::Mutex::new(Some(gather_tx))); - - peer_connection.on_ice_gathering_state_change(Box::new({ - let gather_tx = gather_tx.clone(); - move |state| { - info!("ICE gathering state: {:?}", state); - if state == RTCIceGathererState::Complete { - if let Some(tx) = gather_tx.lock().unwrap().take() { - let _ = tx.send(()); - } - } - Box::pin(async {}) - } - })); - - // Create answer (DTLS role is already configured via SettingEngine if ice-lite) - let answer = peer_connection.create_answer(None).await?; - peer_connection.set_local_description(answer.clone()).await?; - info!("Local description set, waiting for ICE gathering..."); - - // Wait for ICE gathering (with timeout) - let gather_result = tokio::time::timeout( - std::time::Duration::from_secs(5), - gather_rx - ).await; - - match gather_result { - Ok(_) => info!("ICE gathering complete"), - Err(_) => warn!("ICE gathering timeout - proceeding"), - } - - // Get final SDP (already has DTLS setup fixed if ice-lite) - let final_sdp = peer_connection.local_description().await - .map(|d| d.sdp) - .unwrap_or_else(|| answer.sdp.clone()); - - info!("Final SDP length: {}", final_sdp.len()); - - // Log SDP content for debugging - debug!("=== ANSWER SDP ==="); - for line in final_sdp.lines() { - debug!("SDP: {}", line); - } - debug!("=== END SDP ==="); - - // Store in static for PLI requests - *PEER_CONNECTION.lock() = Some(peer_connection.clone()); - - self.peer_connection = Some(peer_connection); - - Ok(final_sdp) - } - - /// Create input data channels (reliable for keyboard, partially reliable for mouse) - pub async fn create_input_channel(&mut self) -> Result<()> { - let pc = self.peer_connection.as_ref().context("No peer connection")?; - - // Reliable channel for keyboard and handshake - let dc = pc.create_data_channel( - "input_channel_v1", - Some(webrtc::data_channel::data_channel_init::RTCDataChannelInit { - ordered: Some(true), // Keyboard needs ordering - max_retransmits: Some(0), - ..Default::default() - }), - ).await?; - - info!("Created reliable input channel: {}", dc.label()); - - let event_tx = self.event_tx.clone(); - - dc.on_open(Box::new(move || { - info!("Input channel opened"); - Box::pin(async {}) - })); - - let event_tx_msg = event_tx.clone(); - dc.on_message(Box::new(move |msg| { - let tx = event_tx_msg.clone(); - let data = msg.data.to_vec(); - Box::pin(async move { - debug!("Input channel message: {} bytes", data.len()); - if data.len() >= 2 && data[0] == 0x0e { - let _ = tx.send(WebRtcEvent::DataChannelMessage( - "input_handshake".to_string(), - data, - )).await; - } - }) - })); - - self.input_channel = Some(dc); - - // Partially reliable channel for mouse - lower latency! - // Uses maxPacketLifeTime instead of retransmits for time-sensitive data - let mouse_dc = pc.create_data_channel( - "input_channel_partially_reliable", - Some(webrtc::data_channel::data_channel_init::RTCDataChannelInit { - ordered: Some(false), // Unordered for lower latency - max_packet_life_time: Some(8), // 8ms lifetime for low-latency mouse - ..Default::default() - }), - ).await?; - - info!("Created partially reliable mouse channel: {}", mouse_dc.label()); - - mouse_dc.on_open(Box::new(move || { - info!("Mouse channel opened (partially reliable)"); - Box::pin(async {}) - })); - - self.mouse_channel = Some(mouse_dc); - - Ok(()) - } - - /// Send input event over reliable data channel (keyboard, handshake) - pub async fn send_input(&mut self, data: &[u8]) -> Result<()> { - let dc = self.input_channel.as_ref().context("No input channel")?; - dc.send(&Bytes::copy_from_slice(data)).await?; - Ok(()) - } - - /// Explicitly send controller input (aliases send_input/input_channel_v1 for now) - /// Used to enforce logical separation - pub async fn send_controller_input(&mut self, data: &[u8]) -> Result<()> { - // "input_channel_v1 needs to be only controller" - // We use the reliable channel (v1) for controller - self.send_input(data).await - } - - /// Send mouse input over partially reliable channel (lower latency) - /// Falls back to reliable channel if mouse channel not ready - pub async fn send_mouse_input(&mut self, data: &[u8]) -> Result<()> { - // Prefer the partially reliable channel for mouse - if let Some(ref mouse_dc) = self.mouse_channel { - if mouse_dc.ready_state() == webrtc::data_channel::data_channel_state::RTCDataChannelState::Open { - mouse_dc.send(&Bytes::copy_from_slice(data)).await?; - return Ok(()); - } - } - // Fall back to reliable channel? - // User reports "controller needs to be only path not same as mouse" - // Removing fallback to ensure mouse never pollutes controller channel - // self.send_input(data).await - warn!("Mouse channel not ready, dropping mouse event"); - Ok(()) - } - - /// Check if mouse channel is ready - pub fn is_mouse_channel_ready(&self) -> bool { - self.mouse_channel.as_ref() - .map(|dc| dc.ready_state() == webrtc::data_channel::data_channel_state::RTCDataChannelState::Open) - .unwrap_or(false) - } - - /// Send handshake response - pub async fn send_handshake_response(&mut self, major: u8, minor: u8, flags: u8) -> Result<()> { - let response = vec![0x0e, major, minor, flags]; - self.send_input(&response).await?; - self.handshake_complete = true; - info!("Sent handshake response, input ready"); - Ok(()) - } - - /// Add remote ICE candidate - pub async fn add_ice_candidate(&self, candidate: &str, sdp_mid: Option<&str>, sdp_mline_index: Option, ufrag: Option) -> Result<()> { - let pc = self.peer_connection.as_ref().context("No peer connection")?; - - let candidate = webrtc::ice_transport::ice_candidate::RTCIceCandidateInit { - candidate: candidate.to_string(), - sdp_mid: sdp_mid.map(|s| s.to_string()), - sdp_mline_index, - username_fragment: ufrag, - }; - - pc.add_ice_candidate(candidate).await?; - info!("Added remote ICE candidate"); - Ok(()) - } - - pub fn is_handshake_complete(&self) -> bool { - self.handshake_complete - } - - /// Get RTT (round-trip time) from ICE candidate pair stats - /// Returns None if no active candidate pair or stats unavailable - pub async fn get_rtt_ms(&self) -> Option { - let pc = self.peer_connection.as_ref()?; - let stats = pc.get_stats().await; - - // Look for ICE candidate pair stats with RTT - for (_, stat) in stats.reports.iter() { - if let webrtc::stats::StatsReportType::CandidatePair(pair) = stat { - // Only use nominated/active pairs - if pair.nominated && pair.current_round_trip_time > 0.0 { - // current_round_trip_time is in seconds, convert to ms - return Some((pair.current_round_trip_time * 1000.0) as f32); - } - } - } - None - } - - /// Get comprehensive network stats (RTT, jitter, packet loss) - pub async fn get_network_stats(&self) -> NetworkStats { - let mut stats = NetworkStats::default(); - - let Some(pc) = self.peer_connection.as_ref() else { - return stats; - }; - - let report = pc.get_stats().await; - - // Debug: log candidate pair stats once - static LOGGED_STATS: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); - let should_log = !LOGGED_STATS.swap(true, std::sync::atomic::Ordering::Relaxed); - - for (id, stat) in report.reports.iter() { - match stat { - webrtc::stats::StatsReportType::CandidatePair(pair) => { - if should_log { - info!("CandidatePair {}: nominated={}, state={:?}, rtt={}s", - id, pair.nominated, pair.state, pair.current_round_trip_time); - } - // Use any pair with RTT data (not just nominated - ice-lite may behave differently) - if pair.current_round_trip_time > 0.0 && stats.rtt_ms == 0.0 { - stats.rtt_ms = (pair.current_round_trip_time * 1000.0) as f32; - } - if pair.nominated { - stats.bytes_received = pair.bytes_received; - stats.bytes_sent = pair.bytes_sent; - stats.packets_received = pair.packets_received as u64; - } - } - webrtc::stats::StatsReportType::InboundRTP(inbound) => { - // Video track stats - packets_received available - if inbound.kind == "video" { - stats.video_packets_received = inbound.packets_received; - } - } - _ => {} - } - } - - stats - } -} - -/// Network statistics from WebRTC -#[derive(Debug, Clone, Default)] -pub struct NetworkStats { - pub rtt_ms: f32, - pub packets_received: u64, - pub video_packets_received: u64, - pub bytes_received: u64, - pub bytes_sent: u64, -} diff --git a/opennow-streamer/src/webrtc/sdp.rs b/opennow-streamer/src/webrtc/sdp.rs deleted file mode 100644 index 8310a64..0000000 --- a/opennow-streamer/src/webrtc/sdp.rs +++ /dev/null @@ -1,480 +0,0 @@ -//! SDP Manipulation -//! -//! Parse and modify SDP for codec preferences and ICE fixes. - -use crate::app::VideoCodec; -use log::{debug, info, warn}; -use std::collections::HashMap; - -/// Fix 0.0.0.0 in SDP with actual server IP -/// NOTE: Do NOT add ICE candidates to the offer SDP! The offer contains the -/// SERVER's candidates. Adding our own candidates here corrupts ICE negotiation. -/// Server candidates should come via trickle ICE through signaling. -pub fn fix_server_ip(sdp: &str, server_ip: &str) -> String { - // Only fix the connection line, don't touch candidates - let modified = sdp.replace("c=IN IP4 0.0.0.0", &format!("c=IN IP4 {}", server_ip)); - info!("Fixed connection IP to {}", server_ip); - modified -} - -/// Normalize codec name (HEVC -> H265) -fn normalize_codec_name(name: &str) -> String { - let upper = name.to_uppercase(); - match upper.as_str() { - "HEVC" => "H265".to_string(), - _ => upper, - } -} - -/// Force a specific video codec in SDP -pub fn prefer_codec(sdp: &str, codec: &VideoCodec) -> String { - let codec_name = match codec { - VideoCodec::H264 => "H264", - VideoCodec::H265 => "H265", - VideoCodec::AV1 => "AV1", - }; - - info!("Forcing codec: {}", codec_name); - - // Detect line ending style - let line_ending = if sdp.contains("\r\n") { "\r\n" } else { "\n" }; - - // Use .lines() which handles both \r\n and \n correctly - let lines: Vec<&str> = sdp.lines().collect(); - let mut result: Vec = Vec::new(); - - // First pass: collect codec -> payload type mapping - // Normalize HEVC -> H265 for consistent lookup - let mut codec_payloads: HashMap> = HashMap::new(); - let mut in_video = false; - - for line in &lines { - if line.starts_with("m=video") { - in_video = true; - } else if line.starts_with("m=") && in_video { - in_video = false; - } - - if in_video { - // Parse a=rtpmap:96 H264/90000 - if let Some(rtpmap) = line.strip_prefix("a=rtpmap:") { - let parts: Vec<&str> = rtpmap.split_whitespace().collect(); - if parts.len() >= 2 { - let pt = parts[0].to_string(); - let raw_codec = parts[1].split('/').next().unwrap_or(""); - let normalized_codec = normalize_codec_name(raw_codec); - debug!( - "Found codec {} (normalized: {}) with payload type {}", - raw_codec, normalized_codec, pt - ); - codec_payloads.entry(normalized_codec).or_default().push(pt); - } - } - } - } - - info!( - "Available video codecs in SDP: {:?}", - codec_payloads.keys().collect::>() - ); - - // Get preferred codec payload types - let preferred = codec_payloads.get(codec_name).cloned().unwrap_or_default(); - if preferred.is_empty() { - info!( - "Codec {} not found in SDP - keeping original SDP unchanged", - codec_name - ); - return sdp.to_string(); - } - - info!( - "Found {} payload type(s) for {}: {:?}", - preferred.len(), - codec_name, - preferred - ); - - // Use HashSet for easier comparison - let preferred_set: std::collections::HashSet = preferred.iter().cloned().collect(); - - // Second pass: filter SDP - in_video = false; - for line in &lines { - if line.starts_with("m=video") { - in_video = true; - - // Rewrite m=video line to only include preferred payloads - let parts: Vec<&str> = line.split_whitespace().collect(); - if parts.len() >= 4 { - let header = parts[..3].join(" "); - let payload_types: Vec<&str> = parts[3..] - .iter() - .filter(|pt| preferred_set.contains(&pt.to_string())) - .copied() - .collect(); - - if !payload_types.is_empty() { - let new_line = format!("{} {}", header, payload_types.join(" ")); - debug!("Rewritten m=video line: {}", new_line); - result.push(new_line); - continue; - } else { - // No matching payload types - keep original m=video line - warn!( - "No matching payload types for {} in m=video line, keeping original", - codec_name - ); - result.push(line.to_string()); - continue; - } - } - } else if line.starts_with("m=") && in_video { - in_video = false; - } - - if in_video { - // Filter rtpmap, fmtp, rtcp-fb lines - only keep lines for preferred codec - if let Some(rest) = line - .strip_prefix("a=rtpmap:") - .or_else(|| line.strip_prefix("a=fmtp:")) - .or_else(|| line.strip_prefix("a=rtcp-fb:")) - { - let pt = rest.split_whitespace().next().unwrap_or(""); - if !preferred_set.contains(pt) { - debug!("Filtering out line for payload type {}: {}", pt, line); - continue; // Skip non-preferred codec attributes - } - } - } - - result.push(line.to_string()); - } - - let filtered_sdp = result.join(line_ending); - info!( - "SDP filtered: {} -> {} bytes", - sdp.len(), - filtered_sdp.len() - ); - filtered_sdp -} - -/// Extract video codec from SDP -pub fn extract_video_codec(sdp: &str) -> Option { - let mut in_video = false; - - for line in sdp.lines() { - if line.starts_with("m=video") { - in_video = true; - } else if line.starts_with("m=") && in_video { - break; - } - - if in_video && line.starts_with("a=rtpmap:") { - // a=rtpmap:96 H264/90000 - if let Some(codec_part) = line.split_whitespace().nth(1) { - return Some(codec_part.split('/').next()?.to_string()); - } - } - } - - None -} - -/// Extract resolution from SDP -pub fn extract_resolution(sdp: &str) -> Option<(u32, u32)> { - for line in sdp.lines() { - // Look for a=imageattr or custom resolution attributes - if line.starts_with("a=fmtp:") && line.contains("max-fs=") { - // Parse max-fs for resolution - } - } - None -} - -/// Check if the offer SDP indicates an ice-lite server -pub fn is_ice_lite(sdp: &str) -> bool { - for line in sdp.lines() { - if line.trim() == "a=ice-lite" { - return true; - } - } - false -} - -/// Fix DTLS setup for ice-lite servers -/// -/// When the server is ice-lite and offers `a=setup:actpass`, we MUST respond -/// with `a=setup:active` (not passive). This makes us initiate the DTLS handshake. -/// -/// If we respond with `a=setup:passive`, both sides wait for the other to start -/// DTLS, resulting in a handshake timeout. -pub fn fix_dtls_setup_for_ice_lite(answer_sdp: &str) -> String { - info!("Fixing DTLS setup for ice-lite: changing passive -> active"); - - // Replace all instances of a=setup:passive with a=setup:active - let fixed = answer_sdp.replace("a=setup:passive", "a=setup:active"); - - // Log for debugging - let passive_count = answer_sdp.matches("a=setup:passive").count(); - let active_count = fixed.matches("a=setup:active").count(); - info!( - "DTLS setup fix: replaced {} passive entries, now have {} active entries", - passive_count, active_count - ); - - fixed -} - -/// Inject additional SSRCs into the video section of the offer SDP -/// -/// GFN server uses sequential SSRCs (1, 2, 3, 4...) for video streams when -/// resolution changes occur. However, webrtc-rs requires SSRCs to be declared -/// in the SDP or have MID header extensions (which GFN doesn't send). -/// -/// This function injects `a=ssrc:N` lines for SSRCs 2, 3, 4 into the video -/// section, similar to how the official GFN client (Bifrost2.dll) does it. -/// This allows webrtc-rs to accept packets from these SSRCs when the server -/// switches resolution. -/// -/// Based on reverse engineering of official GFN client: -/// - Bifrost2.dll contains: "a=ssrc:2 cname:odrerir", "a=ssrc:3 cname:odrerir" -/// - Uses "provisional stream" concept to handle SSRC changes -pub fn inject_provisional_ssrcs(sdp: &str) -> String { - let line_ending = if sdp.contains("\r\n") { "\r\n" } else { "\n" }; - let lines: Vec<&str> = sdp.lines().collect(); - let mut result: Vec = Vec::new(); - - let mut in_video = false; - let mut video_msid: Option<(String, String)> = None; // (stream_id, track_id) - let mut existing_ssrcs: Vec = Vec::new(); - let mut injected = false; - - // First pass: find existing video SSRCs and msid - for line in &lines { - if line.starts_with("m=video") { - in_video = true; - } else if line.starts_with("m=") && in_video { - in_video = false; - } - - if in_video { - // Parse a=ssrc:N ... - if let Some(rest) = line.strip_prefix("a=ssrc:") { - if let Some(ssrc_str) = rest.split_whitespace().next() { - if let Ok(ssrc) = ssrc_str.parse::() { - existing_ssrcs.push(ssrc); - } - } - } - // Parse a=msid:stream_id track_id - if let Some(rest) = line.strip_prefix("a=msid:") { - let parts: Vec<&str> = rest.split_whitespace().collect(); - if parts.len() >= 2 { - video_msid = Some((parts[0].to_string(), parts[1].to_string())); - } - } - } - } - - // Determine which SSRCs to inject (2, 3, 4 that don't already exist) - let ssrcs_to_inject: Vec = (2..=4) - .filter(|ssrc| !existing_ssrcs.contains(ssrc)) - .collect(); - - if ssrcs_to_inject.is_empty() { - debug!("No provisional SSRCs needed - all already declared"); - return sdp.to_string(); - } - - info!( - "Injecting provisional SSRCs {:?} for video (existing: {:?})", - ssrcs_to_inject, existing_ssrcs - ); - - // Second pass: find injection point - // - If there are existing a=ssrc lines, inject after the last one - // - If no a=ssrc lines exist, inject before the next m= line (end of video section) - in_video = false; - let mut last_ssrc_line_idx: Option = None; - let mut video_section_end_idx: Option = None; - let mut video_section_start_idx: Option = None; - - for (idx, line) in lines.iter().enumerate() { - if line.starts_with("m=video") { - in_video = true; - video_section_start_idx = Some(idx); - } else if line.starts_with("m=") && in_video { - // Found start of next section - this is where video section ends - video_section_end_idx = Some(idx); - in_video = false; - } - - if in_video && line.starts_with("a=ssrc:") { - last_ssrc_line_idx = Some(idx); - } - } - - // If we're still in video section at end of file, end is after last line - if in_video && video_section_end_idx.is_none() { - video_section_end_idx = Some(lines.len()); - } - - // Determine injection point: - // - After last a=ssrc line if exists - // - Otherwise, before the next m= section (or end of file) - // - If still no valid point, inject after m=video line - let _injection_after_idx = last_ssrc_line_idx - .or_else(|| video_section_end_idx.map(|idx| idx.saturating_sub(1))) - .or(video_section_start_idx); - - // Third pass: build result with injected SSRCs - in_video = false; - for (idx, line) in lines.iter().enumerate() { - // If we need to inject BEFORE this line (when inserting at section end) - if !injected && video_section_end_idx == Some(idx) && last_ssrc_line_idx.is_none() { - // Inject at end of video section (before next m= line) - let (stream_id, track_id) = video_msid - .clone() - .unwrap_or_else(|| ("odrerir".to_string(), "video".to_string())); - - for ssrc in &ssrcs_to_inject { - result.push(format!("a=ssrc:{} msid:{} {}", ssrc, stream_id, track_id)); - result.push(format!("a=ssrc:{} cname:odrerir", ssrc)); - } - - injected = true; - info!( - "Injected {} provisional SSRCs at end of video section", - ssrcs_to_inject.len() - ); - } - - result.push(line.to_string()); - - if line.starts_with("m=video") { - in_video = true; - } else if line.starts_with("m=") && in_video { - in_video = false; - } - - // Inject after the last a=ssrc line in video section - if in_video && Some(idx) == last_ssrc_line_idx && !injected { - // Use the same msid as existing video track, or generate one - let (stream_id, track_id) = video_msid - .clone() - .unwrap_or_else(|| ("odrerir".to_string(), "video".to_string())); - - for ssrc in &ssrcs_to_inject { - // Add ssrc with msid (required for webrtc-rs to create track) - result.push(format!("a=ssrc:{} msid:{} {}", ssrc, stream_id, track_id)); - result.push(format!("a=ssrc:{} cname:odrerir", ssrc)); - } - - injected = true; - info!( - "Injected {} provisional SSRCs after existing SSRC declarations", - ssrcs_to_inject.len() - ); - } - } - - // Handle edge case: video section at end of file with no ssrc lines - if !injected && video_section_start_idx.is_some() { - let (stream_id, track_id) = video_msid - .unwrap_or_else(|| ("odrerir".to_string(), "video".to_string())); - - for ssrc in &ssrcs_to_inject { - result.push(format!("a=ssrc:{} msid:{} {}", ssrc, stream_id, track_id)); - result.push(format!("a=ssrc:{} cname:odrerir", ssrc)); - } - - info!( - "Injected {} provisional SSRCs at end of SDP (video section at EOF)", - ssrcs_to_inject.len() - ); - } - - result.join(line_ending) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_fix_server_ip() { - let sdp = "c=IN IP4 0.0.0.0\r\nm=video 9 UDP/TLS/RTP/SAVPF 96\r\n"; - let fixed = fix_server_ip(sdp, "192.168.1.1"); - assert!(fixed.contains("c=IN IP4 192.168.1.1")); - // Should NOT add candidates - that corrupts ICE negotiation - assert!(!fixed.contains("a=candidate:")); - } - - #[test] - fn test_inject_provisional_ssrcs_with_existing() { - // SDP with existing SSRC 1 - let sdp = "v=0\r\n\ - m=video 9 UDP/TLS/RTP/SAVPF 96\r\n\ - a=msid:stream1 video1\r\n\ - a=ssrc:1 msid:stream1 video1\r\n\ - a=ssrc:1 cname:test\r\n\ - m=audio 9 UDP/TLS/RTP/SAVPF 111\r\n"; - - let result = inject_provisional_ssrcs(sdp); - - // Should inject SSRCs 2, 3, 4 - assert!(result.contains("a=ssrc:2 msid:stream1 video1")); - assert!(result.contains("a=ssrc:3 msid:stream1 video1")); - assert!(result.contains("a=ssrc:4 msid:stream1 video1")); - assert!(result.contains("a=ssrc:2 cname:odrerir")); - assert!(result.contains("a=ssrc:3 cname:odrerir")); - assert!(result.contains("a=ssrc:4 cname:odrerir")); - - // Original SSRC should still be there - assert!(result.contains("a=ssrc:1 msid:stream1 video1")); - } - - #[test] - fn test_inject_provisional_ssrcs_without_existing() { - // SDP without any SSRC lines - let sdp = "v=0\r\n\ - m=video 9 UDP/TLS/RTP/SAVPF 96\r\n\ - a=msid:stream1 video1\r\n\ - a=rtpmap:96 H264/90000\r\n\ - m=audio 9 UDP/TLS/RTP/SAVPF 111\r\n"; - - let result = inject_provisional_ssrcs(sdp); - - // Should inject SSRCs 2, 3, 4 (no SSRC 1 since none existed) - assert!(result.contains("a=ssrc:2 msid:stream1 video1")); - assert!(result.contains("a=ssrc:3 msid:stream1 video1")); - assert!(result.contains("a=ssrc:4 msid:stream1 video1")); - - // SSRCs should be injected before the audio section - let video_pos = result.find("m=video").unwrap(); - let audio_pos = result.find("m=audio").unwrap(); - let ssrc2_pos = result.find("a=ssrc:2").unwrap(); - assert!(ssrc2_pos > video_pos && ssrc2_pos < audio_pos); - } - - #[test] - fn test_inject_provisional_ssrcs_already_declared() { - // SDP with SSRCs 1, 2, 3, 4 already declared - let sdp = "v=0\r\n\ - m=video 9 UDP/TLS/RTP/SAVPF 96\r\n\ - a=ssrc:1 cname:test\r\n\ - a=ssrc:2 cname:test\r\n\ - a=ssrc:3 cname:test\r\n\ - a=ssrc:4 cname:test\r\n\ - m=audio 9 UDP/TLS/RTP/SAVPF 111\r\n"; - - let result = inject_provisional_ssrcs(sdp); - - // Should not inject anything - all SSRCs already exist - // Count occurrences of a=ssrc:2 - let count = result.matches("a=ssrc:2").count(); - assert_eq!(count, 1, "Should not duplicate existing SSRC 2"); - } -} From 862ef6144e653482d4393fea323654f180316dfe Mon Sep 17 00:00:00 2001 From: Zortos Date: Wed, 25 Feb 2026 14:51:49 +0100 Subject: [PATCH 2/4] chore: Legal stuff [skip ci] --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9ec1a32..f5c59fa 100644 --- a/README.md +++ b/README.md @@ -44,11 +44,15 @@ > **Warning** > OpenNOW is under active development. Bugs and performance issues are expected while features are finalized. +> **Trademark & Affiliation Notice** +> OpenNOW is an independent community project and is **not affiliated with, endorsed by, or sponsored by NVIDIA Corporation**. +> **NVIDIA** and **GeForce NOW** are trademarks of NVIDIA Corporation. You must use your own GeForce NOW account. + --- ## What is OpenNOW? -OpenNOW is a community-built desktop client for [NVIDIA GeForce NOW](https://www.nvidia.com/en-us/geforce-now/), built with Electron and TypeScript. It gives you a fully open-source, cross-platform alternative to the official app — with zero telemetry, full transparency, and features the official client doesn't have. +OpenNOW is an independent, community-built desktop client for [NVIDIA GeForce NOW](https://www.nvidia.com/en-us/geforce-now/), built with Electron and TypeScript. It provides a fully open-source, cross-platform alternative to the official app with zero telemetry, full transparency, and power-user features. - 🔓 **Fully open source** — audit every line, fork it, improve it - 🚫 **No telemetry** — OpenNOW collects nothing @@ -201,7 +205,7 @@ opennow-stable/src/ ## FAQ **Is this the official GeForce NOW client?** -No. OpenNOW is a community-built alternative. It uses the same NVIDIA streaming infrastructure but is not affiliated with or endorsed by NVIDIA. +No. OpenNOW is an independent third-party client and is not affiliated with, endorsed by, or sponsored by NVIDIA. NVIDIA and GeForce NOW are trademarks of NVIDIA Corporation. **Was this project built in Rust before?** Yes. OpenNOW originally used Rust/Tauri but switched to Electron for better cross-platform compatibility and faster development. From 48a2623cc0c7805c2ed26d701b994e97f0761be5 Mon Sep 17 00:00:00 2001 From: Zortos Date: Wed, 25 Feb 2026 14:56:46 +0100 Subject: [PATCH 3/4] chore: More legal stuffs [skip ci] --- opennow-stable/src/main/gfn/errorCodes.ts | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/opennow-stable/src/main/gfn/errorCodes.ts b/opennow-stable/src/main/gfn/errorCodes.ts index 3c08d48..e78fc79 100644 --- a/opennow-stable/src/main/gfn/errorCodes.ts +++ b/opennow-stable/src/main/gfn/errorCodes.ts @@ -1,11 +1,10 @@ /** - * GFN CloudMatch Error Codes + * CloudMatch error codes. * - * Error code mappings extracted from the official GFN web client. - * These provide user-friendly error messages for session failures. + * These mappings provide user-friendly messages for session failures. */ -/** GFN Session Error Codes from official client */ +/** Session error code constants. */ export enum GfnErrorCode { // Success codes Success = 15859712, @@ -296,7 +295,7 @@ export const ERROR_MESSAGES: Map = new Map([ 3237093656, { title: "Under Maintenance", - description: "GeForce NOW is currently under maintenance. Please try again later.", + description: "The service is currently under maintenance. Please try again later.", }, ], [ @@ -394,14 +393,14 @@ export const ERROR_MESSAGES: Map = new Map([ 3237093684, { title: "Region Not Supported", - description: "GeForce NOW is not available in your region.", + description: "The service is not available in your region.", }, ], [ 3237093685, { title: "Region Banned", - description: "GeForce NOW is not available in your region.", + description: "The service is not available in your region.", }, ], [ @@ -534,7 +533,7 @@ export const ERROR_MESSAGES: Map = new Map([ 3237093722, { title: "Storage Error", - description: "GFN storage is not available.", + description: "Service storage is not available.", }, ], @@ -646,7 +645,7 @@ export interface SessionErrorInfo { unifiedErrorCode?: number; /** Session error code from session.errorCode */ sessionErrorCode?: number; - /** Computed GFN error code */ + /** Computed service error code */ gfnErrorCode: number; /** User-friendly title */ title: string; @@ -679,7 +678,7 @@ export class SessionError extends Error { public readonly unifiedErrorCode?: number; /** Session error code from session.errorCode */ public readonly sessionErrorCode?: number; - /** Computed GFN error code */ + /** Computed service error code */ public readonly gfnErrorCode: number; /** User-friendly title */ public readonly title: string; @@ -733,7 +732,7 @@ export class SessionError extends Error { const unifiedErrorCode = json.requestStatus?.unifiedErrorCode; const sessionErrorCode = json.session?.errorCode; - // Compute GFN error code using official client logic + // Compute normalized service error code const gfnErrorCode = SessionError.computeErrorCode(statusCode, unifiedErrorCode); // Get user-friendly message @@ -756,7 +755,7 @@ export class SessionError extends Error { } /** - * Compute GFN error code from CloudMatch response (matching official client logic) + * Compute service error code from CloudMatch response */ private static computeErrorCode(statusCode: number, unifiedErrorCode?: number): number { // Base error code From 7710dac1969d4058d3131b4e41c43348f609a6a7 Mon Sep 17 00:00:00 2001 From: Jared Date: Wed, 25 Feb 2026 18:16:39 -0600 Subject: [PATCH 4/4] feat(settings): add mouse sensitivity adjustment to settings page --- opennow-stable/src/renderer/src/App.tsx | 9 ++++++ .../renderer/src/components/SettingsPage.tsx | 30 +++++++++++++++++++ .../src/renderer/src/gfn/webrtcClient.ts | 16 ++++++++-- 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/opennow-stable/src/renderer/src/App.tsx b/opennow-stable/src/renderer/src/App.tsx index f62a8bd..b16db90 100644 --- a/opennow-stable/src/renderer/src/App.tsx +++ b/opennow-stable/src/renderer/src/App.tsx @@ -692,6 +692,7 @@ export function App(): JSX.Element { audioElement: audioRef.current, microphoneMode: settings.microphoneMode, microphoneDeviceId: settings.microphoneDeviceId || undefined, + mouseSensitivity: settings.mouseSensitivity, onLog: (line: string) => console.log(`[WebRTC] ${line}`), onStats: (stats) => setDiagnostics(stats), onEscHoldProgress: (visible, progress) => { @@ -760,6 +761,14 @@ export function App(): JSX.Element { if (settingsLoaded) { await window.openNow.setSetting(key, value); } + // If a running client exists, push certain settings live + if (key === "mouseSensitivity") { + try { + (clientRef.current as any)?.setMouseSensitivity?.(value as number); + } catch { + // ignore + } + } }, [settingsLoaded]); // Login handler diff --git a/opennow-stable/src/renderer/src/components/SettingsPage.tsx b/opennow-stable/src/renderer/src/components/SettingsPage.tsx index a19c115..0589288 100644 --- a/opennow-stable/src/renderer/src/components/SettingsPage.tsx +++ b/opennow-stable/src/renderer/src/components/SettingsPage.tsx @@ -1416,6 +1416,36 @@ export function SettingsPage({ settings, regions, onSettingChange }: SettingsPag Export Logs + + {/* Mouse Sensitivity */} +
+ +
+ handleChange("mouseSensitivity", parseFloat(e.target.value))} + /> + { + const v = parseFloat(e.target.value || "0"); + if (Number.isFinite(v)) handleChange("mouseSensitivity", Math.max(0.1, Math.min(4, v))); + }} + /> + Multiplier applied to mouse movement (1.00 = default) +
+
diff --git a/opennow-stable/src/renderer/src/gfn/webrtcClient.ts b/opennow-stable/src/renderer/src/gfn/webrtcClient.ts index 3911ba3..a6001e8 100644 --- a/opennow-stable/src/renderer/src/gfn/webrtcClient.ts +++ b/opennow-stable/src/renderer/src/gfn/webrtcClient.ts @@ -183,6 +183,8 @@ interface ClientOptions { microphoneMode?: MicrophoneMode; /** Preferred microphone device ID */ microphoneDeviceId?: string; + /** Mouse sensitivity multiplier (1.0 = default) */ + mouseSensitivity?: number; onLog: (line: string) => void; onStats?: (stats: StreamDiagnostics) => void; onEscHoldProgress?: (visible: boolean, progress: number) => void; @@ -503,6 +505,7 @@ export class GfnWebRtcClient { private mouseFlushLastTickMs = 0; private pendingMouseTimestampUs: bigint | null = null; private mouseDeltaFilter = new MouseDeltaFilter(); + private mouseSensitivity = 1; private partialReliableThresholdMs = GfnWebRtcClient.DEFAULT_PARTIAL_RELIABLE_THRESHOLD_MS; private inputQueuePeakBufferedBytesWindow = 0; @@ -557,6 +560,7 @@ export class GfnWebRtcClient { options.videoElement.srcObject = this.videoStream; options.audioElement.srcObject = this.audioStream; options.audioElement.muted = true; + this.mouseSensitivity = options.mouseSensitivity ?? 1; // Configure video element for lowest latency playback this.configureVideoElementForLowLatency(options.videoElement); @@ -605,6 +609,13 @@ export class GfnWebRtcClient { this.log("Video element configured for low-latency playback"); } + /** Update mouse sensitivity multiplier at runtime. */ + public setMouseSensitivity(value: number): void { + const v = Number.isFinite(value) ? value : 1; + this.mouseSensitivity = Math.max(0.01, v); + this.log(`Mouse sensitivity set to ${this.mouseSensitivity}`); + } + /** * Configure an RTCRtpReceiver for minimum jitter buffer delay. * @@ -1840,8 +1851,9 @@ export class GfnWebRtcClient { return; } - this.pendingMouseDx += Math.round(this.mouseDeltaFilter.getX()); - this.pendingMouseDy += Math.round(this.mouseDeltaFilter.getY()); + // Apply user-configured mouse sensitivity multiplier before queuing + this.pendingMouseDx += Math.round(this.mouseDeltaFilter.getX() * this.mouseSensitivity); + this.pendingMouseDy += Math.round(this.mouseDeltaFilter.getY() * this.mouseSensitivity); this.pendingMouseTimestampUs = timestampUs(eventTimestampMs); };