diff --git a/app/src/main/graphql/FindSceneMarkers.graphql b/app/src/main/graphql/FindSceneMarkers.graphql index 2ba3d9e9f..ca1c2b833 100644 --- a/app/src/main/graphql/FindSceneMarkers.graphql +++ b/app/src/main/graphql/FindSceneMarkers.graphql @@ -66,6 +66,7 @@ fragment VideoSceneData on Scene { preview stream sprite + vtt } sceneStreams { url diff --git a/app/src/main/java/com/github/damontecres/stashapp/data/Scene.kt b/app/src/main/java/com/github/damontecres/stashapp/data/Scene.kt index b3befa3b7..e7396736d 100644 --- a/app/src/main/java/com/github/damontecres/stashapp/data/Scene.kt +++ b/app/src/main/java/com/github/damontecres/stashapp/data/Scene.kt @@ -17,6 +17,7 @@ data class Scene( val screenshotUrl: String?, val streams: Map, val spriteUrl: String?, + val vttUrl: String?, val duration: Double?, val resumeTime: Double?, val videoCodec: String?, @@ -49,6 +50,7 @@ data class Scene( screenshotUrl = data.paths.screenshot, streams = streams, spriteUrl = data.paths.sprite, + vttUrl = data.paths.vtt, duration = fileData?.duration, resumeTime = data.resume_time, videoCodec = fileData?.video_codec, @@ -78,6 +80,7 @@ data class Scene( screenshotUrl = data.paths.screenshot, streams = streams, spriteUrl = data.paths.sprite, + vttUrl = data.paths.vtt, duration = fileData?.duration, resumeTime = data.resume_time, videoCodec = fileData?.video_codec, @@ -115,6 +118,7 @@ data class Scene( screenshotUrl = video.paths.screenshot, streams = streams, spriteUrl = video.paths.sprite, + vttUrl = video.paths.vtt, duration = fileData?.duration, resumeTime = video.resume_time, videoCodec = fileData?.video_codec, diff --git a/app/src/main/java/com/github/damontecres/stashapp/ui/components/playback/PlaybackOverlay.kt b/app/src/main/java/com/github/damontecres/stashapp/ui/components/playback/PlaybackOverlay.kt index e502cbe1d..836c902d4 100644 --- a/app/src/main/java/com/github/damontecres/stashapp/ui/components/playback/PlaybackOverlay.kt +++ b/app/src/main/java/com/github/damontecres/stashapp/ui/components/playback/PlaybackOverlay.kt @@ -153,7 +153,6 @@ fun PlaybackOverlay( onPlaybackActionClick: (PlaybackAction) -> Unit, onSeekBarChange: (Float) -> Unit, showDebugInfo: Boolean, - spriteImageLoaded: Boolean, moreButtonOptions: MoreButtonOptions, subtitleIndex: Int?, audioIndex: Int?, @@ -163,6 +162,7 @@ fun PlaybackOverlay( playlistInfo: PlaylistInfo?, videoDecoder: String?, audioDecoder: String?, + spriteData: List, modifier: Modifier = Modifier, seekPreviewPlaceholder: Painter? = null, seekBarInteractionSource: MutableInteractionSource = remember { MutableInteractionSource() }, @@ -394,7 +394,7 @@ fun PlaybackOverlay( } val yOffsetDp = 180.dp + - (if (spriteImageLoaded) (160.dp) else 24.dp) + + (if (spriteData.isNotEmpty()) (160.dp) else 24.dp) + (if (markers.isEmpty()) (-24).dp else 0.dp) val heightPx = with(LocalDensity.current) { yOffsetDp.toPx().toInt() } SeekPreviewImage( @@ -406,13 +406,10 @@ fun PlaybackOverlay( yOffset = heightPx, // yPercentage = 1 - controlHeight, ), - previewImageUrl = previewImageUrl, - imageLoaded = spriteImageLoaded, imageLoader = imageLoader, duration = playerControls.duration, seekProgress = seekProgress, - videoWidth = scene.videoWidth, - videoHeight = scene.videoHeight, + spriteData = spriteData, placeHolder = seekPreviewPlaceholder, ) } @@ -438,13 +435,10 @@ fun Modifier.offsetByPercent( @Composable fun SeekPreviewImage( - imageLoaded: Boolean, - previewImageUrl: String?, imageLoader: ImageLoader, duration: Long, seekProgress: Float, - videoWidth: Int?, - videoHeight: Int?, + spriteData: List, modifier: Modifier = Modifier, placeHolder: Painter? = null, ) { @@ -455,41 +449,39 @@ fun SeekPreviewImage( verticalArrangement = Arrangement.spacedBy(8.dp), horizontalAlignment = Alignment.CenterHorizontally, ) { - if (imageLoaded && - previewImageUrl.isNotNullOrBlank() && - videoWidth != null && - videoHeight != null - ) { - val height = 160.dp - val width = height * (videoWidth.toFloat() / videoHeight) - val heightPx = with(LocalDensity.current) { height.toPx().toInt() } - val widthPx = with(LocalDensity.current) { width.toPx().toInt() } + if (spriteData.isNotEmpty()) { + val position = (duration * seekProgress.toDouble()).milliseconds + spriteData.firstOrNull { position >= it.start && position < it.end }?.let { s -> + val height = 160.dp + val width = height * (s.w.toFloat() / s.h) + val heightPx = with(LocalDensity.current) { height.toPx().toInt() } + val widthPx = with(LocalDensity.current) { width.toPx().toInt() } - AsyncImage( - modifier = - Modifier - .width(width) - .height(height) - .background(Color.Black) - .border(1.5.dp, color = MaterialTheme.colorScheme.border), - model = - ImageRequest - .Builder(context) - .data(previewImageUrl) - .memoryCachePolicy(CachePolicy.ENABLED) - .transformations( - CoilPreviewTransformation( - widthPx, - heightPx, - duration, - (duration * seekProgress).toLong(), - ), - ).build(), - contentScale = ContentScale.None, - imageLoader = imageLoader, - contentDescription = null, - placeholder = placeHolder, - ) + AsyncImage( + modifier = + Modifier + .width(width) + .height(height) + .background(Color.Black) + .border(1.5.dp, color = MaterialTheme.colorScheme.border), + model = + ImageRequest + .Builder(context) + .data(s.url) + .memoryCachePolicy(CachePolicy.ENABLED) + .transformations( + CoilPreviewTransformation( + s, + widthPx, + heightPx, + ), + ).build(), + contentScale = ContentScale.None, + imageLoader = imageLoader, + contentDescription = null, + placeholder = placeHolder, + ) + } } Text( text = (seekProgress * duration / 1000).toLong().seconds.toString(), @@ -656,6 +648,7 @@ private fun PlaybackOverlayPreview() { screenshotUrl = "", streams = mapOf(), spriteUrl = "", + vttUrl = "", duration = 600.2, resumeTime = 0.0, videoCodec = "h264", @@ -698,7 +691,6 @@ private fun PlaybackOverlayPreview() { seekPreviewEnabled = true, nextEnabled = true, seekEnabled = true, - spriteImageLoaded = false, moreButtonOptions = MoreButtonOptions(mapOf()), subtitleIndex = 1, modifier = @@ -730,6 +722,7 @@ private fun PlaybackOverlayPreview() { format = Format.Builder().build(), ), ), + spriteData = emptyList(), videoDecoder = "OMX.video.decoder", audioDecoder = "OMX.audio.decoder", ) diff --git a/app/src/main/java/com/github/damontecres/stashapp/ui/components/playback/PlaybackPageContent.kt b/app/src/main/java/com/github/damontecres/stashapp/ui/components/playback/PlaybackPageContent.kt index 19a5639b8..32d63fd13 100644 --- a/app/src/main/java/com/github/damontecres/stashapp/ui/components/playback/PlaybackPageContent.kt +++ b/app/src/main/java/com/github/damontecres/stashapp/ui/components/playback/PlaybackPageContent.kt @@ -67,6 +67,8 @@ import androidx.media3.common.text.CueGroup import androidx.media3.common.util.UnstableApi import androidx.media3.common.util.Util import androidx.media3.exoplayer.ExoPlayer +import androidx.media3.extractor.text.SubtitleParser +import androidx.media3.extractor.text.webvtt.WebvttParser import androidx.media3.ui.SubtitleView import androidx.media3.ui.compose.PlayerSurface import androidx.media3.ui.compose.SURFACE_TYPE_SURFACE_VIEW @@ -115,6 +117,7 @@ import com.github.damontecres.stashapp.util.ComposePager import com.github.damontecres.stashapp.util.LoggingCoroutineExceptionHandler import com.github.damontecres.stashapp.util.MutationEngine import com.github.damontecres.stashapp.util.QueryEngine +import com.github.damontecres.stashapp.util.StashClient import com.github.damontecres.stashapp.util.StashCoroutineExceptionHandler import com.github.damontecres.stashapp.util.StashServer import com.github.damontecres.stashapp.util.findActivity @@ -128,8 +131,11 @@ import kotlinx.coroutines.Job import kotlinx.coroutines.cancelChildren import kotlinx.coroutines.launch import kotlinx.coroutines.withContext +import okhttp3.Request import java.util.Locale import kotlin.properties.Delegates +import kotlin.time.Duration +import kotlin.time.Duration.Companion.microseconds import kotlin.time.Duration.Companion.milliseconds const val TAG = "PlaybackPageContent" @@ -152,7 +158,7 @@ class PlaybackViewModel : ViewModel() { val markers = MutableLiveData>(listOf()) val oCount = MutableLiveData(0) val rating100 = MutableLiveData(0) - val spriteImageLoaded = MutableLiveData(false) + val spriteImageLoaded = MutableLiveData>(emptyList()) private val _videoFilter = MutableLiveData(null) val videoFilter = ThrottledLiveData(_videoFilter, 500L) @@ -190,7 +196,7 @@ class PlaybackViewModel : ViewModel() { this.oCount.value = 0 this.rating100.value = 0 this.markers.value = listOf() - this.spriteImageLoaded.value = false + this.spriteImageLoaded.value = emptyList() if (trackActivity) { Log.v( @@ -244,7 +250,7 @@ class PlaybackViewModel : ViewModel() { viewModelScope.launch(sceneJob + StashCoroutineExceptionHandler()) { val context = StashApplication.getApplication() val imageLoader = SingletonImageLoader.get(context) - if (tag.item.spriteUrl.isNotNullOrBlank()) { + if (tag.item.spriteUrl.isNotNullOrBlank() && tag.item.vttUrl.isNotNullOrBlank()) { val request = ImageRequest .Builder(context) @@ -253,11 +259,73 @@ class PlaybackViewModel : ViewModel() { .scale(Scale.FILL) .build() val result = imageLoader.enqueue(request).job.await() - spriteImageLoaded.value = result.image != null + if (result.image != null) { + spriteImageLoaded.value = fetchSprites(tag.item.id, tag.item.vttUrl) + } } } } + @OptIn(UnstableApi::class) + private suspend fun fetchSprites( + sceneId: String, + vttUrl: String, + ): List = + withContext(Dispatchers.Default) { + val res = + withContext(Dispatchers.IO) { + server.okHttpClient + .newCall( + Request.Builder().url(vttUrl).build(), + ).execute() + } + if (res.isSuccessful) { + res.body.use { + it?.bytes()?.let { + try { + val baseUrl = StashClient.getServerRoot(server.url) + val regex = Regex("(\\w+\\.\\w+)#xywh=(\\d+),(\\d+),(\\d+),(\\d+)") + val spriteData = mutableListOf() + WebvttParser().parse(it, SubtitleParser.OutputOptions.allCues()) { + val start = it.startTimeUs.microseconds + val end = it.endTimeUs.microseconds + it.cues.firstOrNull()?.text?.let { cue -> + val m = regex.matchEntire(cue) + if (m != null) { + val url = "$baseUrl/scene/${m.groupValues[1]}" + val x = m.groupValues[2].toInt() + val y = m.groupValues[3].toInt() + val w = m.groupValues[4].toInt() + val h = m.groupValues[5].toInt() + val sprite = + SpriteData( + start = start, + end = end, + url = url, + x = x, + y = y, + w = w, + h = h, + ) +// Log.v(TAG, "sprite=$sprite") + spriteData.add(sprite) + } + } + } + return@withContext spriteData + } catch (ex: Exception) { + Log.w(TAG, "Error parsing sprites for $sceneId", ex) + return@withContext emptyList() + } + } + emptyList() + } + } else { + Log.d(TAG, "No sprites for $sceneId") + return@withContext emptyList() + } + } + private fun refreshScene(sceneId: String) { // Fetch o count & markers viewModelScope.launch(sceneJob + exceptionHandler) { @@ -366,6 +434,16 @@ val playbackScaleOptions = ContentScale.FillHeight to "Fill Height", ) +data class SpriteData( + val start: Duration, + val end: Duration, + val url: String, + val x: Int, + val y: Int, + val w: Int, + val h: Int, +) + @OptIn(UnstableApi::class) @Composable fun PlaybackPageContent( @@ -397,7 +475,7 @@ fun PlaybackPageContent( val markers by viewModel.markers.observeAsState(listOf()) val oCount by viewModel.oCount.observeAsState(0) val rating100 by viewModel.rating100.observeAsState(0) - val spriteImageLoaded by viewModel.spriteImageLoaded.observeAsState(false) + val spriteImageLoaded by viewModel.spriteImageLoaded.observeAsState(emptyList()) var currentTracks by remember { mutableStateOf>(listOf()) } var captions by remember { mutableStateOf>(listOf()) } var subtitles by remember { mutableStateOf?>(null) } @@ -862,7 +940,6 @@ fun PlaybackPageContent( seekEnabled = seekBarState.isEnabled, seekPreviewEnabled = !isMarkerPlaylist, showDebugInfo = showDebugInfo, - spriteImageLoaded = spriteImageLoaded, moreButtonOptions = MoreButtonOptions( buildMap { @@ -893,6 +970,7 @@ fun PlaybackPageContent( }, videoDecoder = videoDecoder, audioDecoder = audioDecoder, + spriteData = spriteImageLoaded, ) } } diff --git a/app/src/main/java/com/github/damontecres/stashapp/ui/pages/PlaybackPage.kt b/app/src/main/java/com/github/damontecres/stashapp/ui/pages/PlaybackPage.kt index 4cfc8c406..f1d4ba17c 100644 --- a/app/src/main/java/com/github/damontecres/stashapp/ui/pages/PlaybackPage.kt +++ b/app/src/main/java/com/github/damontecres/stashapp/ui/pages/PlaybackPage.kt @@ -2,18 +2,16 @@ package com.github.damontecres.stashapp.ui.pages import android.content.Context import android.util.Log -import android.widget.Toast import androidx.compose.foundation.background import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.runtime.Composable import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.collectAsState import androidx.compose.runtime.getValue import androidx.compose.runtime.livedata.observeAsState import androidx.compose.runtime.mutableStateListOf -import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.runtime.rememberCoroutineScope -import androidx.compose.runtime.setValue import androidx.compose.ui.Modifier import androidx.compose.ui.graphics.Color import androidx.compose.ui.platform.LocalContext @@ -24,7 +22,6 @@ import androidx.media3.common.Player import com.apollographql.apollo.api.Optional import com.github.damontecres.stashapp.StashExoPlayer import com.github.damontecres.stashapp.api.fragment.FullMarkerData -import com.github.damontecres.stashapp.api.fragment.FullSceneData import com.github.damontecres.stashapp.api.fragment.StashData import com.github.damontecres.stashapp.api.fragment.VideoSceneData import com.github.damontecres.stashapp.api.type.CriterionModifier @@ -46,9 +43,9 @@ import com.github.damontecres.stashapp.ui.FilterViewModel import com.github.damontecres.stashapp.ui.components.CircularProgress import com.github.damontecres.stashapp.ui.components.ItemOnClicker import com.github.damontecres.stashapp.ui.components.playback.PlaybackPageContent +import com.github.damontecres.stashapp.ui.util.OneTimeLaunchedEffect import com.github.damontecres.stashapp.util.AlphabetSearchUtils import com.github.damontecres.stashapp.util.LoggingCoroutineExceptionHandler -import com.github.damontecres.stashapp.util.QueryEngine import com.github.damontecres.stashapp.util.SkipParams import com.github.damontecres.stashapp.util.StashServer import kotlinx.coroutines.launch @@ -67,9 +64,10 @@ fun PlaybackPage( playbackMode: PlaybackMode, itemOnClick: ItemOnClicker, modifier: Modifier = Modifier, + viewModel: PlaybackPageViewModel = viewModel(), ) { - var scene by remember { mutableStateOf(null) } - val scope = rememberCoroutineScope() + OneTimeLaunchedEffect { viewModel.init(server, sceneId) } + val state by viewModel.state.collectAsState() val context = LocalContext.current val playbackMode = @@ -80,26 +78,7 @@ fun PlaybackPage( playbackMode } } - - LaunchedEffect(server, sceneId) { - scope.launch( - LoggingCoroutineExceptionHandler( - server, - scope, - toastMessage = "Error fetching scene", - ), - ) { - val fullScene = QueryEngine(server).getScene(sceneId) - if (fullScene != null) { - scene = fullScene - } else { - Log.w("PlaybackPage", "Scene $sceneId not found") - Toast.makeText(context, "Scene $sceneId not found", Toast.LENGTH_LONG).show() - } - } - } - Log.d("PlaybackPage", "scene=${scene?.id}") - scene?.let { + state?.let { state -> val player = remember { val skipParams = @@ -125,7 +104,7 @@ fun PlaybackPage( playWhenReady = true } } - val playbackScene = remember { Scene.fromFullSceneData(it) } + val playbackScene = state.scene val decision = remember { getStreamDecision( diff --git a/app/src/main/java/com/github/damontecres/stashapp/ui/pages/PlaybackPageViewModel.kt b/app/src/main/java/com/github/damontecres/stashapp/ui/pages/PlaybackPageViewModel.kt new file mode 100644 index 000000000..bcd2f79e1 --- /dev/null +++ b/app/src/main/java/com/github/damontecres/stashapp/ui/pages/PlaybackPageViewModel.kt @@ -0,0 +1,51 @@ +package com.github.damontecres.stashapp.ui.pages + +import android.util.Log +import android.widget.Toast +import androidx.lifecycle.ViewModel +import androidx.lifecycle.viewModelScope +import com.github.damontecres.stashapp.StashApplication +import com.github.damontecres.stashapp.api.fragment.FullSceneData +import com.github.damontecres.stashapp.data.Scene +import com.github.damontecres.stashapp.util.LoggingCoroutineExceptionHandler +import com.github.damontecres.stashapp.util.QueryEngine +import com.github.damontecres.stashapp.util.StashServer +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.launch + +class PlaybackPageViewModel : ViewModel() { + private lateinit var server: StashServer + private lateinit var sceneId: String + + val state = MutableStateFlow(null) + + fun init( + server: StashServer, + sceneId: String, + ) { + this.server = server + this.sceneId = sceneId + Log.d("PlaybackViewModel", "scene=$sceneId") + viewModelScope.launch( + LoggingCoroutineExceptionHandler( + server, + viewModelScope, + toastMessage = "Error fetching scene", + ), + ) { + val fullScene = QueryEngine(server).getScene(sceneId) + if (fullScene != null) { + val scene = Scene.fromFullSceneData(fullScene) + state.value = PlaybackState(fullScene, scene) + } else { + Log.w("PlaybackViewModel", "Scene $sceneId not found") + Toast.makeText(StashApplication.getApplication(), "Scene $sceneId not found", Toast.LENGTH_LONG).show() + } + } + } +} + +data class PlaybackState( + val fullScene: FullSceneData, + val scene: Scene, +) diff --git a/app/src/main/java/com/github/damontecres/stashapp/ui/util/CoilPreviewTransformation.kt b/app/src/main/java/com/github/damontecres/stashapp/ui/util/CoilPreviewTransformation.kt index 939a9152d..da653c8ce 100644 --- a/app/src/main/java/com/github/damontecres/stashapp/ui/util/CoilPreviewTransformation.kt +++ b/app/src/main/java/com/github/damontecres/stashapp/ui/util/CoilPreviewTransformation.kt @@ -5,38 +5,23 @@ import androidx.core.graphics.scale import coil3.size.Size import coil3.size.pxOrElse import coil3.transform.Transformation -import com.github.damontecres.stashapp.util.StashPreviewLoader.GlideThumbnailTransformation.Companion.MAX_COLUMNS -import com.github.damontecres.stashapp.util.StashPreviewLoader.GlideThumbnailTransformation.Companion.MAX_LINES +import com.github.damontecres.stashapp.ui.components.playback.SpriteData class CoilPreviewTransformation( + val s: SpriteData, val targetWidth: Int, val targetHeight: Int, - duration: Long, - position: Long, ) : Transformation() { - private val x: Int - private val y: Int - - init { - val square = position / (duration / (MAX_LINES * MAX_COLUMNS)) - y = square.toInt() / MAX_LINES - x = square.toInt() % MAX_COLUMNS - } - override val cacheKey: String - get() = "CoilPreviewTransformation_$x,$y" + get() = "CoilPreviewTransformation_$s" override suspend fun transform( input: Bitmap, size: Size, - ): Bitmap { - val width = input.width / MAX_COLUMNS - val height = input.height / MAX_LINES -// Log.d(TAG, "input.width=${input.width}, input.height=${input.height}, width=$width, height=$height, size=$size") - return Bitmap - .createBitmap(input, x * width, y * height, width, height) + ): Bitmap = + Bitmap + .createBitmap(input, s.x, s.y, s.w, s.h) .scale(size.width.pxOrElse { targetWidth }, size.height.pxOrElse { targetHeight }) - } companion object { private const val TAG = "CoilPreviewTransformation" diff --git a/app/src/main/java/com/github/damontecres/stashapp/util/Constants.kt b/app/src/main/java/com/github/damontecres/stashapp/util/Constants.kt index a1e19e0f5..b247b344d 100644 --- a/app/src/main/java/com/github/damontecres/stashapp/util/Constants.kt +++ b/app/src/main/java/com/github/damontecres/stashapp/util/Constants.kt @@ -534,6 +534,7 @@ val FullSceneData.asVideoSceneData: VideoSceneData paths.preview, paths.stream, paths.sprite, + paths.vtt, ), sceneStreams.map { VideoSceneData.SceneStream(it.url, it.mime_type, it.label) }, captions?.map { VideoSceneData.Caption("", it.caption) }, @@ -886,6 +887,16 @@ fun CoroutineScope.launchIO( launch(Dispatchers.IO + exceptionHandler, block = block) } +fun CoroutineScope.launchDefault( + exceptionHandler: CoroutineExceptionHandler? = StashCoroutineExceptionHandler(), + block: suspend CoroutineScope.() -> Unit, +): Job = + if (exceptionHandler == null) { + launch(Dispatchers.Default, block = block) + } else { + launch(Dispatchers.Default + exceptionHandler, block = block) + } + fun Bundle.putDataType(dataType: DataType): Bundle { this.putString("dataType", dataType.name) return this