Skip to content

Use vtt for extracting preview sprites #765

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
damontecres merged 2 commits into from
Feb 22, 2026
Merged

Use vtt for extracting preview sprites #765

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5ac8a2c
Use vtt for extracting preview sprites
damontecres Feb 22, 2026
057cf7c
Use vtt url & remove debug logging
damontecres Feb 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions app/src/main/graphql/FindSceneMarkers.graphql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ fragment VideoSceneData on Scene {
preview
stream
sprite
vtt
}
sceneStreams {
url
Expand Down
4 changes: 4 additions & 0 deletions app/src/main/java/com/github/damontecres/stashapp/data/Scene.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ data class Scene(
val screenshotUrl: String?,
val streams: Map<String, String>,
val spriteUrl: String?,
val vttUrl: String?,
val duration: Double?,
val resumeTime: Double?,
val videoCodec: String?,
Expand Down Expand Up @@ -49,6 +50,7 @@ data class Scene(
screenshotUrl = data.paths.screenshot,
streams = streams,
spriteUrl = data.paths.sprite,
vttUrl = data.paths.vtt,
duration = fileData?.duration,
resumeTime = data.resume_time,
videoCodec = fileData?.video_codec,
Expand Down Expand Up @@ -78,6 +80,7 @@ data class Scene(
screenshotUrl = data.paths.screenshot,
streams = streams,
spriteUrl = data.paths.sprite,
vttUrl = data.paths.vtt,
duration = fileData?.duration,
resumeTime = data.resume_time,
videoCodec = fileData?.video_codec,
Expand Down Expand Up @@ -115,6 +118,7 @@ data class Scene(
screenshotUrl = video.paths.screenshot,
streams = streams,
spriteUrl = video.paths.sprite,
vttUrl = video.paths.vtt,
duration = fileData?.duration,
resumeTime = video.resume_time,
videoCodec = fileData?.video_codec,
Expand Down
83 changes: 38 additions & 45 deletions app/src/main/java/com/github/damontecres/stashapp/ui/components/playback/PlaybackOverlay.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,6 @@ fun PlaybackOverlay(
onPlaybackActionClick: (PlaybackAction) -> Unit,
onSeekBarChange: (Float) -> Unit,
showDebugInfo: Boolean,
spriteImageLoaded: Boolean,
moreButtonOptions: MoreButtonOptions,
subtitleIndex: Int?,
audioIndex: Int?,
Expand All @@ -163,6 +162,7 @@ fun PlaybackOverlay(
playlistInfo: PlaylistInfo?,
videoDecoder: String?,
audioDecoder: String?,
spriteData: List<SpriteData>,
modifier: Modifier = Modifier,
seekPreviewPlaceholder: Painter? = null,
seekBarInteractionSource: MutableInteractionSource = remember { MutableInteractionSource() },
Expand Down Expand Up @@ -394,7 +394,7 @@ fun PlaybackOverlay(
}
val yOffsetDp =
180.dp +
(if (spriteImageLoaded) (160.dp) else 24.dp) +
(if (spriteData.isNotEmpty()) (160.dp) else 24.dp) +
(if (markers.isEmpty()) (-24).dp else 0.dp)
val heightPx = with(LocalDensity.current) { yOffsetDp.toPx().toInt() }
SeekPreviewImage(
Expand All @@ -406,13 +406,10 @@ fun PlaybackOverlay(
yOffset = heightPx,
// yPercentage = 1 - controlHeight,
),
previewImageUrl = previewImageUrl,
imageLoaded = spriteImageLoaded,
imageLoader = imageLoader,
duration = playerControls.duration,
seekProgress = seekProgress,
videoWidth = scene.videoWidth,
videoHeight = scene.videoHeight,
spriteData = spriteData,
placeHolder = seekPreviewPlaceholder,
)
}
Expand All @@ -438,13 +435,10 @@ fun Modifier.offsetByPercent(

@Composable
fun SeekPreviewImage(
imageLoaded: Boolean,
previewImageUrl: String?,
imageLoader: ImageLoader,
duration: Long,
seekProgress: Float,
videoWidth: Int?,
videoHeight: Int?,
spriteData: List<SpriteData>,
modifier: Modifier = Modifier,
placeHolder: Painter? = null,
) {
Expand All @@ -455,41 +449,39 @@ fun SeekPreviewImage(
verticalArrangement = Arrangement.spacedBy(8.dp),
horizontalAlignment = Alignment.CenterHorizontally,
) {
if (imageLoaded &&
previewImageUrl.isNotNullOrBlank() &&
videoWidth != null &&
videoHeight != null
) {
val height = 160.dp
val width = height * (videoWidth.toFloat() / videoHeight)
val heightPx = with(LocalDensity.current) { height.toPx().toInt() }
val widthPx = with(LocalDensity.current) { width.toPx().toInt() }
if (spriteData.isNotEmpty()) {
val position = (duration * seekProgress.toDouble()).milliseconds
spriteData.firstOrNull { position >= it.start && position < it.end }?.let { s ->
val height = 160.dp
val width = height * (s.w.toFloat() / s.h)
val heightPx = with(LocalDensity.current) { height.toPx().toInt() }
val widthPx = with(LocalDensity.current) { width.toPx().toInt() }

AsyncImage(
modifier =
Modifier
.width(width)
.height(height)
.background(Color.Black)
.border(1.5.dp, color = MaterialTheme.colorScheme.border),
model =
ImageRequest
.Builder(context)
.data(previewImageUrl)
.memoryCachePolicy(CachePolicy.ENABLED)
.transformations(
CoilPreviewTransformation(
widthPx,
heightPx,
duration,
(duration * seekProgress).toLong(),
),
).build(),
contentScale = ContentScale.None,
imageLoader = imageLoader,
contentDescription = null,
placeholder = placeHolder,
)
AsyncImage(
modifier =
Modifier
.width(width)
.height(height)
.background(Color.Black)
.border(1.5.dp, color = MaterialTheme.colorScheme.border),
model =
ImageRequest
.Builder(context)
.data(s.url)
.memoryCachePolicy(CachePolicy.ENABLED)
.transformations(
CoilPreviewTransformation(
s,
widthPx,
heightPx,
),
).build(),
contentScale = ContentScale.None,
imageLoader = imageLoader,
contentDescription = null,
placeholder = placeHolder,
)
}
}
Text(
text = (seekProgress * duration / 1000).toLong().seconds.toString(),
Expand Down Expand Up @@ -656,6 +648,7 @@ private fun PlaybackOverlayPreview() {
screenshotUrl = "",
streams = mapOf(),
spriteUrl = "",
vttUrl = "",
duration = 600.2,
resumeTime = 0.0,
videoCodec = "h264",
Expand Down Expand Up @@ -698,7 +691,6 @@ private fun PlaybackOverlayPreview() {
seekPreviewEnabled = true,
nextEnabled = true,
seekEnabled = true,
spriteImageLoaded = false,
moreButtonOptions = MoreButtonOptions(mapOf()),
subtitleIndex = 1,
modifier =
Expand Down Expand Up @@ -730,6 +722,7 @@ private fun PlaybackOverlayPreview() {
format = Format.Builder().build(),
),
),
spriteData = emptyList(),
videoDecoder = "OMX.video.decoder",
audioDecoder = "OMX.audio.decoder",
)
Expand Down
90 changes: 84 additions & 6 deletions ...c/main/java/com/github/damontecres/stashapp/ui/components/playback/PlaybackPageContent.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,8 @@ import androidx.media3.common.text.CueGroup
import androidx.media3.common.util.UnstableApi
import androidx.media3.common.util.Util
import androidx.media3.exoplayer.ExoPlayer
import androidx.media3.extractor.text.SubtitleParser
import androidx.media3.extractor.text.webvtt.WebvttParser
import androidx.media3.ui.SubtitleView
import androidx.media3.ui.compose.PlayerSurface
import androidx.media3.ui.compose.SURFACE_TYPE_SURFACE_VIEW
Expand Down Expand Up @@ -115,6 +117,7 @@ import com.github.damontecres.stashapp.util.ComposePager
import com.github.damontecres.stashapp.util.LoggingCoroutineExceptionHandler
import com.github.damontecres.stashapp.util.MutationEngine
import com.github.damontecres.stashapp.util.QueryEngine
import com.github.damontecres.stashapp.util.StashClient
import com.github.damontecres.stashapp.util.StashCoroutineExceptionHandler
import com.github.damontecres.stashapp.util.StashServer
import com.github.damontecres.stashapp.util.findActivity
Expand All @@ -128,8 +131,11 @@ import kotlinx.coroutines.Job
import kotlinx.coroutines.cancelChildren
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import okhttp3.Request
import java.util.Locale
import kotlin.properties.Delegates
import kotlin.time.Duration
import kotlin.time.Duration.Companion.microseconds
import kotlin.time.Duration.Companion.milliseconds

const val TAG = "PlaybackPageContent"
Expand All @@ -152,7 +158,7 @@ class PlaybackViewModel : ViewModel() {
val markers = MutableLiveData<List<BasicMarker>>(listOf())
val oCount = MutableLiveData(0)
val rating100 = MutableLiveData(0)
val spriteImageLoaded = MutableLiveData(false)
val spriteImageLoaded = MutableLiveData<List<SpriteData>>(emptyList())

private val _videoFilter = MutableLiveData<VideoFilter?>(null)
val videoFilter = ThrottledLiveData(_videoFilter, 500L)
Expand Down Expand Up @@ -190,7 +196,7 @@ class PlaybackViewModel : ViewModel() {
this.oCount.value = 0
this.rating100.value = 0
this.markers.value = listOf()
this.spriteImageLoaded.value = false
this.spriteImageLoaded.value = emptyList()

if (trackActivity) {
Log.v(
Expand Down Expand Up @@ -244,7 +250,7 @@ class PlaybackViewModel : ViewModel() {
viewModelScope.launch(sceneJob + StashCoroutineExceptionHandler()) {
val context = StashApplication.getApplication()
val imageLoader = SingletonImageLoader.get(context)
if (tag.item.spriteUrl.isNotNullOrBlank()) {
if (tag.item.spriteUrl.isNotNullOrBlank() && tag.item.vttUrl.isNotNullOrBlank()) {
val request =
ImageRequest
.Builder(context)
Expand All @@ -253,11 +259,73 @@ class PlaybackViewModel : ViewModel() {
.scale(Scale.FILL)
.build()
val result = imageLoader.enqueue(request).job.await()
spriteImageLoaded.value = result.image != null
if (result.image != null) {
spriteImageLoaded.value = fetchSprites(tag.item.id, tag.item.vttUrl)
}
}
}
}

@OptIn(UnstableApi::class)
private suspend fun fetchSprites(
sceneId: String,
vttUrl: String,
): List<SpriteData> =
withContext(Dispatchers.Default) {
val res =
withContext(Dispatchers.IO) {
server.okHttpClient
.newCall(
Request.Builder().url(vttUrl).build(),
).execute()
}
if (res.isSuccessful) {
res.body.use {
it?.bytes()?.let {
try {
val baseUrl = StashClient.getServerRoot(server.url)
val regex = Regex("(\\w+\\.\\w+)#xywh=(\\d+),(\\d+),(\\d+),(\\d+)")
val spriteData = mutableListOf<SpriteData>()
WebvttParser().parse(it, SubtitleParser.OutputOptions.allCues()) {
val start = it.startTimeUs.microseconds
val end = it.endTimeUs.microseconds
it.cues.firstOrNull()?.text?.let { cue ->
val m = regex.matchEntire(cue)
if (m != null) {
val url = "$baseUrl/scene/${m.groupValues[1]}"
val x = m.groupValues[2].toInt()
val y = m.groupValues[3].toInt()
val w = m.groupValues[4].toInt()
val h = m.groupValues[5].toInt()
val sprite =
SpriteData(
start = start,
end = end,
url = url,
x = x,
y = y,
w = w,
h = h,
)
// Log.v(TAG, "sprite=$sprite")
spriteData.add(sprite)
}
}
}
return@withContext spriteData
} catch (ex: Exception) {
Log.w(TAG, "Error parsing sprites for $sceneId", ex)
return@withContext emptyList()
}
}
emptyList()
}
} else {
Log.d(TAG, "No sprites for $sceneId")
return@withContext emptyList()
}
}

private fun refreshScene(sceneId: String) {
// Fetch o count & markers
viewModelScope.launch(sceneJob + exceptionHandler) {
Expand Down Expand Up @@ -366,6 +434,16 @@ val playbackScaleOptions =
ContentScale.FillHeight to "Fill Height",
)

data class SpriteData(
val start: Duration,
val end: Duration,
val url: String,
val x: Int,
val y: Int,
val w: Int,
val h: Int,
)

@OptIn(UnstableApi::class)
@Composable
fun PlaybackPageContent(
Expand Down Expand Up @@ -397,7 +475,7 @@ fun PlaybackPageContent(
val markers by viewModel.markers.observeAsState(listOf())
val oCount by viewModel.oCount.observeAsState(0)
val rating100 by viewModel.rating100.observeAsState(0)
val spriteImageLoaded by viewModel.spriteImageLoaded.observeAsState(false)
val spriteImageLoaded by viewModel.spriteImageLoaded.observeAsState(emptyList())
var currentTracks by remember { mutableStateOf<List<TrackSupport>>(listOf()) }
var captions by remember { mutableStateOf<List<TrackSupport>>(listOf()) }
var subtitles by remember { mutableStateOf<List<Cue>?>(null) }
Expand Down Expand Up @@ -862,7 +940,6 @@ fun PlaybackPageContent(
seekEnabled = seekBarState.isEnabled,
seekPreviewEnabled = !isMarkerPlaylist,
showDebugInfo = showDebugInfo,
spriteImageLoaded = spriteImageLoaded,
moreButtonOptions =
MoreButtonOptions(
buildMap {
Expand Down Expand Up @@ -893,6 +970,7 @@ fun PlaybackPageContent(
},
videoDecoder = videoDecoder,
audioDecoder = audioDecoder,
spriteData = spriteImageLoaded,
)
}
}
Expand Down
Loading
Loading