From 0c9bffc3ab912af5561f592253c4b0b28aeb76d3 Mon Sep 17 00:00:00 2001 From: justinzeus Date: Sun, 8 Nov 2026 00:41:26 +0100 Subject: [PATCH] Add a subtitle delay renderer wrapper for ExoPlayer --- .../services/SubtitleDelayRenderer.kt | 216 ++++++++++++++++++ .../services/SubtitleDelayRendererTest.kt | 165 +++++++++++++ 2 files changed, 381 insertions(+) create mode 100644 app/src/main/java/com/github/damontecres/wholphin/services/SubtitleDelayRenderer.kt create mode 100644 app/src/test/java/com/github/damontecres/wholphin/services/SubtitleDelayRendererTest.kt diff --git a/app/src/main/java/com/github/damontecres/wholphin/services/SubtitleDelayRenderer.kt b/app/src/main/java/com/github/damontecres/wholphin/services/SubtitleDelayRenderer.kt new file mode 100644 index 00000000..88feaae1 --- /dev/null +++ b/app/src/main/java/com/github/damontecres/wholphin/services/SubtitleDelayRenderer.kt @@ -0,0 +1,216 @@ +@file:OptIn(markerClass = [UnstableApi::class]) + +package com.github.damontecres.wholphin.services + +import android.os.Handler +import androidx.annotation.OptIn +import androidx.media3.common.C +import androidx.media3.common.Format +import androidx.media3.common.Timeline +import androidx.media3.common.util.UnstableApi +import androidx.media3.exoplayer.ForwardingRenderer +import androidx.media3.exoplayer.Renderer +import androidx.media3.exoplayer.RendererConfiguration +import androidx.media3.exoplayer.RenderersFactory +import androidx.media3.exoplayer.audio.AudioRendererEventListener +import androidx.media3.exoplayer.metadata.MetadataOutput +import androidx.media3.exoplayer.source.MediaSource +import androidx.media3.exoplayer.source.SampleStream +import androidx.media3.exoplayer.text.TextOutput +import androidx.media3.exoplayer.video.VideoRendererEventListener +import io.github.peerless2012.ass.media.render.AssRenderer + +/** + * Delays the subtitles of the wrapped renderer by shifting the playback position it is given. + * + * A positive delay shows subtitles later, a negative delay earlier, the same as mpv's `sub-delay`. + * The delay is set on the playback thread with a [MSG_SET_SUBTITLE_DELAY] message whose payload is + * the delay in microseconds as a [Long]. + * + * Wrapping hides the renderer's type from ExoPlayer's `instanceof TextRenderer` checks, which only + * matter for clipped streams and server-side inserted ads. + */ +class SubtitleDelayRenderer( + internal val delegate: Renderer, +) : ForwardingRenderer(delegate) { + private val period = Timeline.Period() + private var delayUs = 0L + private var timeline: Timeline = Timeline.EMPTY + private var mediaPeriodId: MediaSource.MediaPeriodId? = null + private var streamOffsetUs = 0L + private var periodEndPositionUs = C.TIME_UNSET + private var lastPositionUs = C.TIME_UNSET + + override fun handleMessage( + messageType: Int, + message: Any?, + ) { + if (messageType == MSG_SET_SUBTITLE_DELAY) { + delayUs = message as Long + } else { + super.handleMessage(messageType, message) + } + } + + override fun enable( + configuration: RendererConfiguration, + formats: Array, + stream: SampleStream, + positionUs: Long, + joining: Boolean, + mayRenderStartOfStream: Boolean, + startPositionUs: Long, + offsetUs: Long, + mediaPeriodId: MediaSource.MediaPeriodId, + ) { + lastPositionUs = positionUs + updateStream(offsetUs, mediaPeriodId) + super.enable( + configuration, + formats, + stream, + positionUs - delayUs, + joining, + mayRenderStartOfStream, + startPositionUs, + offsetUs, + mediaPeriodId, + ) + } + + override fun replaceStream( + formats: Array, + stream: SampleStream, + startPositionUs: Long, + offsetUs: Long, + mediaPeriodId: MediaSource.MediaPeriodId, + ) { + updateStream(offsetUs, mediaPeriodId) + super.replaceStream(formats, stream, startPositionUs, offsetUs, mediaPeriodId) + } + + override fun setTimeline(timeline: Timeline) { + this.timeline = timeline + updatePeriodEndPosition() + super.setTimeline(timeline) + } + + override fun render( + positionUs: Long, + elapsedRealtimeUs: Long, + ) { + lastPositionUs = positionUs + super.render(positionUs - delayUs, elapsedRealtimeUs) + } + + override fun resetPosition( + positionUs: Long, + sampleStreamIsResetToKeyFrame: Boolean, + ) { + lastPositionUs = positionUs + super.resetPosition(positionUs - delayUs, sampleStreamIsResetToKeyFrame) + } + + override fun supportsResetPositionWithoutKeyFrameReset(positionUs: Long): Boolean = + super.supportsResetPositionWithoutKeyFrameReset(positionUs - delayUs) + + override fun getDurationToProgressUs( + positionUs: Long, + elapsedRealtimeUs: Long, + ): Long = super.getDurationToProgressUs(positionUs - delayUs, elapsedRealtimeUs) + + /** + * A text renderer only reads the end of its stream once its position is within a second of the + * period end, so a positive delay above a second would hold back + * [androidx.media3.common.Player.STATE_ENDED] by the delay minus that second. Once the real + * position passes the period end there is nothing left to show, so the renderer counts as ended. + */ + override fun isEnded(): Boolean = + super.isEnded() || + ( + delayUs > 0 && + periodEndPositionUs != C.TIME_UNSET && + lastPositionUs >= periodEndPositionUs + ) + + override fun disable() { + mediaPeriodId = null + periodEndPositionUs = C.TIME_UNSET + super.disable() + } + + private fun updateStream( + offsetUs: Long, + mediaPeriodId: MediaSource.MediaPeriodId, + ) { + streamOffsetUs = offsetUs + this.mediaPeriodId = mediaPeriodId + updatePeriodEndPosition() + } + + private fun updatePeriodEndPosition() { + val periodIndex = mediaPeriodId?.let { timeline.getIndexOfPeriod(it.periodUid) } ?: C.INDEX_UNSET + val durationUs = + if (periodIndex == C.INDEX_UNSET) { + C.TIME_UNSET + } else { + timeline.getPeriod(periodIndex, period).durationUs + } + periodEndPositionUs = if (durationUs == C.TIME_UNSET) C.TIME_UNSET else streamOffsetUs + durationUs + } + + companion object { + /** Sets the subtitle delay; the payload is the delay in microseconds as a [Long]. */ + const val MSG_SET_SUBTITLE_DELAY = Renderer.MSG_CUSTOM_BASE + } +} + +/** + * Wraps the subtitle renderers of [delegate] in a [SubtitleDelayRenderer]: media3's text renderer + * and the libass [AssRenderer], which draws at the position it is rendered with. + */ +class SubtitleDelayRenderersFactory( + private val delegate: RenderersFactory, +) : RenderersFactory { + override fun createRenderers( + eventHandler: Handler, + videoRendererEventListener: VideoRendererEventListener, + audioRendererEventListener: AudioRendererEventListener, + textRendererOutput: TextOutput, + metadataRendererOutput: MetadataOutput, + ): Array { + val renderers = + delegate.createRenderers( + eventHandler, + videoRendererEventListener, + audioRendererEventListener, + textRendererOutput, + metadataRendererOutput, + ) + return Array(renderers.size) { index -> + val renderer = renderers[index] + if (renderer.trackType == C.TRACK_TYPE_TEXT || renderer is AssRenderer) { + SubtitleDelayRenderer(renderer) + } else { + renderer + } + } + } + + override fun createSecondaryRenderer( + renderer: Renderer, + eventHandler: Handler, + videoRendererEventListener: VideoRendererEventListener, + audioRendererEventListener: AudioRendererEventListener, + textRendererOutput: TextOutput, + metadataRendererOutput: MetadataOutput, + ): Renderer? = + delegate.createSecondaryRenderer( + (renderer as? SubtitleDelayRenderer)?.delegate ?: renderer, + eventHandler, + videoRendererEventListener, + audioRendererEventListener, + textRendererOutput, + metadataRendererOutput, + ) +} diff --git a/app/src/test/java/com/github/damontecres/wholphin/services/SubtitleDelayRendererTest.kt b/app/src/test/java/com/github/damontecres/wholphin/services/SubtitleDelayRendererTest.kt new file mode 100644 index 00000000..af8464e8 --- /dev/null +++ b/app/src/test/java/com/github/damontecres/wholphin/services/SubtitleDelayRendererTest.kt @@ -0,0 +1,165 @@ +@file:OptIn(markerClass = [UnstableApi::class]) + +package com.github.damontecres.wholphin.services + +import android.os.Handler +import android.os.Looper +import androidx.annotation.OptIn +import androidx.media3.common.C +import androidx.media3.common.MediaItem +import androidx.media3.common.util.UnstableApi +import androidx.media3.exoplayer.Renderer +import androidx.media3.exoplayer.RendererConfiguration +import androidx.media3.exoplayer.RenderersFactory +import androidx.media3.exoplayer.source.MediaSource +import androidx.media3.exoplayer.source.SinglePeriodTimeline +import io.github.peerless2012.ass.media.AssHandler +import io.github.peerless2012.ass.media.factory.AssRenderersFactory +import io.github.peerless2012.ass.media.render.AssRenderer +import io.github.peerless2012.ass.media.type.AssRenderType +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertSame +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config + +@RunWith(RobolectricTestRunner::class) +@Config(manifest = Config.NONE, sdk = [30]) +class SubtitleDelayRendererTest { + private val delegate = mockk(relaxed = true) + private val renderer = SubtitleDelayRenderer(delegate) + private val timeline = SinglePeriodTimeline(DURATION_US, true, false, false, null, MediaItem.EMPTY) + private val mediaPeriodId = MediaSource.MediaPeriodId(timeline.getUidOfPeriod(0)) + + @Test + fun shiftsEveryPositionByTheDelay() { + for (delayUs in listOf(2_000_000L, -500_000L)) { + val delegate = mockk(relaxed = true) + val renderer = SubtitleDelayRenderer(delegate) + renderer.handleMessage(SubtitleDelayRenderer.MSG_SET_SUBTITLE_DELAY, delayUs) + val positionUs = OFFSET_US + 5_000_000L + val shiftedUs = positionUs - delayUs + + enable(renderer, positionUs) + renderer.render(positionUs, 7L) + renderer.resetPosition(positionUs, true) + renderer.getDurationToProgressUs(positionUs, 7L) + renderer.supportsResetPositionWithoutKeyFrameReset(positionUs) + + verify { + delegate.enable(any(), any(), any(), shiftedUs, false, true, OFFSET_US, OFFSET_US, mediaPeriodId) + delegate.render(shiftedUs, 7L) + delegate.resetPosition(shiftedUs, true) + delegate.getDurationToProgressUs(shiftedUs, 7L) + delegate.supportsResetPositionWithoutKeyFrameReset(shiftedUs) + } + } + } + + @Test + fun delayMessageChangesTheShiftAndOtherMessagesPassThrough() { + renderer.render(10_000_000L, 0L) + renderer.handleMessage(SubtitleDelayRenderer.MSG_SET_SUBTITLE_DELAY, 250_000L) + renderer.render(10_000_000L, 0L) + renderer.handleMessage(Renderer.MSG_SET_VOLUME, 0.5f) + + verify { + delegate.render(10_000_000L, 0L) + delegate.render(9_750_000L, 0L) + delegate.handleMessage(Renderer.MSG_SET_VOLUME, 0.5f) + } + verify(exactly = 0) { delegate.handleMessage(SubtitleDelayRenderer.MSG_SET_SUBTITLE_DELAY, any()) } + } + + @Test + fun endsAtThePeriodEndWithAPositiveDelay() { + every { delegate.isEnded } returns false + enable(renderer, OFFSET_US) + renderer.setTimeline(timeline) + renderer.handleMessage(SubtitleDelayRenderer.MSG_SET_SUBTITLE_DELAY, 2_000_000L) + + renderer.render(OFFSET_US + DURATION_US - 1, 0L) + assertFalse(renderer.isEnded) + + renderer.render(OFFSET_US + DURATION_US, 0L) + assertTrue(renderer.isEnded) + } + + @Test + fun otherwiseMirrorsTheDelegateAtThePeriodEnd() { + enable(renderer, OFFSET_US) + renderer.setTimeline(timeline) + for (delayUs in listOf(0L, -2_000_000L)) { + renderer.handleMessage(SubtitleDelayRenderer.MSG_SET_SUBTITLE_DELAY, delayUs) + renderer.render(OFFSET_US + DURATION_US, 0L) + for (delegateEnded in listOf(false, true)) { + every { delegate.isEnded } returns delegateEnded + assertEquals(delegateEnded, renderer.isEnded) + } + } + } + + @Test + fun doesNotEndWithoutAKnownPeriodDuration() { + every { delegate.isEnded } returns false + val unknownDuration = SinglePeriodTimeline(C.TIME_UNSET, false, false, false, null, MediaItem.EMPTY) + enable(renderer, OFFSET_US, MediaSource.MediaPeriodId(unknownDuration.getUidOfPeriod(0))) + renderer.setTimeline(unknownDuration) + renderer.handleMessage(SubtitleDelayRenderer.MSG_SET_SUBTITLE_DELAY, 2_000_000L) + + renderer.render(OFFSET_US + DURATION_US * 10, 0L) + + assertFalse(renderer.isEnded) + } + + @Test + fun factoryWrapsOnlySubtitleRenderers() { + val video = fakeRenderer(C.TRACK_TYPE_VIDEO) + val audio = fakeRenderer(C.TRACK_TYPE_AUDIO) + val text = fakeRenderer(C.TRACK_TYPE_TEXT) + val base = + mockk { + every { createRenderers(any(), any(), any(), any(), any()) } returns arrayOf(video, audio, text) + } + val factory = + SubtitleDelayRenderersFactory(AssRenderersFactory(AssHandler(AssRenderType.OVERLAY_OPEN_GL), base)) + + val renderers = + factory.createRenderers(Handler(Looper.getMainLooper()), mockk(), mockk(), mockk(), mockk()) + + assertSame(video, renderers[0]) + assertSame(audio, renderers[1]) + assertSame(text, (renderers[2] as SubtitleDelayRenderer).delegate) + assertTrue((renderers.last() as SubtitleDelayRenderer).delegate is AssRenderer) + assertEquals(4, renderers.size) + } + + private fun enable( + renderer: Renderer, + positionUs: Long, + periodId: MediaSource.MediaPeriodId = mediaPeriodId, + ) = renderer.enable( + RendererConfiguration.DEFAULT, + arrayOf(), + mockk(), + positionUs, + false, + true, + OFFSET_US, + OFFSET_US, + periodId, + ) + + private fun fakeRenderer(trackType: Int) = mockk(relaxed = true) { every { this@mockk.trackType } returns trackType } + + private companion object { + const val OFFSET_US = 1_000_000_000_000L + const val DURATION_US = 10_000_000L + } +}