Apply the subtitle delay on ExoPlayer #74
2 changed files with 381 additions and 0 deletions
|
|
@ -0,0 +1,216 @@
|
|||
@file:OptIn(markerClass = [UnstableApi::class])
|
||||
|
||||
package com.github.damontecres.wholphin.services
|
||||
|
||||
import android.os.Handler
|
||||
import androidx.annotation.OptIn
|
||||
import androidx.media3.common.C
|
||||
import androidx.media3.common.Format
|
||||
import androidx.media3.common.Timeline
|
||||
import androidx.media3.common.util.UnstableApi
|
||||
import androidx.media3.exoplayer.ForwardingRenderer
|
||||
import androidx.media3.exoplayer.Renderer
|
||||
import androidx.media3.exoplayer.RendererConfiguration
|
||||
import androidx.media3.exoplayer.RenderersFactory
|
||||
import androidx.media3.exoplayer.audio.AudioRendererEventListener
|
||||
import androidx.media3.exoplayer.metadata.MetadataOutput
|
||||
import androidx.media3.exoplayer.source.MediaSource
|
||||
import androidx.media3.exoplayer.source.SampleStream
|
||||
import androidx.media3.exoplayer.text.TextOutput
|
||||
import androidx.media3.exoplayer.video.VideoRendererEventListener
|
||||
import io.github.peerless2012.ass.media.render.AssRenderer
|
||||
|
||||
/**
|
||||
* Delays the subtitles of the wrapped renderer by shifting the playback position it is given.
|
||||
*
|
||||
* A positive delay shows subtitles later, a negative delay earlier, the same as mpv's `sub-delay`.
|
||||
* The delay is set on the playback thread with a [MSG_SET_SUBTITLE_DELAY] message whose payload is
|
||||
* the delay in microseconds as a [Long].
|
||||
*
|
||||
* Wrapping hides the renderer's type from ExoPlayer's `instanceof TextRenderer` checks, which only
|
||||
* matter for clipped streams and server-side inserted ads.
|
||||
*/
|
||||
class SubtitleDelayRenderer(
|
||||
internal val delegate: Renderer,
|
||||
) : ForwardingRenderer(delegate) {
|
||||
private val period = Timeline.Period()
|
||||
private var delayUs = 0L
|
||||
private var timeline: Timeline = Timeline.EMPTY
|
||||
private var mediaPeriodId: MediaSource.MediaPeriodId? = null
|
||||
private var streamOffsetUs = 0L
|
||||
private var periodEndPositionUs = C.TIME_UNSET
|
||||
private var lastPositionUs = C.TIME_UNSET
|
||||
|
||||
override fun handleMessage(
|
||||
messageType: Int,
|
||||
message: Any?,
|
||||
) {
|
||||
if (messageType == MSG_SET_SUBTITLE_DELAY) {
|
||||
delayUs = message as Long
|
||||
} else {
|
||||
super.handleMessage(messageType, message)
|
||||
}
|
||||
}
|
||||
|
||||
override fun enable(
|
||||
configuration: RendererConfiguration,
|
||||
formats: Array<out Format>,
|
||||
stream: SampleStream,
|
||||
positionUs: Long,
|
||||
joining: Boolean,
|
||||
mayRenderStartOfStream: Boolean,
|
||||
startPositionUs: Long,
|
||||
offsetUs: Long,
|
||||
mediaPeriodId: MediaSource.MediaPeriodId,
|
||||
) {
|
||||
lastPositionUs = positionUs
|
||||
updateStream(offsetUs, mediaPeriodId)
|
||||
super.enable(
|
||||
configuration,
|
||||
formats,
|
||||
stream,
|
||||
positionUs - delayUs,
|
||||
joining,
|
||||
mayRenderStartOfStream,
|
||||
startPositionUs,
|
||||
offsetUs,
|
||||
mediaPeriodId,
|
||||
)
|
||||
}
|
||||
|
||||
override fun replaceStream(
|
||||
formats: Array<out Format>,
|
||||
stream: SampleStream,
|
||||
startPositionUs: Long,
|
||||
offsetUs: Long,
|
||||
mediaPeriodId: MediaSource.MediaPeriodId,
|
||||
) {
|
||||
updateStream(offsetUs, mediaPeriodId)
|
||||
super.replaceStream(formats, stream, startPositionUs, offsetUs, mediaPeriodId)
|
||||
}
|
||||
|
||||
override fun setTimeline(timeline: Timeline) {
|
||||
this.timeline = timeline
|
||||
updatePeriodEndPosition()
|
||||
super.setTimeline(timeline)
|
||||
}
|
||||
|
||||
override fun render(
|
||||
positionUs: Long,
|
||||
elapsedRealtimeUs: Long,
|
||||
) {
|
||||
lastPositionUs = positionUs
|
||||
super.render(positionUs - delayUs, elapsedRealtimeUs)
|
||||
}
|
||||
|
||||
override fun resetPosition(
|
||||
positionUs: Long,
|
||||
sampleStreamIsResetToKeyFrame: Boolean,
|
||||
) {
|
||||
lastPositionUs = positionUs
|
||||
super.resetPosition(positionUs - delayUs, sampleStreamIsResetToKeyFrame)
|
||||
}
|
||||
|
||||
override fun supportsResetPositionWithoutKeyFrameReset(positionUs: Long): Boolean =
|
||||
super.supportsResetPositionWithoutKeyFrameReset(positionUs - delayUs)
|
||||
|
||||
override fun getDurationToProgressUs(
|
||||
positionUs: Long,
|
||||
elapsedRealtimeUs: Long,
|
||||
): Long = super.getDurationToProgressUs(positionUs - delayUs, elapsedRealtimeUs)
|
||||
|
||||
/**
|
||||
* A text renderer only reads the end of its stream once its position is within a second of the
|
||||
* period end, so a positive delay above a second would hold back
|
||||
* [androidx.media3.common.Player.STATE_ENDED] by the delay minus that second. Once the real
|
||||
* position passes the period end there is nothing left to show, so the renderer counts as ended.
|
||||
*/
|
||||
override fun isEnded(): Boolean =
|
||||
super.isEnded() ||
|
||||
(
|
||||
delayUs > 0 &&
|
||||
periodEndPositionUs != C.TIME_UNSET &&
|
||||
lastPositionUs >= periodEndPositionUs
|
||||
)
|
||||
|
||||
override fun disable() {
|
||||
mediaPeriodId = null
|
||||
periodEndPositionUs = C.TIME_UNSET
|
||||
super.disable()
|
||||
}
|
||||
|
||||
private fun updateStream(
|
||||
offsetUs: Long,
|
||||
mediaPeriodId: MediaSource.MediaPeriodId,
|
||||
) {
|
||||
streamOffsetUs = offsetUs
|
||||
this.mediaPeriodId = mediaPeriodId
|
||||
updatePeriodEndPosition()
|
||||
}
|
||||
|
||||
private fun updatePeriodEndPosition() {
|
||||
val periodIndex = mediaPeriodId?.let { timeline.getIndexOfPeriod(it.periodUid) } ?: C.INDEX_UNSET
|
||||
val durationUs =
|
||||
if (periodIndex == C.INDEX_UNSET) {
|
||||
C.TIME_UNSET
|
||||
} else {
|
||||
timeline.getPeriod(periodIndex, period).durationUs
|
||||
}
|
||||
periodEndPositionUs = if (durationUs == C.TIME_UNSET) C.TIME_UNSET else streamOffsetUs + durationUs
|
||||
}
|
||||
|
||||
companion object {
|
||||
/** Sets the subtitle delay; the payload is the delay in microseconds as a [Long]. */
|
||||
const val MSG_SET_SUBTITLE_DELAY = Renderer.MSG_CUSTOM_BASE
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wraps the subtitle renderers of [delegate] in a [SubtitleDelayRenderer]: media3's text renderer
|
||||
* and the libass [AssRenderer], which draws at the position it is rendered with.
|
||||
*/
|
||||
class SubtitleDelayRenderersFactory(
|
||||
private val delegate: RenderersFactory,
|
||||
) : RenderersFactory {
|
||||
override fun createRenderers(
|
||||
eventHandler: Handler,
|
||||
videoRendererEventListener: VideoRendererEventListener,
|
||||
audioRendererEventListener: AudioRendererEventListener,
|
||||
textRendererOutput: TextOutput,
|
||||
metadataRendererOutput: MetadataOutput,
|
||||
): Array<Renderer> {
|
||||
val renderers =
|
||||
delegate.createRenderers(
|
||||
eventHandler,
|
||||
videoRendererEventListener,
|
||||
audioRendererEventListener,
|
||||
textRendererOutput,
|
||||
metadataRendererOutput,
|
||||
)
|
||||
return Array(renderers.size) { index ->
|
||||
val renderer = renderers[index]
|
||||
if (renderer.trackType == C.TRACK_TYPE_TEXT || renderer is AssRenderer) {
|
||||
SubtitleDelayRenderer(renderer)
|
||||
} else {
|
||||
renderer
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun createSecondaryRenderer(
|
||||
renderer: Renderer,
|
||||
eventHandler: Handler,
|
||||
videoRendererEventListener: VideoRendererEventListener,
|
||||
audioRendererEventListener: AudioRendererEventListener,
|
||||
textRendererOutput: TextOutput,
|
||||
metadataRendererOutput: MetadataOutput,
|
||||
): Renderer? =
|
||||
delegate.createSecondaryRenderer(
|
||||
(renderer as? SubtitleDelayRenderer)?.delegate ?: renderer,
|
||||
eventHandler,
|
||||
videoRendererEventListener,
|
||||
audioRendererEventListener,
|
||||
textRendererOutput,
|
||||
metadataRendererOutput,
|
||||
)
|
||||
}
|
||||
|
|
@ -0,0 +1,165 @@
|
|||
@file:OptIn(markerClass = [UnstableApi::class])
|
||||
|
||||
package com.github.damontecres.wholphin.services
|
||||
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
import androidx.annotation.OptIn
|
||||
import androidx.media3.common.C
|
||||
import androidx.media3.common.MediaItem
|
||||
import androidx.media3.common.util.UnstableApi
|
||||
import androidx.media3.exoplayer.Renderer
|
||||
import androidx.media3.exoplayer.RendererConfiguration
|
||||
import androidx.media3.exoplayer.RenderersFactory
|
||||
import androidx.media3.exoplayer.source.MediaSource
|
||||
import androidx.media3.exoplayer.source.SinglePeriodTimeline
|
||||
import io.github.peerless2012.ass.media.AssHandler
|
||||
import io.github.peerless2012.ass.media.factory.AssRenderersFactory
|
||||
import io.github.peerless2012.ass.media.render.AssRenderer
|
||||
import io.github.peerless2012.ass.media.type.AssRenderType
|
||||
import io.mockk.every
|
||||
import io.mockk.mockk
|
||||
import io.mockk.verify
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertFalse
|
||||
import org.junit.Assert.assertSame
|
||||
import org.junit.Assert.assertTrue
|
||||
import org.junit.Test
|
||||
import org.junit.runner.RunWith
|
||||
import org.robolectric.RobolectricTestRunner
|
||||
import org.robolectric.annotation.Config
|
||||
|
||||
@RunWith(RobolectricTestRunner::class)
|
||||
@Config(manifest = Config.NONE, sdk = [30])
|
||||
class SubtitleDelayRendererTest {
|
||||
private val delegate = mockk<Renderer>(relaxed = true)
|
||||
private val renderer = SubtitleDelayRenderer(delegate)
|
||||
private val timeline = SinglePeriodTimeline(DURATION_US, true, false, false, null, MediaItem.EMPTY)
|
||||
private val mediaPeriodId = MediaSource.MediaPeriodId(timeline.getUidOfPeriod(0))
|
||||
|
||||
@Test
|
||||
fun shiftsEveryPositionByTheDelay() {
|
||||
for (delayUs in listOf(2_000_000L, -500_000L)) {
|
||||
val delegate = mockk<Renderer>(relaxed = true)
|
||||
val renderer = SubtitleDelayRenderer(delegate)
|
||||
renderer.handleMessage(SubtitleDelayRenderer.MSG_SET_SUBTITLE_DELAY, delayUs)
|
||||
val positionUs = OFFSET_US + 5_000_000L
|
||||
val shiftedUs = positionUs - delayUs
|
||||
|
||||
enable(renderer, positionUs)
|
||||
renderer.render(positionUs, 7L)
|
||||
renderer.resetPosition(positionUs, true)
|
||||
renderer.getDurationToProgressUs(positionUs, 7L)
|
||||
renderer.supportsResetPositionWithoutKeyFrameReset(positionUs)
|
||||
|
||||
verify {
|
||||
delegate.enable(any(), any(), any(), shiftedUs, false, true, OFFSET_US, OFFSET_US, mediaPeriodId)
|
||||
delegate.render(shiftedUs, 7L)
|
||||
delegate.resetPosition(shiftedUs, true)
|
||||
delegate.getDurationToProgressUs(shiftedUs, 7L)
|
||||
delegate.supportsResetPositionWithoutKeyFrameReset(shiftedUs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun delayMessageChangesTheShiftAndOtherMessagesPassThrough() {
|
||||
renderer.render(10_000_000L, 0L)
|
||||
renderer.handleMessage(SubtitleDelayRenderer.MSG_SET_SUBTITLE_DELAY, 250_000L)
|
||||
renderer.render(10_000_000L, 0L)
|
||||
renderer.handleMessage(Renderer.MSG_SET_VOLUME, 0.5f)
|
||||
|
||||
verify {
|
||||
delegate.render(10_000_000L, 0L)
|
||||
delegate.render(9_750_000L, 0L)
|
||||
delegate.handleMessage(Renderer.MSG_SET_VOLUME, 0.5f)
|
||||
}
|
||||
verify(exactly = 0) { delegate.handleMessage(SubtitleDelayRenderer.MSG_SET_SUBTITLE_DELAY, any()) }
|
||||
}
|
||||
|
||||
@Test
|
||||
fun endsAtThePeriodEndWithAPositiveDelay() {
|
||||
every { delegate.isEnded } returns false
|
||||
enable(renderer, OFFSET_US)
|
||||
renderer.setTimeline(timeline)
|
||||
renderer.handleMessage(SubtitleDelayRenderer.MSG_SET_SUBTITLE_DELAY, 2_000_000L)
|
||||
|
||||
renderer.render(OFFSET_US + DURATION_US - 1, 0L)
|
||||
assertFalse(renderer.isEnded)
|
||||
|
||||
renderer.render(OFFSET_US + DURATION_US, 0L)
|
||||
assertTrue(renderer.isEnded)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun otherwiseMirrorsTheDelegateAtThePeriodEnd() {
|
||||
enable(renderer, OFFSET_US)
|
||||
renderer.setTimeline(timeline)
|
||||
for (delayUs in listOf(0L, -2_000_000L)) {
|
||||
renderer.handleMessage(SubtitleDelayRenderer.MSG_SET_SUBTITLE_DELAY, delayUs)
|
||||
renderer.render(OFFSET_US + DURATION_US, 0L)
|
||||
for (delegateEnded in listOf(false, true)) {
|
||||
every { delegate.isEnded } returns delegateEnded
|
||||
assertEquals(delegateEnded, renderer.isEnded)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun doesNotEndWithoutAKnownPeriodDuration() {
|
||||
every { delegate.isEnded } returns false
|
||||
val unknownDuration = SinglePeriodTimeline(C.TIME_UNSET, false, false, false, null, MediaItem.EMPTY)
|
||||
enable(renderer, OFFSET_US, MediaSource.MediaPeriodId(unknownDuration.getUidOfPeriod(0)))
|
||||
renderer.setTimeline(unknownDuration)
|
||||
renderer.handleMessage(SubtitleDelayRenderer.MSG_SET_SUBTITLE_DELAY, 2_000_000L)
|
||||
|
||||
renderer.render(OFFSET_US + DURATION_US * 10, 0L)
|
||||
|
||||
assertFalse(renderer.isEnded)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun factoryWrapsOnlySubtitleRenderers() {
|
||||
val video = fakeRenderer(C.TRACK_TYPE_VIDEO)
|
||||
val audio = fakeRenderer(C.TRACK_TYPE_AUDIO)
|
||||
val text = fakeRenderer(C.TRACK_TYPE_TEXT)
|
||||
val base =
|
||||
mockk<RenderersFactory> {
|
||||
every { createRenderers(any(), any(), any(), any(), any()) } returns arrayOf(video, audio, text)
|
||||
}
|
||||
val factory =
|
||||
SubtitleDelayRenderersFactory(AssRenderersFactory(AssHandler(AssRenderType.OVERLAY_OPEN_GL), base))
|
||||
|
||||
val renderers =
|
||||
factory.createRenderers(Handler(Looper.getMainLooper()), mockk(), mockk(), mockk(), mockk())
|
||||
|
||||
assertSame(video, renderers[0])
|
||||
assertSame(audio, renderers[1])
|
||||
assertSame(text, (renderers[2] as SubtitleDelayRenderer).delegate)
|
||||
assertTrue((renderers.last() as SubtitleDelayRenderer).delegate is AssRenderer)
|
||||
assertEquals(4, renderers.size)
|
||||
}
|
||||
|
||||
private fun enable(
|
||||
renderer: Renderer,
|
||||
positionUs: Long,
|
||||
periodId: MediaSource.MediaPeriodId = mediaPeriodId,
|
||||
) = renderer.enable(
|
||||
RendererConfiguration.DEFAULT,
|
||||
arrayOf(),
|
||||
mockk(),
|
||||
positionUs,
|
||||
false,
|
||||
true,
|
||||
OFFSET_US,
|
||||
OFFSET_US,
|
||||
periodId,
|
||||
)
|
||||
|
||||
private fun fakeRenderer(trackType: Int) = mockk<Renderer>(relaxed = true) { every { this@mockk.trackType } returns trackType }
|
||||
|
||||
private companion object {
|
||||
const val OFFSET_US = 1_000_000_000_000L
|
||||
const val DURATION_US = 10_000_000L
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue