diff --git a/app/src/main/java/com/github/damontecres/wholphin/preferences/AppPreferencesSerializer.kt b/app/src/main/java/com/github/damontecres/wholphin/preferences/AppPreferencesSerializer.kt index c6dcd8ec..cb47ca94 100644 --- a/app/src/main/java/com/github/damontecres/wholphin/preferences/AppPreferencesSerializer.kt +++ b/app/src/main/java/com/github/damontecres/wholphin/preferences/AppPreferencesSerializer.kt @@ -184,6 +184,8 @@ class AppPreferencesSerializer videoTunnelingEnabled = false preferAc3Surround = ExperimentalPreference.PreferAc3ForSurround.defaultValue disableAudioOffload = ExperimentalPreference.DisableAudioOffload.defaultValue + preferDolbyVisionOverHdr10Plus = + ExperimentalPreference.PreferDolbyVisionOverHdr10Plus.defaultValue }.build() }.build() diff --git a/app/src/main/java/com/github/damontecres/wholphin/preferences/ExperimentalPreference.kt b/app/src/main/java/com/github/damontecres/wholphin/preferences/ExperimentalPreference.kt index f18df846..0647a900 100644 --- a/app/src/main/java/com/github/damontecres/wholphin/preferences/ExperimentalPreference.kt +++ b/app/src/main/java/com/github/damontecres/wholphin/preferences/ExperimentalPreference.kt @@ -37,6 +37,17 @@ object ExperimentalPreference { summaryOff = R.string.disabled, ) + val PreferDolbyVisionOverHdr10Plus = + AppSwitchPreference( + title = R.string.prefer_dovi_over_hdr10_plus, + defaultValue = false, + getter = { it.experimentalPreferences.preferDolbyVisionOverHdr10Plus }, + setter = { prefs, value -> + prefs.updateExperimentalPreferences { preferDolbyVisionOverHdr10Plus = value } + }, + summary = R.string.prefer_dovi_over_hdr10_plus_summary, + ) + val PreferAc3ForSurround = AppSwitchPreference( title = R.string.prefer_ac3_for_surround, @@ -81,6 +92,7 @@ val experimentalPreferences = preferences = listOf( ExperimentalPreference.VideoTunneling, + ExperimentalPreference.PreferDolbyVisionOverHdr10Plus, ExperimentalPreference.PreferAc3ForSurround, ExperimentalPreference.DisableAudioOffload, ), diff --git a/app/src/main/java/com/github/damontecres/wholphin/services/PlayerFactory.kt b/app/src/main/java/com/github/damontecres/wholphin/services/PlayerFactory.kt index cf4128fb..9b16718d 100644 --- a/app/src/main/java/com/github/damontecres/wholphin/services/PlayerFactory.kt +++ b/app/src/main/java/com/github/damontecres/wholphin/services/PlayerFactory.kt @@ -32,6 +32,8 @@ import com.github.damontecres.wholphin.preferences.MediaExtensionStatus import com.github.damontecres.wholphin.preferences.PlayerBackend import com.github.damontecres.wholphin.preferences.get import com.github.damontecres.wholphin.services.hilt.AuthOkHttpClient +import com.github.damontecres.wholphin.util.BitstreamFilteringCodecAdapterFactory +import com.github.damontecres.wholphin.util.Hdr10PlusMaskingFilter import com.github.damontecres.wholphin.util.WholphinDispatchers import dagger.hilt.android.qualifiers.ApplicationContext import io.github.peerless2012.ass.media.AssHandler @@ -89,6 +91,9 @@ class PlayerFactory val useLibAss = prefs.overrides.assPlaybackMode == AssPlaybackMode.ASS_LIBASS val decodeAv1 = prefs.overrides.decodeAv1 + val preferDolbyVision = + appPreferences.experimentalPreferences + .get { preferDolbyVisionOverHdr10Plus } ?: false Timber.v( "extensions=%s, assPlaybackMode=%s", extensions, @@ -104,7 +109,7 @@ class PlayerFactory val dataSourceFactory = DefaultDataSource.Factory(context) val extractorsFactory = createExtractorsFactory() var renderersFactory: RenderersFactory = - WholphinRenderersFactory(context, decodeAv1) + WholphinRenderersFactory(context, decodeAv1, preferDolbyVision) .setEnableDecoderFallback(true) .setExtensionRendererMode(rendererMode) @@ -262,6 +267,7 @@ data class PlayerCreation( class WholphinRenderersFactory( context: Context, private val av1Enabled: Boolean, + private val preferDolbyVisionOverHdr10Plus: Boolean = false, ) : DefaultRenderersFactory(context) { @OptIn(ExperimentalApi::class) override fun buildVideoRenderers( @@ -274,10 +280,19 @@ class WholphinRenderersFactory( allowedVideoJoiningTimeMs: Long, out: ArrayList, ) { + val videoCodecAdapterFactory = + if (preferDolbyVisionOverHdr10Plus) { + BitstreamFilteringCodecAdapterFactory( + codecAdapterFactory, + listOf(Hdr10PlusMaskingFilter()), + ) + } else { + codecAdapterFactory + } var videoRendererBuilder = MediaCodecVideoRenderer .Builder(context) - .setCodecAdapterFactory(codecAdapterFactory) + .setCodecAdapterFactory(videoCodecAdapterFactory) .setMediaCodecSelector(mediaCodecSelector) .setAllowedJoiningTimeMs(allowedVideoJoiningTimeMs) .setEnableDecoderFallback(enableDecoderFallback) diff --git a/app/src/main/java/com/github/damontecres/wholphin/util/BitstreamFilteringCodecAdapterFactory.kt b/app/src/main/java/com/github/damontecres/wholphin/util/BitstreamFilteringCodecAdapterFactory.kt new file mode 100644 index 00000000..cb1486f2 --- /dev/null +++ b/app/src/main/java/com/github/damontecres/wholphin/util/BitstreamFilteringCodecAdapterFactory.kt @@ -0,0 +1,128 @@ +@file:OptIn(markerClass = [UnstableApi::class]) + +package com.github.damontecres.wholphin.util + +import android.media.MediaCodec +import androidx.annotation.OptIn +import androidx.media3.common.MimeTypes +import androidx.media3.common.util.UnstableApi +import androidx.media3.exoplayer.mediacodec.ForwardingMediaCodecAdapter +import androidx.media3.exoplayer.mediacodec.MediaCodecAdapter +import timber.log.Timber +import java.nio.ByteBuffer + +/** + * Rewrites the bitstream of an access unit after the extractor and before it reaches the decoder. + * + * Implementations edit the buffer in place. Overwriting bytes leaves the access unit the size it + * was, removing bytes does not: a filter which drops part of an access unit has to move the bytes + * it keeps so that they stay contiguous from `offset`, and report the shorter length. The buffer's + * position and limit are not to be modified. + * + * The buffer is the decoder's input buffer, so it is direct: `hasArray()` is false and `array()` + * throws. Read and write it with the absolute `get(int)` and `put(int, Byte)`, or through a + * [ByteBuffer.duplicate] for bulk moves, never through a backing array. The absolute bulk + * `put(int, ByteBuffer, int, int)` is API 34 and `minSdk` is 23, so moving bytes means absolute + * puts or a duplicate. + */ +fun interface BitstreamFilter { + /** + * Filters the access unit in `[offset, offset + size)`. + * + * @return the new size of the access unit, which is never larger than [size]. Filters which + * only overwrite bytes return [size] unchanged. + */ + fun filter( + data: ByteBuffer, + offset: Int, + size: Int, + ): Int +} + +/** + * A [MediaCodecAdapter.Factory] which runs [filters] over the samples of Dolby Vision streams + * before they are queued to the decoder. + * + * The wrapper is only applied when a Dolby Vision decoder was actually selected, so playback which + * falls back to a regular HEVC/AVC decoder sees the bitstream as it came out of the extractor. + */ +class BitstreamFilteringCodecAdapterFactory( + private val delegate: MediaCodecAdapter.Factory, + private val filters: List, +) : MediaCodecAdapter.Factory { + override fun createAdapter(configuration: MediaCodecAdapter.Configuration): MediaCodecAdapter { + val adapter = delegate.createAdapter(configuration) + return if (filters.isNotEmpty() && configuration.codecInfo.mimeType == MimeTypes.VIDEO_DOLBY_VISION) { + Timber.i( + "Filtering the bitstream of Dolby Vision playback on %s with %s", + configuration.codecInfo.name, + filters.joinToString { it.javaClass.simpleName }, + ) + BitstreamFilteringCodecAdapter(adapter, filters) + } else { + adapter + } + } +} + +/** + * Runs each filter over the access unit in `[offset, offset + size)`, handing the next filter the + * length the previous one reported, and returns the length that is left to queue. A filter which + * reports a length it cannot have produced is ignored rather than trusted, since queueing the + * wrong length would feed the decoder a truncated or over long access unit. + */ +internal fun applyBitstreamFilters( + filters: List, + data: ByteBuffer, + offset: Int, + size: Int, +): Int { + var filteredSize = size + for (filter in filters) { + if (filteredSize <= 0) { + break + } + val newSize = filter.filter(data, offset, filteredSize) + if (newSize in 0..filteredSize) { + filteredSize = newSize + } else { + Timber.w( + "%s returned %d bytes for a %d byte access unit, keeping the access unit as it is", + filter.javaClass.simpleName, + newSize, + filteredSize, + ) + } + } + return filteredSize +} + +/** + * Filters each input buffer before it is queued. Encrypted samples go through + * [MediaCodecAdapter.queueSecureInputBuffer] and are forwarded untouched. + */ +internal class BitstreamFilteringCodecAdapter( + delegate: MediaCodecAdapter, + private val filters: List, +) : ForwardingMediaCodecAdapter(delegate) { + override fun queueInputBuffer( + index: Int, + offset: Int, + size: Int, + presentationTimeUs: Long, + flags: Int, + ) { + var filteredSize = size + if (size > 0 && (flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) { + getInputBuffer(index)?.let { buffer -> + filteredSize = applyBitstreamFilters(filters, buffer, offset, size) + } + } + super.queueInputBuffer(index, offset, filteredSize, presentationTimeUs, flags) + } + + override fun release() { + Timber.d("Releasing bitstream filtering adapter, %s", filters.joinToString()) + super.release() + } +} diff --git a/app/src/main/java/com/github/damontecres/wholphin/util/Hdr10PlusSeiFilter.kt b/app/src/main/java/com/github/damontecres/wholphin/util/Hdr10PlusSeiFilter.kt new file mode 100644 index 00000000..7b5d73e0 --- /dev/null +++ b/app/src/main/java/com/github/damontecres/wholphin/util/Hdr10PlusSeiFilter.kt @@ -0,0 +1,266 @@ +@file:OptIn(markerClass = [UnstableApi::class]) + +package com.github.damontecres.wholphin.util + +import androidx.annotation.OptIn +import androidx.media3.common.util.UnstableApi +import androidx.media3.container.NalUnitUtil +import java.nio.ByteBuffer + +/** + * Masks HDR10+ (SMPTE ST 2094-40) SEI messages in HEVC access units so that decoders ignore them. + * + * Some decoder pipelines (notably on MediaTek based TVs) latch on to HDR10+ when a stream carries + * both Dolby Vision RPUs and HDR10+ dynamic metadata, even when a Dolby Vision decoder is selected. + * Overwriting the SEI payload type of the HDR10+ messages with an undefined value makes decoders + * skip them without changing the size of the access unit, so no NAL units need to be removed or + * rewritten. + */ +object Hdr10PlusSeiFilter { + /** + * Highest VCL NAL unit type in H.265 Table 7-1. Prefix SEI NAL units always precede the VCL + * NAL units of an access unit, so the scan can stop at the first one. + */ + private const val MAX_VCL_NAL_UNIT_TYPE = 31 + + /** `user_data_registered_itu_t_t35`, the SEI payload type used by HDR10+ (H.265 D.2.1). */ + private const val PAYLOAD_TYPE_USER_DATA_REGISTERED_ITU_T_T35 = 4 + + /** + * An SEI payload type H.265 does not define. H.265 D.3.1 requires a decoder to ignore SEI + * messages whose payload type it does not recognise, so writing this over the payload type + * makes the decoder skip the message without anything being removed or resized. + */ + private const val PAYLOAD_TYPE_MASKED = 254 + + /** + * ITU-T T.35 header identifying an HDR10+ message: `itu_t_t35_country_code` 0xB5 (United + * States), `itu_t_t35_terminal_provider_code` 0x003C (Samsung, which defined HDR10+) and + * `itu_t_t35_terminal_provider_oriented_code` 0x0001. FFmpeg declares the first two as + * `ITU_T_T35_COUNTRY_CODE_US` and `ITU_T_T35_PROVIDER_CODE_SAMSUNG` in `libavcodec/itut35.h`. + */ + private val HDR10_PLUS_T35_HEADER = intArrayOf(0xB5, 0x00, 0x3C, 0x00, 0x01) + + /** + * Masks every HDR10+ SEI message found in the HEVC access unit in `[offset, offset + size)` by + * overwriting its SEI payload type in place. The access unit must contain Annex B start code + * delimited NAL units (which is what media3's extractors emit); buffers with any other layout + * are left untouched. The buffer's position and limit are not modified. + * + * @return the number of masked SEI messages + */ + fun maskHdr10PlusSeiMessages( + data: ByteBuffer, + offset: Int, + size: Int, + ): Int { + val end = offset + size + if (!startsWithStartCode(data, offset, end)) { + return 0 + } + var masked = 0 + var searchFrom = offset + while (searchFrom < end) { + val nalStart = findNalUnitStart(data, searchFrom, end) + if (nalStart < 0 || nalStart >= end) { + break + } + val nalEnd = findNalUnitEnd(data, nalStart, end) + // nal_unit_type is the 6 bits after forbidden_zero_bit, H.265 section 7.3.1.2 + val nalUnitType = (data.get(nalStart).toInt() shr 1) and 0x3F + if (nalUnitType <= MAX_VCL_NAL_UNIT_TYPE) { + // HDR10+ SEI messages precede the VCL NAL units, so stop scanning here + break + } + if (nalUnitType == NalUnitUtil.H265_NAL_UNIT_TYPE_PREFIX_SEI) { + masked += maskInSeiNalUnit(data, nalStart + 2, nalEnd) + } + searchFrom = nalEnd + } + return masked + } + + private fun startsWithStartCode( + data: ByteBuffer, + offset: Int, + end: Int, + ): Boolean { + if (end - offset < 4) { + return false + } + return data.get(offset) == 0.toByte() && + data.get(offset + 1) == 0.toByte() && + ( + data.get(offset + 2) == 1.toByte() || + (data.get(offset + 2) == 0.toByte() && data.get(offset + 3) == 1.toByte()) + ) + } + + /** + * Returns the index of the first NAL unit byte after the next start code, or -1 if none. + * Start codes are the Annex B byte stream format. + */ + private fun findNalUnitStart( + data: ByteBuffer, + from: Int, + end: Int, + ): Int { + for (i in from..end - 4) { + if (data.get(i) == 0.toByte() && + data.get(i + 1) == 0.toByte() && + data.get(i + 2) == 1.toByte() + ) { + return i + 3 + } + } + return -1 + } + + /** Returns the index just past the NAL unit starting at [from], i.e. the next start code or [end]. */ + private fun findNalUnitEnd( + data: ByteBuffer, + from: Int, + end: Int, + ): Int { + for (i in from..end - 3) { + if (data.get(i) == 0.toByte() && + data.get(i + 1) == 0.toByte() && + data.get(i + 2) == 1.toByte() + ) { + return i + } + } + return end + } + + /** + * Parses the SEI messages in the NAL unit payload in `[payloadStart, nalEnd)` per H.265 + * section D.2.1 and masks the payload type of each HDR10+ message. Parsing is conservative: + * on any malformed data it stops without touching the remainder. + */ + private fun maskInSeiNalUnit( + data: ByteBuffer, + payloadStart: Int, + nalEnd: Int, + ): Int { + // Trim trailing_zero_8bits so the last byte is the RBSP trailing bits + var end = nalEnd + while (end > payloadStart && data.get(end - 1) == 0.toByte()) { + end-- + } + val reader = RbspReader(data, payloadStart, end) + var masked = 0 + while (reader.bytesRemaining > 1) { + var payloadType = 0 + var b = reader.readByte() + while (b == 0xFF) { + payloadType += 255 + b = reader.readByte() + } + if (b < 0) { + return masked + } + val payloadTypePosition = reader.lastBytePosition + payloadType += b + + var payloadSize = 0 + b = reader.readByte() + while (b == 0xFF) { + payloadSize += 255 + b = reader.readByte() + } + if (b < 0) { + return masked + } + payloadSize += b + if (payloadSize > reader.bytesRemaining) { + return masked + } + + var remainingPayload = payloadSize + var isHdr10Plus = + payloadType == PAYLOAD_TYPE_USER_DATA_REGISTERED_ITU_T_T35 && + payloadSize >= HDR10_PLUS_T35_HEADER.size + if (isHdr10Plus) { + for (expected in HDR10_PLUS_T35_HEADER) { + remainingPayload-- + if (reader.readByte() != expected) { + isHdr10Plus = false + break + } + } + } + repeat(remainingPayload) { + if (reader.readByte() < 0) { + return masked + } + } + if (isHdr10Plus) { + data.put(payloadTypePosition, PAYLOAD_TYPE_MASKED.toByte()) + masked++ + } + } + return masked + } + + /** + * Reads RBSP bytes from `[position, end)`, skipping the `emulation_prevention_three_byte` + * that H.265 section 7.4.2 inserts after two consecutive zero bytes. + */ + private class RbspReader( + private val data: ByteBuffer, + private var position: Int, + private val end: Int, + ) { + private var zeroRun = 0 + + /** Raw buffer index of the byte returned by the last [readByte] call. */ + var lastBytePosition = -1 + private set + + /** Upper bound of the remaining RBSP bytes. */ + val bytesRemaining: Int + get() = end - position + + /** Returns the next RBSP byte, or -1 if the end was reached. */ + fun readByte(): Int { + if (position >= end) { + return -1 + } + var b = data.get(position).toInt() and 0xFF + if (zeroRun >= 2 && b == 0x03) { + // Skip the emulation_prevention_three_byte + zeroRun = 0 + position++ + if (position >= end) { + return -1 + } + b = data.get(position).toInt() and 0xFF + } + zeroRun = if (b == 0) zeroRun + 1 else 0 + lastBytePosition = position + position++ + return b + } + } +} + +/** + * The [BitstreamFilter] form of [Hdr10PlusSeiFilter]. Masking overwrites one byte per message, so + * the access unit keeps the size it was handed. Instances count what they masked, which is what + * the adapter logs when it is released. + */ +class Hdr10PlusMaskingFilter : BitstreamFilter { + private var maskedCount = 0 + + override fun filter( + data: ByteBuffer, + offset: Int, + size: Int, + ): Int { + maskedCount += Hdr10PlusSeiFilter.maskHdr10PlusSeiMessages(data, offset, size) + return size + } + + override fun toString(): String = "HDR10+ SEI masking ($maskedCount messages masked)" +} diff --git a/app/src/main/proto/WholphinDataStore.proto b/app/src/main/proto/WholphinDataStore.proto index 98698f7c..3fba78f6 100644 --- a/app/src/main/proto/WholphinDataStore.proto +++ b/app/src/main/proto/WholphinDataStore.proto @@ -228,6 +228,7 @@ message ExperimentalPreferences { bool video_tunneling_enabled = 2; bool prefer_ac3_surround = 3; bool disable_audio_offload = 4; + bool prefer_dolby_vision_over_hdr10_plus = 5; } message AppPreferences { diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index cff98add..1b0915b5 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -487,6 +487,8 @@ DTS Direct play Dolby Vision Profile 7 Ignores device compatibility checks + Prefer Dolby Vision over HDR10+ + Removes HDR10+ metadata from Dolby Vision videos, for TVs that otherwise play them as HDR10+ Discover Request diff --git a/app/src/test/java/com/github/damontecres/wholphin/util/BitstreamFilterChainTest.kt b/app/src/test/java/com/github/damontecres/wholphin/util/BitstreamFilterChainTest.kt new file mode 100644 index 00000000..8175371e --- /dev/null +++ b/app/src/test/java/com/github/damontecres/wholphin/util/BitstreamFilterChainTest.kt @@ -0,0 +1,136 @@ +package com.github.damontecres.wholphin.util + +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import java.nio.ByteBuffer + +class BitstreamFilterChainTest { + @Test + fun keepsTheSizeWhenThereIsNoFilter() { + assertEquals(DATA.size, applyBitstreamFilters(emptyList(), buffer(), 0, DATA.size)) + } + + @Test + fun keepsTheSizeOfAnInPlaceFilter() { + val inPlace = + BitstreamFilter { data, offset, size -> + data.put(offset, 0xFF.toByte()) + size + } + val target = buffer() + + val size = applyBitstreamFilters(listOf(inPlace), target, 0, DATA.size) + + assertEquals(DATA.size, size) + assertArrayEquals(byteArrayOf(0xFF.toByte()) + DATA.copyOfRange(1, DATA.size), target.array()) + } + + @Test + fun reportsTheShorterSizeOfAFilterWhichDropsBytes() { + // What a Dolby Vision profile 7 to 8.1 conversion does: it removes the enhancement layer, + // compacts what it keeps and comes out shorter than it went in. The buffer is direct, + // like the decoder input buffer a filter is handed on a device: it has no backing array, + // so the bytes are moved with absolute gets and puts. + val dropsTheThirdAndFourthBytes = + BitstreamFilter { data, offset, size -> + for (i in 2 until size - 2) { + data.put(offset + i, data.get(offset + i + 2)) + } + size - 2 + } + val target = directBuffer() + + val size = applyBitstreamFilters(listOf(dropsTheThirdAndFourthBytes), target, 0, DATA.size) + + assertFalse("the test should exercise a buffer without a backing array", target.hasArray()) + assertEquals(DATA.size - 2, size) + assertEquals(listOf(1, 2, 5, 6, 7, 8), (0 until size).map { target.get(it) }) + } + + @Test + fun handsTheNextFilterTheSizeReportedByThePreviousOne() { + val seen = mutableListOf() + val dropTwo = + BitstreamFilter { _, _, size -> + seen.add(size) + size - 2 + } + val dropOne = + BitstreamFilter { _, _, size -> + seen.add(size) + size - 1 + } + + val size = applyBitstreamFilters(listOf(dropTwo, dropOne), buffer(), 0, DATA.size) + + assertEquals(listOf(DATA.size, DATA.size - 2), seen) + assertEquals(DATA.size - 3, size) + } + + @Test + fun ignoresAFilterWhichReportsMoreBytesThanItWasGiven() { + val seen = mutableListOf() + val tooLong = BitstreamFilter { _, _, size -> size + 1 } + val recording = + BitstreamFilter { _, _, size -> + seen.add(size) + size + } + + val size = applyBitstreamFilters(listOf(tooLong, recording), buffer(), 0, DATA.size) + + assertEquals(DATA.size, size) + assertEquals(listOf(DATA.size), seen) + } + + @Test + fun ignoresAFilterWhichReportsANegativeSize() { + val negative = BitstreamFilter { _, _, _ -> -1 } + + assertEquals(DATA.size, applyBitstreamFilters(listOf(negative), buffer(), 0, DATA.size)) + } + + @Test + fun stopsOnceNothingIsLeftOfTheAccessUnit() { + var called = false + val dropsEverything = BitstreamFilter { _, _, _ -> 0 } + val next = + BitstreamFilter { _, _, size -> + called = true + size + } + + val size = applyBitstreamFilters(listOf(dropsEverything, next), buffer(), 0, DATA.size) + + assertEquals(0, size) + assertTrue("a filter should not be handed an empty access unit", !called) + } + + @Test + fun filtersAreGivenTheOffsetTheyWereCalledWith() { + val offsets = mutableListOf() + val recording = + BitstreamFilter { _, offset, size -> + offsets.add(offset) + size + } + + applyBitstreamFilters(listOf(recording, recording), buffer(), 2, DATA.size - 2) + + assertEquals(listOf(2, 2), offsets) + } + + companion object { + private val DATA = byteArrayOf(1, 2, 3, 4, 5, 6, 7, 8) + + private fun buffer() = ByteBuffer.wrap(DATA.copyOf()) + + private fun directBuffer() = + ByteBuffer.allocateDirect(DATA.size).also { buffer -> + DATA.forEachIndexed { index, byte -> buffer.put(index, byte) } + } + } +} diff --git a/app/src/test/java/com/github/damontecres/wholphin/util/BitstreamFilteringCodecAdapterTest.kt b/app/src/test/java/com/github/damontecres/wholphin/util/BitstreamFilteringCodecAdapterTest.kt new file mode 100644 index 00000000..3c8fdd23 --- /dev/null +++ b/app/src/test/java/com/github/damontecres/wholphin/util/BitstreamFilteringCodecAdapterTest.kt @@ -0,0 +1,103 @@ +@file:OptIn(markerClass = [UnstableApi::class]) + +package com.github.damontecres.wholphin.util + +import android.media.MediaCodec +import androidx.annotation.OptIn +import androidx.media3.common.util.UnstableApi +import androidx.media3.exoplayer.mediacodec.MediaCodecAdapter +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import org.junit.Assert.assertEquals +import org.junit.Test +import java.nio.ByteBuffer + +/** + * Covers the hand off to the decoder: the adapter has to queue the size the filters report, not + * the size it was called with, and it has to leave the buffers no filter may touch alone. + */ +class BitstreamFilteringCodecAdapterTest { + private val delegate = mockk(relaxed = true) + private val buffer = ByteBuffer.allocateDirect(SIZE) + + @Test + fun queuesTheSizeTheFilterReports() { + every { delegate.getInputBuffer(INDEX) } returns buffer + val adapter = adapterWith(BitstreamFilter { _, _, size -> size - 2 }) + + adapter.queueInputBuffer(INDEX, OFFSET, SIZE, PRESENTATION_TIME_US, 0) + + verify { delegate.queueInputBuffer(INDEX, OFFSET, SIZE - 2, PRESENTATION_TIME_US, 0) } + } + + @Test + fun queuesTheSizeItWasGivenWhenTheFilterLeavesTheAccessUnitAlone() { + every { delegate.getInputBuffer(INDEX) } returns buffer + val adapter = adapterWith(BitstreamFilter { _, _, size -> size }) + + adapter.queueInputBuffer(INDEX, OFFSET, SIZE, PRESENTATION_TIME_US, 0) + + verify { delegate.queueInputBuffer(INDEX, OFFSET, SIZE, PRESENTATION_TIME_US, 0) } + } + + @Test + fun leavesCodecConfigBuffersAlone() { + every { delegate.getInputBuffer(INDEX) } returns buffer + val filter = RecordingFilter() + val adapter = adapterWith(filter) + + adapter.queueInputBuffer(INDEX, OFFSET, SIZE, PRESENTATION_TIME_US, MediaCodec.BUFFER_FLAG_CODEC_CONFIG) + + assertEquals(0, filter.calls) + verify { + delegate.queueInputBuffer(INDEX, OFFSET, SIZE, PRESENTATION_TIME_US, MediaCodec.BUFFER_FLAG_CODEC_CONFIG) + } + } + + @Test + fun leavesEmptyBuffersAlone() { + every { delegate.getInputBuffer(INDEX) } returns buffer + val filter = RecordingFilter() + val adapter = adapterWith(filter) + + adapter.queueInputBuffer(INDEX, OFFSET, 0, PRESENTATION_TIME_US, 0) + + assertEquals(0, filter.calls) + verify { delegate.queueInputBuffer(INDEX, OFFSET, 0, PRESENTATION_TIME_US, 0) } + } + + @Test + fun queuesTheSizeItWasGivenWhenThereIsNoInputBuffer() { + every { delegate.getInputBuffer(INDEX) } returns null + val filter = RecordingFilter() + val adapter = adapterWith(filter) + + adapter.queueInputBuffer(INDEX, OFFSET, SIZE, PRESENTATION_TIME_US, 0) + + assertEquals(0, filter.calls) + verify { delegate.queueInputBuffer(INDEX, OFFSET, SIZE, PRESENTATION_TIME_US, 0) } + } + + private fun adapterWith(filter: BitstreamFilter) = BitstreamFilteringCodecAdapter(delegate, listOf(filter)) + + private class RecordingFilter : BitstreamFilter { + var calls = 0 + + override fun filter( + data: ByteBuffer, + offset: Int, + size: Int, + ): Int { + calls++ + return size + } + } + + companion object { + private const val INDEX = 3 + private const val OFFSET = 1 + private const val SIZE = 8 + private const val PRESENTATION_TIME_US = 1234L + } +} diff --git a/app/src/test/java/com/github/damontecres/wholphin/util/Hdr10PlusSeiFilterTest.kt b/app/src/test/java/com/github/damontecres/wholphin/util/Hdr10PlusSeiFilterTest.kt new file mode 100644 index 00000000..f3f75193 --- /dev/null +++ b/app/src/test/java/com/github/damontecres/wholphin/util/Hdr10PlusSeiFilterTest.kt @@ -0,0 +1,277 @@ +package com.github.damontecres.wholphin.util + +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Test +import java.nio.ByteBuffer + +class Hdr10PlusSeiFilterTest { + @Test + fun masksSoleHdr10PlusMessage() { + val accessUnit = annexB(seiNal(seiMessage(4, HDR10_PLUS_PAYLOAD)), vclNal()) + // Payload type byte follows the start code and the 2 byte NAL unit header + val payloadTypeIndex = START_CODE.size + 2 + + val (masked, result) = filter(accessUnit) + + assertEquals(1, masked) + assertArrayEquals(maskTypeByte(accessUnit, payloadTypeIndex), result) + } + + @Test + fun masksWithThreeByteStartCodes() { + val accessUnit = + SHORT_START_CODE + seiNal(seiMessage(4, HDR10_PLUS_PAYLOAD)) + SHORT_START_CODE + vclNal() + val payloadTypeIndex = SHORT_START_CODE.size + 2 + + val (masked, result) = filter(accessUnit) + + assertEquals(1, masked) + assertArrayEquals(maskTypeByte(accessUnit, payloadTypeIndex), result) + } + + @Test + fun masksOnlyHdr10PlusInMultiMessageNal() { + val masteringDisplay = seiMessage(137, MASTERING_DISPLAY_PAYLOAD) + val contentLightLevel = seiMessage(144, CONTENT_LIGHT_LEVEL_PAYLOAD) + val accessUnit = + annexB( + seiNal(masteringDisplay, seiMessage(4, HDR10_PLUS_PAYLOAD), contentLightLevel), + vclNal(), + ) + val payloadTypeIndex = START_CODE.size + 2 + masteringDisplay.size + + val (masked, result) = filter(accessUnit) + + assertEquals(1, masked) + assertArrayEquals(maskTypeByte(accessUnit, payloadTypeIndex), result) + } + + @Test + fun skipsEmulationPreventionBytesInPrecedingPayload() { + // The escaped form of this payload is longer than its payload size, so naive byte + // counting would misparse the HDR10+ message that follows it + val zeroHeavyMessage = seiMessage(137, byteArrayOf(0, 0, 0, 0, 1)) + val accessUnit = + annexB(seiNal(zeroHeavyMessage, seiMessage(4, HDR10_PLUS_PAYLOAD)), vclNal()) + val payloadTypeIndex = START_CODE.size + 2 + escape(zeroHeavyMessage).size + + val (masked, result) = filter(accessUnit) + + assertEquals(1, masked) + assertArrayEquals(maskTypeByte(accessUnit, payloadTypeIndex), result) + } + + @Test + fun handlesMultiBytePayloadType() { + val bigTypeMessage = seiMessage(300, byteArrayOf(0x42)) + val accessUnit = + annexB(seiNal(bigTypeMessage, seiMessage(4, HDR10_PLUS_PAYLOAD)), vclNal()) + val payloadTypeIndex = START_CODE.size + 2 + bigTypeMessage.size + + val (masked, result) = filter(accessUnit) + + assertEquals(1, masked) + assertArrayEquals(maskTypeByte(accessUnit, payloadTypeIndex), result) + } + + @Test + fun ignoresOtherItuTT35Messages() { + val accessUnit = annexB(seiNal(seiMessage(4, CEA_708_PAYLOAD)), vclNal()) + + val (masked, result) = filter(accessUnit) + + assertEquals(0, masked) + assertArrayEquals(accessUnit, result) + } + + @Test + fun ignoresSeiNalUnitsAfterTheVclNalUnit() { + val accessUnit = annexB(vclNal(), seiNal(seiMessage(4, HDR10_PLUS_PAYLOAD))) + + val (masked, result) = filter(accessUnit) + + assertEquals(0, masked) + assertArrayEquals(accessUnit, result) + } + + @Test + fun masksAcrossMultipleNonVclNalUnits() { + val accessUnit = + annexB( + audNal(), + seiNal(seiMessage(137, MASTERING_DISPLAY_PAYLOAD)), + seiNal(seiMessage(4, HDR10_PLUS_PAYLOAD)), + vclNal(), + ) + val payloadTypeIndex = + START_CODE.size + audNal().size + + START_CODE.size + seiNal(seiMessage(137, MASTERING_DISPLAY_PAYLOAD)).size + + START_CODE.size + 2 + + val (masked, result) = filter(accessUnit) + + assertEquals(1, masked) + assertArrayEquals(maskTypeByte(accessUnit, payloadTypeIndex), result) + } + + @Test + fun leavesLengthPrefixedSamplesUntouched() { + val nal = seiNal(seiMessage(4, HDR10_PLUS_PAYLOAD)) + val accessUnit = + byteArrayOf( + (nal.size ushr 24).toByte(), + (nal.size ushr 16).toByte(), + (nal.size ushr 8).toByte(), + nal.size.toByte(), + ) + nal + + val (masked, result) = filter(accessUnit) + + assertEquals(0, masked) + assertArrayEquals(accessUnit, result) + } + + @Test + fun leavesTruncatedSeiNalUnitUntouched() { + // Payload size claims more bytes than the NAL unit contains + val brokenNal = byteArrayOf(0x4E, 0x01, 0x04, 0x30, 0xB5.toByte(), 0x00, 0x3C) + val accessUnit = annexB(brokenNal, vclNal()) + + val (masked, result) = filter(accessUnit) + + assertEquals(0, masked) + assertArrayEquals(accessUnit, result) + } + + @Test + fun leavesShortItuTT35MessageUntouched() { + val accessUnit = annexB(seiNal(seiMessage(4, byteArrayOf(0xB5.toByte(), 0x00, 0x3C))), vclNal()) + + val (masked, result) = filter(accessUnit) + + assertEquals(0, masked) + assertArrayEquals(accessUnit, result) + } + + @Test + fun respectsOffsetAndSize() { + val accessUnit = annexB(seiNal(seiMessage(4, HDR10_PLUS_PAYLOAD)), vclNal()) + val prefix = byteArrayOf(0x55, 0x66) + val data = prefix + accessUnit + val payloadTypeIndex = prefix.size + START_CODE.size + 2 + + val buffer = ByteBuffer.wrap(data.copyOf()) + val masked = Hdr10PlusSeiFilter.maskHdr10PlusSeiMessages(buffer, prefix.size, accessUnit.size) + + assertEquals(1, masked) + assertArrayEquals(maskTypeByte(data, payloadTypeIndex), buffer.array()) + } + + @Test + fun masksMultipleHdr10PlusMessages() { + val firstMessage = seiMessage(4, HDR10_PLUS_PAYLOAD) + val accessUnit = + annexB(seiNal(firstMessage, seiMessage(4, HDR10_PLUS_PAYLOAD)), vclNal()) + val expected = + maskTypeByte( + maskTypeByte(accessUnit, START_CODE.size + 2), + START_CODE.size + 2 + firstMessage.size, + ) + + val (masked, result) = filter(accessUnit) + + assertEquals(2, masked) + assertArrayEquals(expected, result) + } + + @Test + fun maskingFilterKeepsTheAccessUnitSize() { + val accessUnit = annexB(seiNal(seiMessage(4, HDR10_PLUS_PAYLOAD)), vclNal()) + val buffer = ByteBuffer.wrap(accessUnit.copyOf()) + + val size = Hdr10PlusMaskingFilter().filter(buffer, 0, accessUnit.size) + + assertEquals(accessUnit.size, size) + assertArrayEquals(maskTypeByte(accessUnit, START_CODE.size + 2), buffer.array()) + } + + companion object { + private val START_CODE = byteArrayOf(0, 0, 0, 1) + private val SHORT_START_CODE = byteArrayOf(0, 0, 1) + + /** ITU-T T.35 header for SMPTE ST 2094-40 plus a few opaque payload bytes */ + private val HDR10_PLUS_PAYLOAD = + byteArrayOf(0xB5.toByte(), 0x00, 0x3C, 0x00, 0x01, 0x04, 0x01, 0x12, 0x34) + + /** ITU-T T.35 with the ATSC provider code (0x0031), as used for CEA-708 captions */ + private val CEA_708_PAYLOAD = + byteArrayOf(0xB5.toByte(), 0x00, 0x31, 0x47, 0x41, 0x39, 0x34, 0x03) + + private val MASTERING_DISPLAY_PAYLOAD = ByteArray(24) { (it + 1).toByte() } + + private val CONTENT_LIGHT_LEVEL_PAYLOAD = byteArrayOf(0x03, 0xE8.toByte(), 0x01, 0xF4.toByte()) + + /** Encodes an SEI message with the ff-extended payload type and size coding of D.2.1 */ + private fun seiMessage( + payloadType: Int, + payload: ByteArray, + ): ByteArray { + val out = mutableListOf() + var type = payloadType + while (type >= 255) { + out.add(0xFF.toByte()) + type -= 255 + } + out.add(type.toByte()) + var size = payload.size + while (size >= 255) { + out.add(0xFF.toByte()) + size -= 255 + } + out.add(size.toByte()) + payload.forEach { out.add(it) } + return out.toByteArray() + } + + /** Inserts emulation prevention bytes as required by H.265 section 7.4.2 */ + private fun escape(rbsp: ByteArray): ByteArray { + val out = mutableListOf() + var zeroRun = 0 + for (b in rbsp) { + if (zeroRun >= 2 && (b.toInt() and 0xFF) <= 0x03) { + out.add(0x03) + zeroRun = 0 + } + out.add(b) + zeroRun = if (b == 0.toByte()) zeroRun + 1 else 0 + } + return out.toByteArray() + } + + /** Builds a prefix SEI NAL unit (type 39) containing the given messages */ + private fun seiNal(vararg messages: ByteArray): ByteArray { + val rbsp = messages.fold(byteArrayOf()) { acc, message -> acc + message } + 0x80.toByte() + return byteArrayOf(0x4E, 0x01) + escape(rbsp) + } + + /** A minimal VCL NAL unit (type 1, TRAIL_R) */ + private fun vclNal(): ByteArray = byteArrayOf(0x02, 0x01, 0x11, 0x22, 0x33) + + /** A minimal access unit delimiter NAL unit (type 35) */ + private fun audNal(): ByteArray = byteArrayOf(0x46, 0x01, 0x50) + + private fun annexB(vararg nals: ByteArray): ByteArray = nals.fold(byteArrayOf()) { acc, nal -> acc + START_CODE + nal } + + private fun maskTypeByte( + data: ByteArray, + index: Int, + ): ByteArray = data.copyOf().also { it[index] = 0xFE.toByte() } + + private fun filter(data: ByteArray): Pair { + val buffer = ByteBuffer.wrap(data.copyOf()) + val masked = Hdr10PlusSeiFilter.maskHdr10PlusSeiMessages(buffer, 0, data.size) + return masked to buffer.array() + } + } +}