From 57b1d277d98bba83e3f64483df7cd1a9edd4d61c Mon Sep 17 00:00:00 2001 From: Gilles Khouzam Date: Wed, 4 Dec 2024 11:44:19 -0800 Subject: [PATCH] Add new consecutiveDroppedFrames callback This is similar to the `droppedFrames` callback, but represents the number of consecutive frames that we dropped before rendering a frame or seeking or stopping the renderer. While we already have a `maxConsecutiveDroppedFrame` available in the `DecoderCounters`, this doesn't provide enough visibility into the actual statistics of dropped frames. If we get 200 dropped frames and a `maxConsecutive` of 20, we don't know if we dropped 20 frames in a row once and then dropped a single frame 180 times or if we dropped 20 frames 10 times. We could add some code on our `OnDroppedFrames` callback to estimate if two calls are for consecutive frames, but that seems very fragile. Specifying when to invoke the callback is controlled by `minConsecutiveDroppedFramesToNotify` similar to the `maxDroppedFramesToNotify` but that would only notify if more than X consecutive frames were dropped. Adding support for both `MediaCodecVideoRenderer` and `DecoderVideoRenderer`. --- .../decoder/av1/Libgav1VideoRenderer.java | 18 ++- .../ExperimentalFfmpegVideoRenderer.java | 13 +- .../decoder/vp9/LibvpxVideoRenderer.java | 20 ++- .../exoplayer/DefaultRenderersFactory.java | 21 ++- .../media3/exoplayer/ExoPlayerImpl.java | 5 + .../analytics/AnalyticsCollector.java | 12 ++ .../analytics/AnalyticsListener.java | 18 ++- .../analytics/DefaultAnalyticsCollector.java | 9 ++ .../media3/exoplayer/util/EventLogger.java | 7 + .../exoplayer/video/DecoderVideoRenderer.java | 25 +++- .../video/MediaCodecVideoRenderer.java | 61 +++++++-- .../video/VideoRendererEventListener.java | 23 ++++ .../DefaultAnalyticsCollectorTest.java | 13 ++ .../video/DecoderVideoRendererTest.java | 3 +- .../video/MediaCodecVideoRendererTest.java | 121 ++++++++++++++++-- .../playback/gts/DebugRenderersFactory.java | 3 +- .../test/utils/CapturingRenderersFactory.java | 3 +- .../ExperimentalFrameExtractor.java | 3 +- .../transformer/SequenceRenderersFactory.java | 2 + 19 files changed, 342 insertions(+), 38 deletions(-) diff --git a/libraries/decoder_av1/src/main/java/androidx/media3/decoder/av1/Libgav1VideoRenderer.java b/libraries/decoder_av1/src/main/java/androidx/media3/decoder/av1/Libgav1VideoRenderer.java index 68a38bb2a16..65f66ef223e 100644 --- a/libraries/decoder_av1/src/main/java/androidx/media3/decoder/av1/Libgav1VideoRenderer.java +++ b/libraries/decoder_av1/src/main/java/androidx/media3/decoder/av1/Libgav1VideoRenderer.java @@ -78,17 +78,22 @@ public class Libgav1VideoRenderer extends DecoderVideoRenderer { * @param eventListener A listener of events. May be null if delivery of events is not required. * @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between * invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}. + * @param minConsecutiveDroppedFramesToNotify The minimum number of consecutive frames that must + * be dropped for {@link VideoRendererEventListener#onConsecutiveDroppedFrames(int, long)} to + * be called. */ public Libgav1VideoRenderer( long allowedJoiningTimeMs, @Nullable Handler eventHandler, @Nullable VideoRendererEventListener eventListener, - int maxDroppedFramesToNotify) { + int maxDroppedFramesToNotify, + int minConsecutiveDroppedFramesToNotify) { this( allowedJoiningTimeMs, eventHandler, eventListener, maxDroppedFramesToNotify, + minConsecutiveDroppedFramesToNotify, THREAD_COUNT_AUTODETECT, DEFAULT_NUM_OF_INPUT_BUFFERS, DEFAULT_NUM_OF_OUTPUT_BUFFERS); @@ -104,6 +109,9 @@ public Libgav1VideoRenderer( * @param eventListener A listener of events. May be null if delivery of events is not required. * @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between * invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}. + * @param minConsecutiveDroppedFramesToNotify The minimum number of consecutive frames that must + * be dropped for {@link VideoRendererEventListener#onConsecutiveDroppedFrames(int, long)} to + * be called. * @param threads Number of threads libgav1 will use to decode. If {@link * #THREAD_COUNT_AUTODETECT} is passed, then the number of threads to use is autodetected * based on CPU capabilities. @@ -115,10 +123,16 @@ public Libgav1VideoRenderer( @Nullable Handler eventHandler, @Nullable VideoRendererEventListener eventListener, int maxDroppedFramesToNotify, + int minConsecutiveDroppedFramesToNotify, int threads, int numInputBuffers, int numOutputBuffers) { - super(allowedJoiningTimeMs, eventHandler, eventListener, maxDroppedFramesToNotify); + super( + allowedJoiningTimeMs, + eventHandler, + eventListener, + maxDroppedFramesToNotify, + minConsecutiveDroppedFramesToNotify); this.threads = threads; this.numInputBuffers = numInputBuffers; this.numOutputBuffers = numOutputBuffers; diff --git a/libraries/decoder_ffmpeg/src/main/java/androidx/media3/decoder/ffmpeg/ExperimentalFfmpegVideoRenderer.java b/libraries/decoder_ffmpeg/src/main/java/androidx/media3/decoder/ffmpeg/ExperimentalFfmpegVideoRenderer.java index e9b765906b3..05b801f3828 100644 --- a/libraries/decoder_ffmpeg/src/main/java/androidx/media3/decoder/ffmpeg/ExperimentalFfmpegVideoRenderer.java +++ b/libraries/decoder_ffmpeg/src/main/java/androidx/media3/decoder/ffmpeg/ExperimentalFfmpegVideoRenderer.java @@ -57,13 +57,22 @@ public final class ExperimentalFfmpegVideoRenderer extends DecoderVideoRenderer * @param eventListener A listener of events. May be null if delivery of events is not required. * @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between * invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}. + * @param minConsecutiveDroppedFramesToNotify The minimum number of consecutive frames that must + * be dropped for {@link VideoRendererEventListener#onConsecutiveDroppedFrames(int, long)} to + * be called. */ public ExperimentalFfmpegVideoRenderer( long allowedJoiningTimeMs, @Nullable Handler eventHandler, @Nullable VideoRendererEventListener eventListener, - int maxDroppedFramesToNotify) { - super(allowedJoiningTimeMs, eventHandler, eventListener, maxDroppedFramesToNotify); + int maxDroppedFramesToNotify, + int minConsecutiveDroppedFramesToNotify) { + super( + allowedJoiningTimeMs, + eventHandler, + eventListener, + maxDroppedFramesToNotify, + minConsecutiveDroppedFramesToNotify); // TODO: Implement. } diff --git a/libraries/decoder_vp9/src/main/java/androidx/media3/decoder/vp9/LibvpxVideoRenderer.java b/libraries/decoder_vp9/src/main/java/androidx/media3/decoder/vp9/LibvpxVideoRenderer.java index 76da83d405f..bf2e6d24b1f 100644 --- a/libraries/decoder_vp9/src/main/java/androidx/media3/decoder/vp9/LibvpxVideoRenderer.java +++ b/libraries/decoder_vp9/src/main/java/androidx/media3/decoder/vp9/LibvpxVideoRenderer.java @@ -65,7 +65,7 @@ public class LibvpxVideoRenderer extends DecoderVideoRenderer { * can attempt to seamlessly join an ongoing playback. */ public LibvpxVideoRenderer(long allowedJoiningTimeMs) { - this(allowedJoiningTimeMs, null, null, 0); + this(allowedJoiningTimeMs, null, null, 0, 0); } /** @@ -78,17 +78,22 @@ public LibvpxVideoRenderer(long allowedJoiningTimeMs) { * @param eventListener A listener of events. May be null if delivery of events is not required. * @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between * invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}. + * @param minConsecutiveDroppedFramesToNotify The minimum number of consecutive frames that must + * be dropped for {@link VideoRendererEventListener#onConsecutiveDroppedFrames(int, long)} to + * be called. */ public LibvpxVideoRenderer( long allowedJoiningTimeMs, @Nullable Handler eventHandler, @Nullable VideoRendererEventListener eventListener, - int maxDroppedFramesToNotify) { + int maxDroppedFramesToNotify, + int minConsecutiveDroppedFramesToNotify) { this( allowedJoiningTimeMs, eventHandler, eventListener, maxDroppedFramesToNotify, + minConsecutiveDroppedFramesToNotify, getRuntime().availableProcessors(), /* numInputBuffers= */ 4, /* numOutputBuffers= */ 4); @@ -104,6 +109,9 @@ public LibvpxVideoRenderer( * @param eventListener A listener of events. May be null if delivery of events is not required. * @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between * invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}. + * @param minConsecutiveDroppedFramesToNotify The minimum number of consecutive frames that must + * be dropped for {@link VideoRendererEventListener#onConsecutiveDroppedFrames(int, long)} to + * be called. * @param threads Number of threads libvpx will use to decode. * @param numInputBuffers Number of input buffers. * @param numOutputBuffers Number of output buffers. @@ -113,10 +121,16 @@ public LibvpxVideoRenderer( @Nullable Handler eventHandler, @Nullable VideoRendererEventListener eventListener, int maxDroppedFramesToNotify, + int minConsecutiveDroppedFramesToNotify, int threads, int numInputBuffers, int numOutputBuffers) { - super(allowedJoiningTimeMs, eventHandler, eventListener, maxDroppedFramesToNotify); + super( + allowedJoiningTimeMs, + eventHandler, + eventListener, + maxDroppedFramesToNotify, + minConsecutiveDroppedFramesToNotify); this.threads = threads; this.numInputBuffers = numInputBuffers; this.numOutputBuffers = numOutputBuffers; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java index bb2e9a9dd63..72bc55a1577 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java @@ -96,6 +96,12 @@ public class DefaultRenderersFactory implements RenderersFactory { */ public static final int MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY = 50; + /** + * The minimum number of consecutive video frames that would be dropped to + * consider invoking {@link VideoRendererEventListener#onConsecutiveDroppedFrames(int, long)}. + */ + public static final int MIN_CONSECUTIVE_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY = 5; + private static final String TAG = "DefaultRenderersFactory"; private final Context context; @@ -347,7 +353,8 @@ protected void buildVideoRenderers( enableDecoderFallback, eventHandler, eventListener, - MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY); + MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY, + MIN_CONSECUTIVE_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY); out.add(videoRenderer); if (extensionRendererMode == EXTENSION_RENDERER_MODE_OFF) { @@ -366,6 +373,7 @@ protected void buildVideoRenderers( long.class, android.os.Handler.class, androidx.media3.exoplayer.video.VideoRendererEventListener.class, + int.class, int.class); Renderer renderer = (Renderer) @@ -373,7 +381,8 @@ protected void buildVideoRenderers( allowedVideoJoiningTimeMs, eventHandler, eventListener, - MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY); + MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY, + MIN_CONSECUTIVE_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY); out.add(extensionRendererIndex++, renderer); Log.i(TAG, "Loaded LibvpxVideoRenderer."); } catch (ClassNotFoundException e) { @@ -391,6 +400,7 @@ protected void buildVideoRenderers( long.class, android.os.Handler.class, androidx.media3.exoplayer.video.VideoRendererEventListener.class, + int.class, int.class); Renderer renderer = (Renderer) @@ -398,7 +408,8 @@ protected void buildVideoRenderers( allowedVideoJoiningTimeMs, eventHandler, eventListener, - MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY); + MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY, + MIN_CONSECUTIVE_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY); out.add(extensionRendererIndex++, renderer); Log.i(TAG, "Loaded Libgav1VideoRenderer."); } catch (ClassNotFoundException e) { @@ -417,6 +428,7 @@ protected void buildVideoRenderers( long.class, android.os.Handler.class, androidx.media3.exoplayer.video.VideoRendererEventListener.class, + int.class, int.class); Renderer renderer = (Renderer) @@ -424,7 +436,8 @@ protected void buildVideoRenderers( allowedVideoJoiningTimeMs, eventHandler, eventListener, - MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY); + MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY, + MIN_CONSECUTIVE_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY); out.add(extensionRendererIndex++, renderer); Log.i(TAG, "Loaded FfmpegVideoRenderer."); } catch (ClassNotFoundException e) { diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java index 558777d0751..2ef2a2596cb 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java @@ -3033,6 +3033,11 @@ public void onDroppedFrames(int count, long elapsed) { analyticsCollector.onDroppedFrames(count, elapsed); } + @Override + public void onConsecutiveDroppedFrames(int count, long elapsed) { + analyticsCollector.onConsecutiveDroppedFrames(count, elapsed); + } + @Override public void onVideoSizeChanged(VideoSize newVideoSize) { videoSize = newVideoSize; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/AnalyticsCollector.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/AnalyticsCollector.java index 3b36813f35a..3e64a984e91 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/AnalyticsCollector.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/AnalyticsCollector.java @@ -250,6 +250,18 @@ void onVideoInputFormatChanged( */ void onDroppedFrames(int count, long elapsedMs); + /** + * Called to report the number of consecutive frames dropped by the video renderer. Consecutive + * dropped frames are reported once a frame is renderered after a period in which more frames were + * dropped consecutively than the specified threshold. + * + * @param count The number of consecutive dropped frames. + * @param elapsedMs The duration in milliseconds over which the consecutive frames were dropped. + * This duration is timed from when the first frame was dropped, until the time the renderer + * succesfully rendered a frame or the rendered was interrupted (stopped, seeked, disabled). + */ + void onConsecutiveDroppedFrames(int count, long elapsedMs); + /** * Called when a video decoder is released. * diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/AnalyticsListener.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/AnalyticsListener.java index e8850579182..da352b277bc 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/AnalyticsListener.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/AnalyticsListener.java @@ -235,7 +235,8 @@ public int size() { EVENT_VIDEO_CODEC_ERROR, EVENT_AUDIO_TRACK_INITIALIZED, EVENT_AUDIO_TRACK_RELEASED, - EVENT_RENDERER_READY_CHANGED + EVENT_RENDERER_READY_CHANGED, + EVENT_CONSECUTIVE_DROPPED_VIDEO_FRAMES, }) @interface EventFlags {} @@ -448,6 +449,9 @@ public int size() { /** A renderer changed its readiness for playback. */ @UnstableApi int EVENT_RENDERER_READY_CHANGED = 1033; + /** Consecutive video frames have been dropped. */ + @UnstableApi int EVENT_CONSECUTIVE_DROPPED_VIDEO_FRAMES = 1034; + /** Time information of an event. */ @UnstableApi final class EventTime { @@ -1244,6 +1248,18 @@ default void onVideoInputFormatChanged( @UnstableApi default void onDroppedVideoFrames(EventTime eventTime, int droppedFrames, long elapsedMs) {} + /** + * Called after consecutive video frames have been dropped. + * + * @param eventTime The event time. + * @param consecutiveDroppedFrames The number of consecutive frames that have been dropped before the + * last rendered frame. + * @param elapsedMs The duration in milliseconds over which the frames were dropped. This duration + * is timed from the first dropped framed in the sequence. + */ + @UnstableApi + default void onConsecutiveDroppedVideoFrames(EventTime eventTime, int droppedFrames, long elapsedMs) {} + /** * Called when a video renderer releases a decoder. * diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/DefaultAnalyticsCollector.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/DefaultAnalyticsCollector.java index c12e798f547..b92ad3ce39e 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/DefaultAnalyticsCollector.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/DefaultAnalyticsCollector.java @@ -340,6 +340,15 @@ public final void onDroppedFrames(int count, long elapsedMs) { listener -> listener.onDroppedVideoFrames(eventTime, count, elapsedMs)); } + @Override + public final void onConsecutiveDroppedFrames(int count, long elapsedMs) { + EventTime eventTime = generatePlayingMediaPeriodEventTime(); + sendEvent( + eventTime, + AnalyticsListener.EVENT_CONSECUTIVE_DROPPED_VIDEO_FRAMES, + listener -> listener.onConsecutiveDroppedVideoFrames(eventTime, count, elapsedMs)); + } + @Override public final void onVideoDecoderReleased(String decoderName) { EventTime eventTime = generateReadingMediaPeriodEventTime(); diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/util/EventLogger.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/util/EventLogger.java index 897d022e6a5..5155aec3638 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/util/EventLogger.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/util/EventLogger.java @@ -451,6 +451,13 @@ public void onDroppedVideoFrames(EventTime eventTime, int droppedFrames, long el logd(eventTime, "droppedFrames", Integer.toString(droppedFrames)); } + @UnstableApi + @Override + public void onConsecutiveDroppedVideoFrames( + EventTime eventTime, int consecutiveDroppedFrames, long elapsedMs) { + logd(eventTime, "consecutiveDroppedFrames", Integer.toString(consecutiveDroppedFrames)); + } + @UnstableApi @Override public void onVideoDecoderReleased(EventTime eventTime, String decoderName) { diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DecoderVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DecoderVideoRenderer.java index 5bb81f91cae..a91b771345a 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DecoderVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DecoderVideoRenderer.java @@ -113,6 +113,7 @@ public abstract class DecoderVideoRenderer extends BaseRenderer { private final long allowedJoiningTimeMs; private final int maxDroppedFramesToNotify; + private final int minConsecutiveDroppedFramesToNotify; private final EventDispatcher eventDispatcher; private final TimedValueQueue formatQueue; private final DecoderInputBuffer flagsOnlyBuffer; @@ -150,6 +151,7 @@ public abstract class DecoderVideoRenderer extends BaseRenderer { private long droppedFrameAccumulationStartTimeMs; private int droppedFrames; + private long consecutiveDroppedFrameAccumulationStartTimeMs; private int consecutiveDroppedFrameCount; private int buffersInCodecCount; private long lastRenderTimeUs; @@ -165,15 +167,20 @@ public abstract class DecoderVideoRenderer extends BaseRenderer { * @param eventListener A listener of events. May be null if delivery of events is not required. * @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between * invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}. + * @param minConsecutiveDroppedFramesToNotify The minimum number of consecutive frames that must + * be dropped for {@link VideoRendererEventListener#onConsecutiveDroppedFrames(int, long)} to + * be called. */ protected DecoderVideoRenderer( long allowedJoiningTimeMs, @Nullable Handler eventHandler, @Nullable VideoRendererEventListener eventListener, - int maxDroppedFramesToNotify) { + int maxDroppedFramesToNotify, + int minConsecutiveDroppedFramesToNotify) { super(C.TRACK_TYPE_VIDEO); this.allowedJoiningTimeMs = allowedJoiningTimeMs; this.maxDroppedFramesToNotify = maxDroppedFramesToNotify; + this.minConsecutiveDroppedFramesToNotify = minConsecutiveDroppedFramesToNotify; joiningDeadlineMs = C.TIME_UNSET; formatQueue = new TimedValueQueue<>(); flagsOnlyBuffer = DecoderInputBuffer.newNoDataInstance(); @@ -296,7 +303,7 @@ protected void onPositionReset(long positionUs, boolean joining) throws ExoPlayb outputStreamEnded = false; lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED); initialPositionUs = C.TIME_UNSET; - consecutiveDroppedFrameCount = 0; + maybeNotifyConsecutiveDroppedFrames(); if (decoder != null) { flushDecoder(); } @@ -319,6 +326,7 @@ protected void onStarted() { protected void onStopped() { joiningDeadlineMs = C.TIME_UNSET; maybeNotifyDroppedFrames(); + maybeNotifyConsecutiveDroppedFrames(); } @Override @@ -603,8 +611,8 @@ protected void renderOutputBuffer( } else { renderOutputBufferToSurface(outputBuffer, checkNotNull(outputSurface)); } - consecutiveDroppedFrameCount = 0; decoderCounters.renderedOutputBufferCount++; + maybeNotifyConsecutiveDroppedFrames(); maybeNotifyRenderedFirstFrame(); } } @@ -996,6 +1004,17 @@ private void maybeNotifyDroppedFrames() { } } + private void maybeNotifyConsecutiveDroppedFrames() { + if (consecutiveDroppedFrameCount > 0 + && consecutiveDroppedFrameCount >= minConsecutiveDroppedFramesToNotify) { + long elapsedMs = SystemClock.elapsedRealtime() - consecutiveDroppedFrameAccumulationStartTimeMs; + eventDispatcher.consecutiveDroppedFrames(consecutiveDroppedFrameCount, elapsedMs); + } + // Always reset the counter to 0, even if the threshold is not reached. + consecutiveDroppedFrameCount = 0; + consecutiveDroppedFrameAccumulationStartTimeMs = SystemClock.elapsedRealtime(); + } + private static boolean isBufferLate(long earlyUs) { // Class a buffer as late if it should have been presented more than 30 ms ago. return earlyUs < -30000; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index 569646c47f7..f5c130bffd9 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -152,6 +152,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer private final boolean ownsVideoSink; private final EventDispatcher eventDispatcher; private final int maxDroppedFramesToNotify; + private final int minConsecutiveDroppedFramesToNotify; private final boolean deviceNeedsNoPostProcessWorkaround; private final VideoFrameReleaseControl videoFrameReleaseControl; private final VideoFrameReleaseControl.FrameReleaseInfo videoFrameReleaseInfo; @@ -170,6 +171,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer private @C.VideoChangeFrameRateStrategy int changeFrameRateStrategy; private long droppedFrameAccumulationStartTimeMs; private int droppedFrames; + private long consecutiveDroppedFrameAccumulationStartTimeMs; private int consecutiveDroppedFrameCount; private int buffersInCodecCount; private long totalVideoFrameProcessingOffsetUs; @@ -209,7 +211,8 @@ public MediaCodecVideoRenderer( allowedJoiningTimeMs, /* eventHandler= */ null, /* eventListener= */ null, - /* maxDroppedFramesToNotify= */ 0); + /* maxDroppedFramesToNotify= */ 0, + /* minConsecutiveDroppedFramesToNotify= */ 0); } /** @@ -222,6 +225,9 @@ public MediaCodecVideoRenderer( * @param eventListener A listener of events. May be null if delivery of events is not required. * @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between * invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}. + * @param minConsecutiveDroppedFramesToNotify The minimum number of consecutive frames that must + * be dropped for {@link VideoRendererEventListener#onConsecutiveDroppedFrames(int, long)} to + * be invoked. */ public MediaCodecVideoRenderer( Context context, @@ -229,7 +235,8 @@ public MediaCodecVideoRenderer( long allowedJoiningTimeMs, @Nullable Handler eventHandler, @Nullable VideoRendererEventListener eventListener, - int maxDroppedFramesToNotify) { + int maxDroppedFramesToNotify, + int minConsecutiveDroppedFramesToNotify) { this( context, MediaCodecAdapter.Factory.getDefault(context), @@ -239,6 +246,7 @@ public MediaCodecVideoRenderer( eventHandler, eventListener, maxDroppedFramesToNotify, + minConsecutiveDroppedFramesToNotify, /* assumedMinimumCodecOperatingRate= */ 30); } @@ -255,6 +263,9 @@ public MediaCodecVideoRenderer( * @param eventListener A listener of events. May be null if delivery of events is not required. * @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between * invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}. + * @param minConsecutiveDroppedFramesToNotify The minimum number of consecutive frames that must + * be dropped for {@link VideoRendererEventListener#onConsecutiveDroppedFrames(int, long)} to + * be invoked. */ public MediaCodecVideoRenderer( Context context, @@ -263,7 +274,8 @@ public MediaCodecVideoRenderer( boolean enableDecoderFallback, @Nullable Handler eventHandler, @Nullable VideoRendererEventListener eventListener, - int maxDroppedFramesToNotify) { + int maxDroppedFramesToNotify, + int minConsecutiveDroppedFramesToNotify) { this( context, MediaCodecAdapter.Factory.getDefault(context), @@ -273,6 +285,7 @@ public MediaCodecVideoRenderer( eventHandler, eventListener, maxDroppedFramesToNotify, + minConsecutiveDroppedFramesToNotify, /* assumedMinimumCodecOperatingRate= */ 30); } @@ -291,6 +304,9 @@ public MediaCodecVideoRenderer( * @param eventListener A listener of events. May be null if delivery of events is not required. * @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between * invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}. + * @param minConsecutiveDroppedFramesToNotify The minimum number of consecutive frames that must + * be dropped for {@link VideoRendererEventListener#onConsecutiveDroppedFrames(int, long)} to + * be invoked. */ public MediaCodecVideoRenderer( Context context, @@ -300,7 +316,8 @@ public MediaCodecVideoRenderer( boolean enableDecoderFallback, @Nullable Handler eventHandler, @Nullable VideoRendererEventListener eventListener, - int maxDroppedFramesToNotify) { + int maxDroppedFramesToNotify, + int minConsecutiveDroppedFramesToNotify) { this( context, codecAdapterFactory, @@ -310,6 +327,7 @@ public MediaCodecVideoRenderer( eventHandler, eventListener, maxDroppedFramesToNotify, + minConsecutiveDroppedFramesToNotify, /* assumedMinimumCodecOperatingRate= */ 30); } @@ -330,6 +348,9 @@ public MediaCodecVideoRenderer( * @param eventListener A listener of events. May be null if delivery of events is not required. * @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between * invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}. + * @param minConsecutiveDroppedFramesToNotify The minimum number of consecutive frames that must + * be dropped for {@link VideoRendererEventListener#onConsecutiveDroppedFrames(int, long)} to + * be invoked. * @param assumedMinimumCodecOperatingRate A codec operating rate that all codecs instantiated by * this renderer are assumed to meet implicitly (i.e. without the operating rate being set * explicitly using {@link MediaFormat#KEY_OPERATING_RATE}). @@ -343,6 +364,7 @@ public MediaCodecVideoRenderer( @Nullable Handler eventHandler, @Nullable VideoRendererEventListener eventListener, int maxDroppedFramesToNotify, + int minConsecutiveDroppedFramesToNotify, float assumedMinimumCodecOperatingRate) { this( context, @@ -353,14 +375,15 @@ public MediaCodecVideoRenderer( eventHandler, eventListener, maxDroppedFramesToNotify, + minConsecutiveDroppedFramesToNotify, assumedMinimumCodecOperatingRate, /* videoSink= */ (VideoSink) null); } /** * @deprecated Use {@link #MediaCodecVideoRenderer(Context, MediaCodecAdapter.Factory, - * MediaCodecSelector, long, boolean, Handler, VideoRendererEventListener, int, float, - * VideoSink)} instead. + * MediaCodecSelector, long, boolean, Handler, VideoRendererEventListener, int, int, + * float, VideoSink)} instead. */ @Deprecated public MediaCodecVideoRenderer( @@ -372,6 +395,7 @@ public MediaCodecVideoRenderer( @Nullable Handler eventHandler, @Nullable VideoRendererEventListener eventListener, int maxDroppedFramesToNotify, + int minConsecutiveDroppedFramesToNotify, float assumedMinimumCodecOperatingRate, @Nullable VideoSinkProvider videoSinkProvider) { this( @@ -383,6 +407,7 @@ public MediaCodecVideoRenderer( eventHandler, eventListener, maxDroppedFramesToNotify, + minConsecutiveDroppedFramesToNotify, assumedMinimumCodecOperatingRate, videoSinkProvider == null ? null : videoSinkProvider.getSink()); } @@ -404,6 +429,9 @@ public MediaCodecVideoRenderer( * @param eventListener A listener of events. May be null if delivery of events is not required. * @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between * invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}. + * @param minConsecutiveDroppedFramesToNotify The minimum number of consecutive frames that must + * be dropped for {@link VideoRendererEventListener#onConsecutiveDroppedFrames(int, long)} to + * be invoked. * @param assumedMinimumCodecOperatingRate A codec operating rate that all codecs instantiated by * this renderer are assumed to meet implicitly (i.e. without the operating rate being set * explicitly using {@link MediaFormat#KEY_OPERATING_RATE}). @@ -421,6 +449,7 @@ public MediaCodecVideoRenderer( @Nullable Handler eventHandler, @Nullable VideoRendererEventListener eventListener, int maxDroppedFramesToNotify, + int minConsecutiveDroppedFramesToNotify, float assumedMinimumCodecOperatingRate, @Nullable VideoSink videoSink) { super( @@ -431,6 +460,7 @@ public MediaCodecVideoRenderer( assumedMinimumCodecOperatingRate); this.context = context.getApplicationContext(); this.maxDroppedFramesToNotify = maxDroppedFramesToNotify; + this.minConsecutiveDroppedFramesToNotify = minConsecutiveDroppedFramesToNotify; this.videoSink = videoSink; eventDispatcher = new EventDispatcher(eventHandler, eventListener); ownsVideoSink = videoSink == null; @@ -831,7 +861,7 @@ protected void onPositionReset(long positionUs, boolean joining) throws ExoPlayb } } maybeSetupTunnelingForFirstFrame(); - consecutiveDroppedFrameCount = 0; + maybeNotifyConsecutiveDroppedFrames(); } @Override @@ -870,6 +900,7 @@ protected void onStarted() { @Override protected void onStopped() { maybeNotifyDroppedFrames(); + maybeNotifyConsecutiveDroppedFrames(); maybeNotifyVideoFrameProcessingOffset(); if (videoSink != null) { videoSink.onRendererStopped(); @@ -1792,7 +1823,7 @@ protected void renderOutputBuffer(MediaCodecAdapter codec, int index, long prese codec.releaseOutputBuffer(index, true); TraceUtil.endSection(); decoderCounters.renderedOutputBufferCount++; - consecutiveDroppedFrameCount = 0; + maybeNotifyConsecutiveDroppedFrames(); if (videoSink == null) { maybeNotifyVideoSizeChanged(decodedVideoSize); maybeNotifyRenderedFirstFrame(); @@ -1813,7 +1844,7 @@ protected void renderOutputBufferV21( codec.releaseOutputBuffer(index, releaseTimeNs); TraceUtil.endSection(); decoderCounters.renderedOutputBufferCount++; - consecutiveDroppedFrameCount = 0; + maybeNotifyConsecutiveDroppedFrames(); if (videoSink == null) { maybeNotifyVideoSizeChanged(decodedVideoSize); maybeNotifyRenderedFirstFrame(); @@ -1948,6 +1979,18 @@ private void maybeNotifyDroppedFrames() { } } + private void maybeNotifyConsecutiveDroppedFrames() { + if (minConsecutiveDroppedFramesToNotify > 0 + && consecutiveDroppedFrameCount > 0 + && consecutiveDroppedFrameCount >= minConsecutiveDroppedFramesToNotify) { + long elapsedMs = getClock().elapsedRealtime() - consecutiveDroppedFrameAccumulationStartTimeMs ; + eventDispatcher.consecutiveDroppedFrames(consecutiveDroppedFrameCount, elapsedMs); + } + // Always reset the counter to 0, even if the threshold is not reached. + consecutiveDroppedFrameCount = 0; + consecutiveDroppedFrameAccumulationStartTimeMs = getClock().elapsedRealtime(); + } + private void maybeNotifyVideoFrameProcessingOffset() { if (videoFrameProcessingOffsetCount != 0) { eventDispatcher.reportVideoFrameProcessingOffset( diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoRendererEventListener.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoRendererEventListener.java index 14cd50ea3a0..21e5a5faa84 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoRendererEventListener.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoRendererEventListener.java @@ -82,6 +82,19 @@ default void onVideoInputFormatChanged( */ default void onDroppedFrames(int count, long elapsedMs) {} + /** + * Called to report the number of consecutive frames dropped by the video renderer. Consecutive + * dropped frames are reported when a frame is renderered after a the previous consecutive frames + * were not rendered optionally, whenever the consecutive dropped frame count is above a specified + * threshold whilst the renderer is started. + * + * @param count The number of consecutive dropped frames. + * @param elapsedMs The duration in milliseconds over which the consecutive frames were dropped. + * This duration is timed from the first dropped frame occured, until the time the renderer + * rendered a frame. + */ + default void onConsecutiveDroppedFrames(int count, long elapsedMs) {} + /** * Called to report the video processing offset of video frames processed by the video renderer. * @@ -205,6 +218,16 @@ public void droppedFrames(int droppedFrameCount, long elapsedMs) { } } + /** Invokes {@link VideoRendererEventListener#onConsecutiveDroppedFrames(int, long)}. */ + public void consecutiveDroppedFrames(int consecutiveDroppedFrameCount, long elapsedMs) { + if (handler != null) { + handler.post( + () -> + castNonNull(listener) + .onConsecutiveDroppedFrames(consecutiveDroppedFrameCount, elapsedMs)); + } + } + /** Invokes {@link VideoRendererEventListener#onVideoFrameProcessingOffset}. */ public void reportVideoFrameProcessingOffset(long totalProcessingOffsetUs, int frameCount) { if (handler != null) { diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/analytics/DefaultAnalyticsCollectorTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/analytics/DefaultAnalyticsCollectorTest.java index 2e8a941f9d4..da0f8c860f6 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/analytics/DefaultAnalyticsCollectorTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/analytics/DefaultAnalyticsCollectorTest.java @@ -20,6 +20,7 @@ import static androidx.media3.exoplayer.analytics.AnalyticsListener.EVENT_AUDIO_ENABLED; import static androidx.media3.exoplayer.analytics.AnalyticsListener.EVENT_AUDIO_INPUT_FORMAT_CHANGED; import static androidx.media3.exoplayer.analytics.AnalyticsListener.EVENT_AUDIO_POSITION_ADVANCING; +import static androidx.media3.exoplayer.analytics.AnalyticsListener.EVENT_CONSECUTIVE_DROPPED_VIDEO_FRAMES; import static androidx.media3.exoplayer.analytics.AnalyticsListener.EVENT_DOWNSTREAM_FORMAT_CHANGED; import static androidx.media3.exoplayer.analytics.AnalyticsListener.EVENT_DRM_KEYS_LOADED; import static androidx.media3.exoplayer.analytics.AnalyticsListener.EVENT_DRM_SESSION_ACQUIRED; @@ -69,6 +70,7 @@ import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.robolectric.shadows.ShadowLooper.idleMainLooper; @@ -1859,6 +1861,11 @@ public void onEvents_isReportedWithCorrectEventTimes() throws Exception { ArgumentCaptor.forClass(AnalyticsListener.EventTime.class); verify(listener, atLeastOnce()) .onDroppedVideoFrames(individualDroppedFramesEventTimes.capture(), anyInt(), anyLong()); + ArgumentCaptor individualConsecutiveDroppedFramesEventTimes = + ArgumentCaptor.forClass(AnalyticsListener.EventTime.class); + verify(listener, never()) + .onConsecutiveDroppedVideoFrames( + individualConsecutiveDroppedFramesEventTimes.capture(), anyInt(), anyLong()); // Verify the EventTimes reported with onEvents are a non-empty subset of the individual // callback EventTimes. We can only assert they are a non-empty subset because there may be @@ -2393,6 +2400,12 @@ public void onDroppedVideoFrames(EventTime eventTime, int droppedFrames, long el reportedEvents.add(new ReportedEvent(EVENT_DROPPED_VIDEO_FRAMES, eventTime)); } + @Override + public void onConsecutiveDroppedVideoFrames( + EventTime eventTime, int consecutiveDroppedFrames, long elapsedMs) { + reportedEvents.add(new ReportedEvent(EVENT_CONSECUTIVE_DROPPED_VIDEO_FRAMES, eventTime)); + } + @Override public void onVideoDisabled(EventTime eventTime, DecoderCounters decoderCounters) { reportedEvents.add(new ReportedEvent(EVENT_VIDEO_DISABLED, eventTime)); diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/DecoderVideoRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/DecoderVideoRendererTest.java index a411f8d1fe5..80f56c5ab60 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/DecoderVideoRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/DecoderVideoRendererTest.java @@ -82,7 +82,8 @@ public void setUp() { /* allowedJoiningTimeMs= */ 0, new Handler(), eventListener, - /* maxDroppedFramesToNotify= */ -1) { + /* maxDroppedFramesToNotify= */ -1, + /* minConsecutiveDroppedFramesToNotify= */ -1) { private final Phaser inputBuffersInCodecPhaser = new Phaser(); private @C.VideoOutputMode int outputMode; diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java index f188f100140..2a7b6e91f00 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java @@ -22,6 +22,7 @@ import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.oneByteSample; import static androidx.media3.test.utils.FakeTimeline.TimelineWindowDefinition.DEFAULT_WINDOW_OFFSET_IN_FIRST_PERIOD_US; import static com.google.common.truth.Truth.assertThat; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.never; @@ -178,7 +179,8 @@ public void setUp() throws Exception { /* allowedJoiningTimeMs= */ 0, /* eventHandler= */ new Handler(testMainLooper), /* eventListener= */ eventListener, - /* maxDroppedFramesToNotify= */ 1) { + /* maxDroppedFramesToNotify= */ 1, + /* minConsecutiveDroppedFramesToNotify= */ 5) { @Override protected @Capabilities int supportsFormat( MediaCodecSelector mediaCodecSelector, Format format) { @@ -299,6 +301,95 @@ public void render_withVeryLateBuffer_dropsBuffersUpstream() throws Exception { assertThat(decoderCounters.droppedToKeyframeCount).isEqualTo(1); } + @Test + public void render_withConsecutiveDroppedFrames_invokesCallback() throws Exception { + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ VIDEO_H264, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), // First buffer. + oneByteSample(/* timeUs= */ 30_000), // Late buffer. + oneByteSample(/* timeUs= */ 60_000), // Late buffer. + oneByteSample(/* timeUs= */ 90_000), // Late buffer. + oneByteSample(/* timeUs= */ 120_000), // Late buffer. + oneByteSample(/* timeUs= */ 150_000), // Late buffer. + oneByteSample(/* timeUs= */ 180_000), // Rendered buffer. + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + mediaCodecVideoRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {VIDEO_H264}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + /* mediaPeriodId= */ new MediaSource.MediaPeriodId(new Object())); + + mediaCodecVideoRenderer.start(); + mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000); + mediaCodecVideoRenderer.render(20_000, SystemClock.elapsedRealtime() * 1000); + mediaCodecVideoRenderer.setCurrentStreamFinal(); + int posUs = 180_001; // Ensures buffer will be just past the last buffer we want to render. + while (!mediaCodecVideoRenderer.isEnded()) { + mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000); + posUs += 30_000; + } + shadowOf(testMainLooper).idle(); + verify(eventListener).onConsecutiveDroppedFrames(eq(5), anyLong()); + } + + @Test + public void render_withoutEnoughConsecutiveDroppedFrames_doeNotInvokesCallback() throws Exception { + ArgumentCaptor argumentDecoderCounters = + ArgumentCaptor.forClass(DecoderCounters.class); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ VIDEO_H264, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), // First buffer. + oneByteSample(/* timeUs= */ 30_000), // Late buffer. + oneByteSample(/* timeUs= */ 60_000), // Late buffer. + oneByteSample(/* timeUs= */ 90_000), // Late buffer. + oneByteSample(/* timeUs= */ 120_000), // Late buffer. + oneByteSample(/* timeUs= */ 150_000), // Rendered buffer. + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + mediaCodecVideoRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {VIDEO_H264}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + /* mediaPeriodId= */ new MediaSource.MediaPeriodId(new Object())); + + mediaCodecVideoRenderer.start(); + mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000); + mediaCodecVideoRenderer.render(20_000, SystemClock.elapsedRealtime() * 1000); + mediaCodecVideoRenderer.setCurrentStreamFinal(); + int posUs = 150_001; // Ensures buffer will be just past the last buffer we want to render. + while (!mediaCodecVideoRenderer.isEnded()) { + mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000); + posUs += 30_000; + } + shadowOf(testMainLooper).idle(); + verify(eventListener, never()).onConsecutiveDroppedFrames(anyInt(), anyLong()); + verify(eventListener).onVideoEnabled(argumentDecoderCounters.capture()); + assertThat(argumentDecoderCounters.getValue().maxConsecutiveDroppedBufferCount).isEqualTo(4); + } + @Test public void render_withBufferLimitEqualToNumberOfSamples_rendersLastFrameAfterEndOfStream() throws Exception { @@ -328,7 +419,8 @@ public void render_withBufferLimitEqualToNumberOfSamples_rendersLastFrameAfterEn /* enableDecoderFallback= */ false, /* eventHandler= */ new Handler(testMainLooper), /* eventListener= */ eventListener, - /* maxDroppedFramesToNotify= */ 1); + /* maxDroppedFramesToNotify= */ 1, + /* minConsecutiveDroppedFramesToNotify= */ 5); mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface); mediaCodecVideoRenderer.enable( @@ -390,7 +482,8 @@ public void render_withoutSampleDependencies_rendersLastFrameAfterEndOfStream() /* enableDecoderFallback= */ false, /* eventHandler= */ new Handler(testMainLooper), eventListener, - /* maxDroppedFramesToNotify= */ 1); + /* maxDroppedFramesToNotify= */ 1, + /* minConsecutiveDroppedFramesToNotify= */ 5); mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface); FakeTimeline fakeTimeline = new FakeTimeline(); @@ -468,7 +561,8 @@ public void render_withoutSampleDependenciesAndShortDuration_skipsNoDecoderInput /* enableDecoderFallback= */ false, /* eventHandler= */ new Handler(testMainLooper), /* eventListener= */ eventListener, - /* maxDroppedFramesToNotify= */ 1); + /* maxDroppedFramesToNotify= */ 1, + /* minConsecutiveDroppedFramesToNotify= */ 5); mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface); mediaCodecVideoRenderer.setTimeline(fakeTimeline); @@ -578,7 +672,8 @@ public void onContinueLoadingRequested(MediaPeriod source) { /* enableDecoderFallback= */ false, /* eventHandler= */ new Handler(testMainLooper), /* eventListener= */ eventListener, - /* maxDroppedFramesToNotify= */ 1); + /* maxDroppedFramesToNotify= */ 1, + /* minConsecutiveDroppedFrameCountToNotify= */ 5); mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface); mediaCodecVideoRenderer.enable( @@ -683,7 +778,8 @@ public void onContinueLoadingRequested(MediaPeriod source) { /* enableDecoderFallback= */ false, /* eventHandler= */ new Handler(testMainLooper), /* eventListener= */ eventListener, - /* maxDroppedFramesToNotify= */ 1); + /* maxDroppedFramesToNotify= */ 1, + /* minConsecutiveDroppedFrameCountToNotify= */ 5); mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface); mediaCodecVideoRenderer.enable( @@ -1226,7 +1322,8 @@ public void supportsFormat_withDolbyVisionMedia_returnsTrueWhenFallbackToH265orH /* allowedJoiningTimeMs= */ 0, /* eventHandler= */ new Handler(testMainLooper), /* eventListener= */ eventListener, - /* maxDroppedFramesToNotify= */ 1); + /* maxDroppedFramesToNotify= */ 1, + /* minConsecutiveDroppedFramesToNotify= */ 5); renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); @Capabilities @@ -1311,7 +1408,8 @@ public void supportsFormat_withDolbyVision_setsDecoderSupportFlagsByDisplayDolby /* allowedJoiningTimeMs= */ 0, /* eventHandler= */ new Handler(testMainLooper), /* eventListener= */ eventListener, - /* maxDroppedFramesToNotify= */ 1); + /* maxDroppedFramesToNotify= */ 1, + /* minConsecutiveDroppedFramesToNotify= */ 5); renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); @Capabilities int capabilitiesDvheDtr = renderer.supportsFormat(formatDvheDtr); @@ -1371,7 +1469,8 @@ public void getDecoderInfo_withNonPerformantHardwareDecoder_returnsHardwareDecod /* allowedJoiningTimeMs= */ 0, /* eventHandler= */ new Handler(testMainLooper), /* eventListener= */ eventListener, - /* maxDroppedFramesToNotify= */ 1); + /* maxDroppedFramesToNotify= */ 1, + /* minConsecutiveDroppedFramesToNotify= */ 5); renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); List mediaCodecInfoList = @@ -1414,7 +1513,8 @@ public void getDecoderInfo_softwareDecoderPreferred_returnsSoftwareDecoderFirst( /* allowedJoiningTimeMs= */ 0, /* eventHandler= */ new Handler(testMainLooper), /* eventListener= */ eventListener, - /* maxDroppedFramesToNotify= */ 1); + /* maxDroppedFramesToNotify= */ 1, + /* minConsecutiveDroppedFramesToNotify= */ 5); renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); List mediaCodecInfoList = @@ -1452,6 +1552,7 @@ public void setOutputSurface(Surface surface) { /* eventHandler= */ new Handler(testMainLooper), /* eventListener= */ eventListener, /* maxDroppedFramesToNotify= */ 1, + /* minConsecutiveDroppedFramesToNotify= */ 5, /* assumedMinimumCodecOperatingRate= */ 30) { @Override protected @Capabilities int supportsFormat( diff --git a/libraries/test_exoplayer_playback/src/androidTest/java/androidx/media3/test/exoplayer/playback/gts/DebugRenderersFactory.java b/libraries/test_exoplayer_playback/src/androidTest/java/androidx/media3/test/exoplayer/playback/gts/DebugRenderersFactory.java index e676b761019..f18600022ea 100644 --- a/libraries/test_exoplayer_playback/src/androidTest/java/androidx/media3/test/exoplayer/playback/gts/DebugRenderersFactory.java +++ b/libraries/test_exoplayer_playback/src/androidTest/java/androidx/media3/test/exoplayer/playback/gts/DebugRenderersFactory.java @@ -69,7 +69,8 @@ protected void buildVideoRenderers( allowedVideoJoiningTimeMs, eventHandler, eventListener, - MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY)); + MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY, + MIN_CONSECUTIVE_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY)); } /** diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/CapturingRenderersFactory.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/CapturingRenderersFactory.java index 080f1f22cdd..a41fb08c083 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/CapturingRenderersFactory.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/CapturingRenderersFactory.java @@ -134,7 +134,8 @@ public Renderer[] createRenderers( /* enableDecoderFallback= */ false, eventHandler, videoRendererEventListener, - DefaultRenderersFactory.MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY) { + DefaultRenderersFactory.MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY, + DefaultRenderersFactory.MIN_CONSECUTIVE_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY) { @Override protected boolean shouldDropOutputBuffer( long earlyUs, long elapsedRealtimeUs, boolean isLastBuffer) { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java index ba709d4ed1d..ebf2d045cbc 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java @@ -622,7 +622,8 @@ public FrameExtractorRenderer( /* allowedJoiningTimeMs= */ 0, Util.createHandlerForCurrentOrMainLooper(), videoRendererEventListener, - /* maxDroppedFramesToNotify= */ 0); + /* maxDroppedFramesToNotify= */ 0, + /* minConsecutiveDroppedFramesToNotify= */ 0); this.toneMapHdrToSdr = toneMapHdrToSdr; effectsFromPlayer = ImmutableList.of(); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceRenderersFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceRenderersFactory.java index a62b59c89dc..8dc661552a8 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceRenderersFactory.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceRenderersFactory.java @@ -20,6 +20,7 @@ import static androidx.media3.common.util.Assertions.checkStateNotNull; import static androidx.media3.exoplayer.DefaultRenderersFactory.DEFAULT_ALLOWED_VIDEO_JOINING_TIME_MS; import static androidx.media3.exoplayer.DefaultRenderersFactory.MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY; +import static androidx.media3.exoplayer.DefaultRenderersFactory.MIN_CONSECUTIVE_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY; import android.content.Context; import android.graphics.Bitmap; @@ -288,6 +289,7 @@ public SequenceVideoRenderer( eventHandler, videoRendererEventListener, MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY, + MIN_CONSECUTIVE_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY, /* assumedMinimumCodecOperatingRate= */ DEFAULT_FRAME_RATE, videoSink); this.sequence = sequence;