Add Canvas Capture Strategy (#4777)

* WIP

* refactor(replay): Use main thread handler to schedule replay capture

* Remove logging in test

* Format code

* Changelog

* Initial impl of text ignoring canvas approach

* Extend options

* Fix race conditions

* Cleanup

* Draw rects for text, utilize Bitmap.asShared

* Fix processing on main thread, picture canvas size

* Naive way to measure PixelCopy processing duration

* Cache bitmap sampling

* Simplify text drawing

* Add trace sections

* Default to Canvas

* temp

* cleanup

* Add FramesTimingTracker, cleanup

* Cleanup

* Cleanup

* Move from ApiStatus.Internal to ApiStatus.Experimental

* Cleanup

* Address PR feedback

* Update Changelog

* Fix tests

* Fix Tests

* Re-use bitmap, ensure bitmap creation is thread safe

* Fix leaking BitmapShader, guard newer Android APIs

* Cleanup

* Address PR feedback

* Remove unused code

* Ensure shaders are removed everywhere

* Enable Canvas approach for demo app

* Update Changelog

* Address PR comments

---------

Co-authored-by: Roman Zavarnitsyn <[email protected]>
Co-authored-by: Sentry Github Bot <[email protected]>

Author: 3 people

CHANGELOG.md

@@ -13,6 +13,22 @@
     - Note that keys containing spaces are not supported.
 - Add experimental Sentry Android Distribution module for integrating with Sentry Build Distribution to check for and install updates ([#4804](https://github.com/getsentry/sentry-java/pull/4804))
 - Allow passing a different `Handler` to `SystemEventsBreadcrumbsIntegration` and `AndroidConnectionStatusProvider` so their callbacks are deliver to that handler ([#4808](https://github.com/getsentry/sentry-java/pull/4808))
+- Session Replay: Add new _experimental_ Canvas Capture Strategy ([#4777](https://github.com/getsentry/sentry-java/pull/4777))
+  - A new screenshot capture strategy that uses Android's Canvas API for more accurate and reliable text and image masking
+  - Any `.drawText()` or `.drawBitmap()` calls are replaced by rectangles, ensuring no text or images are present in the resulting output
+  - Note: If this strategy is used, all text and images will be masked, regardless of any masking configuration
+  - To enable this feature, set the `screenshotStrategy`, either via code:
+    ```kotlin
+    SentryAndroid.init(context) { options ->
+      options.sessionReplay.screenshotStrategy = ScreenshotStrategyType.CANVAS
+    }
+    ```
+    or AndroidManifest.xml:
+    ```xml
+    <application>
+      <meta-data android:name="io.sentry.session-replay.screenshot-strategy" android:value="canvas" />
+    </application>
+    ```
 
 ### Fixes
 

sentry-android-core/src/main/java/io/sentry/android/core/ManifestMetadataReader.java

@@ -6,6 +6,7 @@
 import io.sentry.ILogger;
 import io.sentry.InitPriority;
 import io.sentry.ProfileLifecycle;
+import io.sentry.ScreenshotStrategyType;
 import io.sentry.SentryFeedbackOptions;
 import io.sentry.SentryIntegrationPackageStorage;
 import io.sentry.SentryLevel;
@@ -111,6 +112,7 @@ final class ManifestMetadataReader {
   static final String REPLAYS_MASK_ALL_IMAGES = "io.sentry.session-replay.mask-all-images";
 
   static final String REPLAYS_DEBUG = "io.sentry.session-replay.debug";
+  static final String REPLAYS_SCREENSHOT_STRATEGY = "io.sentry.session-replay.screenshot-strategy";
 
   static final String FORCE_INIT = "io.sentry.force-init";
 
@@ -476,6 +478,16 @@ static void applyMetadata(
 
         options.getSessionReplay().setDebug(readBool(metadata, logger, REPLAYS_DEBUG, false));
 
+        final @Nullable String screenshotStrategyRaw =
+            readString(metadata, logger, REPLAYS_SCREENSHOT_STRATEGY, null);
+        if (screenshotStrategyRaw != null) {
+          if ("canvas".equals(screenshotStrategyRaw.toLowerCase(Locale.ROOT))) {
+            options.getSessionReplay().setScreenshotStrategy(ScreenshotStrategyType.CANVAS);
+          } else {
+            // always default to PIXEL_COPY
+            options.getSessionReplay().setScreenshotStrategy(ScreenshotStrategyType.PIXEL_COPY);
+          }
+        }
         options.setIgnoredErrors(readList(metadata, logger, IGNORED_ERRORS));
 
         final @Nullable List<String> includes = readList(metadata, logger, IN_APP_INCLUDES);

sentry-android-core/src/test/java/io/sentry/android/core/ManifestMetadataReaderTest.kt

@@ -1835,4 +1835,51 @@ class ManifestMetadataReaderTest {
     // Assert
     assertFalse(fixture.options.feedbackOptions.isShowBranding)
   }
+
+  @Test
+  fun `applyMetadata reads screenshot strategy canvas to options`() {
+    // Arrange
+    val bundle = bundleOf(ManifestMetadataReader.REPLAYS_SCREENSHOT_STRATEGY to "canvas")
+    val context = fixture.getContext(metaData = bundle)
+
+    // Act
+    ManifestMetadataReader.applyMetadata(context, fixture.options, fixture.buildInfoProvider)
+
+    // Assert
+    assertEquals(
+      io.sentry.ScreenshotStrategyType.CANVAS,
+      fixture.options.sessionReplay.screenshotStrategy,
+    )
+  }
+
+  @Test
+  fun `applyMetadata reads screenshot strategy and defaults to PIXEL_COPY for unknown value`() {
+    // Arrange
+    val bundle = bundleOf(ManifestMetadataReader.REPLAYS_SCREENSHOT_STRATEGY to "unknown")
+    val context = fixture.getContext(metaData = bundle)
+
+    // Act
+    ManifestMetadataReader.applyMetadata(context, fixture.options, fixture.buildInfoProvider)
+
+    // Assert
+    assertEquals(
+      io.sentry.ScreenshotStrategyType.PIXEL_COPY,
+      fixture.options.sessionReplay.screenshotStrategy,
+    )
+  }
+
+  @Test
+  fun `applyMetadata reads screenshot strategy and keeps default if not found`() {
+    // Arrange
+    val context = fixture.getContext()
+
+    // Act
+    ManifestMetadataReader.applyMetadata(context, fixture.options, fixture.buildInfoProvider)
+
+    // Assert
+    assertEquals(
+      io.sentry.ScreenshotStrategyType.PIXEL_COPY,
+      fixture.options.sessionReplay.screenshotStrategy,
+    )
+  }
 }

sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt

@@ -4,60 +4,51 @@ import android.annotation.SuppressLint
 import android.annotation.TargetApi
 import android.content.Context
 import android.graphics.Bitmap
-import android.graphics.Canvas
-import android.graphics.Color
-import android.graphics.Matrix
-import android.graphics.Paint
-import android.graphics.Rect
-import android.graphics.RectF
-import android.view.PixelCopy
 import android.view.View
 import android.view.ViewTreeObserver
+import io.sentry.ScreenshotStrategyType
 import io.sentry.SentryLevel.DEBUG
-import io.sentry.SentryLevel.INFO
 import io.sentry.SentryLevel.WARNING
 import io.sentry.SentryOptions
 import io.sentry.SentryReplayOptions
+import io.sentry.android.replay.screenshot.CanvasStrategy
+import io.sentry.android.replay.screenshot.PixelCopyStrategy
+import io.sentry.android.replay.screenshot.ScreenshotStrategy
 import io.sentry.android.replay.util.DebugOverlayDrawable
-import io.sentry.android.replay.util.MainLooperHandler
 import io.sentry.android.replay.util.addOnDrawListenerSafe
-import io.sentry.android.replay.util.getVisibleRects
 import io.sentry.android.replay.util.removeOnDrawListenerSafe
-import io.sentry.android.replay.util.submitSafely
-import io.sentry.android.replay.util.traverse
-import io.sentry.android.replay.viewhierarchy.ViewHierarchyNode
-import io.sentry.android.replay.viewhierarchy.ViewHierarchyNode.ImageViewHierarchyNode
-import io.sentry.android.replay.viewhierarchy.ViewHierarchyNode.TextViewHierarchyNode
 import java.io.File
 import java.lang.ref.WeakReference
-import java.util.concurrent.ScheduledExecutorService
 import java.util.concurrent.atomic.AtomicBoolean
-import kotlin.LazyThreadSafetyMode.NONE
 import kotlin.math.roundToInt
 
 @SuppressLint("UseKtx")
 @TargetApi(26)
 internal class ScreenshotRecorder(
   val config: ScreenshotRecorderConfig,
   val options: SentryOptions,
-  private val mainLooperHandler: MainLooperHandler,
-  private val recorder: ScheduledExecutorService,
-  private val screenshotRecorderCallback: ScreenshotRecorderCallback?,
+  val executorProvider: ExecutorProvider,
+  screenshotRecorderCallback: ScreenshotRecorderCallback?,
 ) : ViewTreeObserver.OnDrawListener {
   private var rootView: WeakReference<View>? = null
-  private val maskingPaint by lazy(NONE) { Paint() }
-  private val singlePixelBitmap: Bitmap by
-    lazy(NONE) { Bitmap.createBitmap(1, 1, Bitmap.Config.ARGB_8888) }
-  private val screenshot =
-    Bitmap.createBitmap(config.recordingWidth, config.recordingHeight, Bitmap.Config.ARGB_8888)
-  private val singlePixelBitmapCanvas: Canvas by lazy(NONE) { Canvas(singlePixelBitmap) }
-  private val prescaledMatrix by
-    lazy(NONE) { Matrix().apply { preScale(config.scaleFactorX, config.scaleFactorY) } }
-  private val contentChanged = AtomicBoolean(false)
   private val isCapturing = AtomicBoolean(true)
-  private val lastCaptureSuccessful = AtomicBoolean(false)
 
   private val debugOverlayDrawable = DebugOverlayDrawable()
+  private val contentChanged = AtomicBoolean(false)
+
+  private val screenshotStrategy: ScreenshotStrategy =
+    when (options.sessionReplay.screenshotStrategy) {
+      ScreenshotStrategyType.CANVAS ->
+        CanvasStrategy(executorProvider, screenshotRecorderCallback, options, config)
+      ScreenshotStrategyType.PIXEL_COPY ->
+        PixelCopyStrategy(
+          executorProvider,
+          screenshotRecorderCallback,
+          options,
+          config,
+          debugOverlayDrawable,
+        )
+    }
 
   fun capture() {
     if (options.sessionReplay.isDebug) {
@@ -75,12 +66,12 @@ internal class ScreenshotRecorder(
         DEBUG,
         "Capturing screenshot, contentChanged: %s, lastCaptureSuccessful: %s",
         contentChanged.get(),
-        lastCaptureSuccessful.get(),
+        screenshotStrategy.lastCaptureSuccessful(),
       )
     }
 
-    if (!contentChanged.get() && lastCaptureSuccessful.get()) {
-      screenshotRecorderCallback?.onScreenshotRecorded(screenshot)
+    if (!contentChanged.get()) {
+      screenshotStrategy.emitLastScreenshot()
       return
     }
 
@@ -98,93 +89,9 @@ internal class ScreenshotRecorder(
 
     try {
       contentChanged.set(false)
-      PixelCopy.request(
-        window,
-        screenshot,
-        { copyResult: Int ->
-          if (copyResult != PixelCopy.SUCCESS) {
-            options.logger.log(INFO, "Failed to capture replay recording: %d", copyResult)
-            lastCaptureSuccessful.set(false)
-            return@request
-          }
-
-          // TODO: handle animations with heuristics (e.g. if we fall under this condition 2 times
-          // in a row, we should capture)
-          if (contentChanged.get()) {
-            options.logger.log(INFO, "Failed to determine view hierarchy, not capturing")
-            lastCaptureSuccessful.set(false)
-            return@request
-          }
-
-          // TODO: disableAllMasking here and dont traverse?
-          val viewHierarchy = ViewHierarchyNode.fromView(root, null, 0, options)
-          root.traverse(viewHierarchy, options)
-
-          recorder.submitSafely(options, "screenshot_recorder.mask") {
-            val debugMasks = mutableListOf<Rect>()
-
-            val canvas = Canvas(screenshot)
-            canvas.setMatrix(prescaledMatrix)
-            viewHierarchy.traverse { node ->
-              if (node.shouldMask && (node.width > 0 && node.height > 0)) {
-                node.visibleRect ?: return@traverse false
-
-                // TODO: investigate why it returns true on RN when it shouldn't
-                //                                    if (viewHierarchy.isObscured(node)) {
-                //                                        return@traverse true
-                //                                    }
-
-                val (visibleRects, color) =
-                  when (node) {
-                    is ImageViewHierarchyNode -> {
-                      listOf(node.visibleRect) to screenshot.dominantColorForRect(node.visibleRect)
-                    }
-
-                    is TextViewHierarchyNode -> {
-                      val textColor =
-                        node.layout?.dominantTextColor ?: node.dominantColor ?: Color.BLACK
-                      node.layout.getVisibleRects(
-                        node.visibleRect,
-                        node.paddingLeft,
-                        node.paddingTop,
-                      ) to textColor
-                    }
-
-                    else -> {
-                      listOf(node.visibleRect) to Color.BLACK
-                    }
-                  }
-
-                maskingPaint.setColor(color)
-                visibleRects.forEach { rect ->
-                  canvas.drawRoundRect(RectF(rect), 10f, 10f, maskingPaint)
-                }
-                if (options.replayController.isDebugMaskingOverlayEnabled()) {
-                  debugMasks.addAll(visibleRects)
-                }
-              }
-              return@traverse true
-            }
-
-            if (options.replayController.isDebugMaskingOverlayEnabled()) {
-              mainLooperHandler.post {
-                if (debugOverlayDrawable.callback == null) {
-                  root.overlay.add(debugOverlayDrawable)
-                }
-                debugOverlayDrawable.updateMasks(debugMasks)
-                root.postInvalidate()
-              }
-            }
-            screenshotRecorderCallback?.onScreenshotRecorded(screenshot)
-            lastCaptureSuccessful.set(true)
-            contentChanged.set(false)
-          }
-        },
-        mainLooperHandler.handler,
-      )
+      screenshotStrategy.capture(root)
     } catch (e: Throwable) {
       options.logger.log(WARNING, "Failed to capture replay recording", e)
-      lastCaptureSuccessful.set(false)
     }
   }
 
@@ -199,6 +106,7 @@ internal class ScreenshotRecorder(
     }
 
     contentChanged.set(true)
+    screenshotStrategy.onContentChanged()
   }
 
   fun bind(root: View) {
@@ -212,6 +120,7 @@ internal class ScreenshotRecorder(
 
     // invalidate the flag to capture the first frame after new window is attached
     contentChanged.set(true)
+    screenshotStrategy.onContentChanged()
   }
 
   fun unbind(root: View?) {
@@ -235,29 +144,9 @@ internal class ScreenshotRecorder(
   fun close() {
     unbind(rootView?.get())
     rootView?.clear()
-    if (!screenshot.isRecycled) {
-      screenshot.recycle()
-    }
+    screenshotStrategy.close()
     isCapturing.set(false)
   }
-
-  private fun Bitmap.dominantColorForRect(rect: Rect): Int {
-    // TODO: maybe this ceremony can be just simplified to
-    // TODO: multiplying the visibleRect by the prescaledMatrix
-    val visibleRect = Rect(rect)
-    val visibleRectF = RectF(visibleRect)
-
-    // since we take screenshot with lower scale, we also
-    // have to apply the same scale to the visibleRect to get the
-    // correct screenshot part to determine the dominant color
-    prescaledMatrix.mapRect(visibleRectF)
-    // round it back to integer values, because drawBitmap below accepts Rect only
-    visibleRectF.round(visibleRect)
-    // draw part of the screenshot (visibleRect) to a single pixel bitmap
-    singlePixelBitmapCanvas.drawBitmap(this, visibleRect, Rect(0, 0, 1, 1), null)
-    // get the pixel color (= dominant color)
-    return singlePixelBitmap.getPixel(0, 0)
-  }
 }
 
 public data class ScreenshotRecorderConfig(