Merge pull request #3203 from getsentry/rz/feat/session-replay-sources

[SR] Add screenshot recorder

buildSrc/src/main/java/Config.kt

@@ -34,7 +34,7 @@ object Config {
 
         val minSdkVersion = 19
         val minSdkVersionOkHttp = 21
-        val minSdkVersionReplay = 26
+        val minSdkVersionReplay = 19
         val minSdkVersionNdk = 19
         val minSdkVersionCompose = 21
         val targetSdkVersion = sdkVersion

sentry-android-replay/api/sentry-android-replay.api

@@ -0,0 +1,62 @@
+public final class io/sentry/android/replay/BuildConfig {
+	public static final field BUILD_TYPE Ljava/lang/String;
+	public static final field DEBUG Z
+	public static final field LIBRARY_PACKAGE_NAME Ljava/lang/String;
+	public static final field VERSION_NAME Ljava/lang/String;
+	public fun <init> ()V
+}
+
+public final class io/sentry/android/replay/WindowRecorder {
+	public fun <init> ()V
+	public final fun startRecording (Landroid/content/Context;)V
+	public final fun stopRecording ()V
+}
+
+public abstract interface class io/sentry/android/replay/video/SimpleFrameMuxer {
+	public abstract fun getVideoTime ()J
+	public abstract fun isStarted ()Z
+	public abstract fun muxVideoFrame (Ljava/nio/ByteBuffer;Landroid/media/MediaCodec$BufferInfo;)V
+	public abstract fun release ()V
+	public abstract fun start (Landroid/media/MediaFormat;)V
+}
+
+public final class io/sentry/android/replay/video/SimpleMp4FrameMuxer : io/sentry/android/replay/video/SimpleFrameMuxer {
+	public fun <init> (Ljava/lang/String;F)V
+	public fun getVideoTime ()J
+	public fun isStarted ()Z
+	public fun muxVideoFrame (Ljava/nio/ByteBuffer;Landroid/media/MediaCodec$BufferInfo;)V
+	public fun release ()V
+	public fun start (Landroid/media/MediaFormat;)V
+}
+
+public final class io/sentry/android/replay/viewhierarchy/ViewHierarchyNode {
+	public static final field Companion Lio/sentry/android/replay/viewhierarchy/ViewHierarchyNode$Companion;
+	public fun <init> (FFIIZLjava/lang/Integer;Landroid/graphics/Rect;)V
+	public synthetic fun <init> (FFIIZLjava/lang/Integer;Landroid/graphics/Rect;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
+	public final fun component1 ()F
+	public final fun component2 ()F
+	public final fun component3 ()I
+	public final fun component4 ()I
+	public final fun component5 ()Z
+	public final fun component6 ()Ljava/lang/Integer;
+	public final fun component7 ()Landroid/graphics/Rect;
+	public final fun copy (FFIIZLjava/lang/Integer;Landroid/graphics/Rect;)Lio/sentry/android/replay/viewhierarchy/ViewHierarchyNode;
+	public static synthetic fun copy$default (Lio/sentry/android/replay/viewhierarchy/ViewHierarchyNode;FFIIZLjava/lang/Integer;Landroid/graphics/Rect;ILjava/lang/Object;)Lio/sentry/android/replay/viewhierarchy/ViewHierarchyNode;
+	public fun equals (Ljava/lang/Object;)Z
+	public final fun getChildren ()Ljava/util/List;
+	public final fun getDominantColor ()Ljava/lang/Integer;
+	public final fun getHeight ()I
+	public final fun getShouldRedact ()Z
+	public final fun getVisibleRect ()Landroid/graphics/Rect;
+	public final fun getWidth ()I
+	public final fun getX ()F
+	public final fun getY ()F
+	public fun hashCode ()I
+	public final fun setChildren (Ljava/util/List;)V
+	public fun toString ()Ljava/lang/String;
+}
+
+public final class io/sentry/android/replay/viewhierarchy/ViewHierarchyNode$Companion {
+	public final fun fromView (Landroid/view/View;)Lio/sentry/android/replay/viewhierarchy/ViewHierarchyNode;
+}
+

sentry-android-replay/build.gradle.kts

@@ -7,7 +7,8 @@ plugins {
     jacoco
     id(Config.QualityPlugins.jacocoAndroid)
     id(Config.QualityPlugins.gradleVersions)
-    id(Config.QualityPlugins.detektPlugin)
+    // TODO: enable it later
+//    id(Config.QualityPlugins.detektPlugin)
 }
 
 android {

sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt

@@ -0,0 +1,190 @@
+package io.sentry.android.replay
+
+import android.annotation.TargetApi
+import android.graphics.Bitmap
+import android.graphics.Canvas
+import android.graphics.Matrix
+import android.graphics.Paint
+import android.graphics.Rect
+import android.graphics.RectF
+import android.os.Handler
+import android.os.HandlerThread
+import android.os.Looper
+import android.os.SystemClock
+import android.util.Log
+import android.view.PixelCopy
+import android.view.View
+import android.view.ViewGroup
+import android.view.ViewTreeObserver
+import io.sentry.android.replay.video.SimpleVideoEncoder
+import io.sentry.android.replay.viewhierarchy.ViewHierarchyNode
+import java.lang.ref.WeakReference
+import java.util.WeakHashMap
+import kotlin.system.measureTimeMillis
+
+// TODO: use ILogger of Sentry and change level
+@TargetApi(26)
+internal class ScreenshotRecorder(
+    val encoder: SimpleVideoEncoder
+) : ViewTreeObserver.OnDrawListener {
+
+    private var rootView: WeakReference<View>? = null
+    private val thread = HandlerThread("SentryReplay").also { it.start() }
+    private val handler = Handler(thread.looper)
+    private val bitmapToVH = WeakHashMap<Bitmap, ViewHierarchyNode>()
+    private val maskingPaint = Paint()
+    private val singlePixelBitmap: Bitmap = Bitmap.createBitmap(
+        1,
+        1,
+        Bitmap.Config.ARGB_8888
+    )
+    private val singlePixelBitmapCanvas: Canvas = Canvas(singlePixelBitmap)
+    private val prescaledMatrix = Matrix().apply {
+        preScale(encoder.muxerConfig.scaleFactor, encoder.muxerConfig.scaleFactor)
+    }
+
+    companion object {
+        const val TAG = "ScreenshotRecorder"
+    }
+
+    private var lastCapturedAtMs: Long? = null
+    override fun onDraw() {
+        // TODO: replace with Debouncer from sentry-core
+        val now = SystemClock.uptimeMillis()
+        if (lastCapturedAtMs != null && (now - lastCapturedAtMs!!) < 500L) {
+            return
+        }
+        lastCapturedAtMs = now
+
+        val root = rootView?.get()
+        if (root == null || root.width <= 0 || root.height <= 0 || !root.isShown) {
+            return
+        }
+
+        val window = root.phoneWindow ?: return
+        val bitmap = Bitmap.createBitmap(
+            root.width,
+            root.height,
+            Bitmap.Config.ARGB_8888
+        )
+
+        val time = measureTimeMillis {
+            val rootNode = ViewHierarchyNode.fromView(root)
+            root.traverse(rootNode)
+            bitmapToVH[bitmap] = rootNode
+        }
+        Log.e("TIME", time.toString())
+
+        // postAtFrontOfQueue to ensure the view hierarchy and bitmap are ase close in-sync as possible
+        Handler(Looper.getMainLooper()).postAtFrontOfQueue {
+            PixelCopy.request(
+                window,
+                bitmap,
+                { copyResult: Int ->
+                    Log.d(TAG, "PixelCopy result: $copyResult")
+                    if (copyResult != PixelCopy.SUCCESS) {
+                        Log.e(TAG, "Failed to capture screenshot")
+                        return@request
+                    }
+
+                    Log.e("BITMAP CAPTURED", bitmap.toString())
+                    val viewHierarchy = bitmapToVH[bitmap]
+
+                    var scaledBitmap: Bitmap? = null
+
+                    if (viewHierarchy == null) {
+                        Log.e(TAG, "Failed to determine view hierarchy, not capturing")
+                        return@request
+                    } else {
+                        scaledBitmap = Bitmap.createScaledBitmap(
+                            bitmap,
+                            encoder.muxerConfig.videoWidth,
+                            encoder.muxerConfig.videoHeight,
+                            true
+                        )
+                        val canvas = Canvas(scaledBitmap)
+                        canvas.setMatrix(prescaledMatrix)
+                        viewHierarchy.traverse {
+                            if (it.shouldRedact && (it.width > 0 && it.height > 0)) {
+                                it.visibleRect ?: return@traverse
+
+                                // TODO: check for view type rather than rely on absence of dominantColor here
+                                val color = if (it.dominantColor == null) {
+                                    singlePixelBitmapCanvas.drawBitmap(bitmap, it.visibleRect, Rect(0, 0, 1, 1), null)
+                                    singlePixelBitmap.getPixel(0, 0)
+                                } else {
+                                    it.dominantColor
+                                }
+
+                                maskingPaint.setColor(color)
+                                canvas.drawRoundRect(RectF(it.visibleRect), 10f, 10f, maskingPaint)
+                            }
+                        }
+                    }
+
+//        val baos = ByteArrayOutputStream()
+//        scaledBitmap.compress(Bitmap.CompressFormat.JPEG, 75, baos)
+//        val bmp = BitmapFactory.decodeByteArray(baos.toByteArray(), 0, baos.size())
+                    scaledBitmap?.let {
+                        encoder.encode(it)
+                        it.recycle()
+                    }
+//        bmp.recycle()
+                    bitmap.recycle()
+                    Log.i(TAG, "Captured a screenshot")
+                },
+                handler
+            )
+        }
+    }
+
+    fun bind(root: View) {
+        // first unbind the current root
+        unbind(rootView?.get())
+        rootView?.clear()
+
+        // next bind the new root
+        rootView = WeakReference(root)
+        root.viewTreeObserver?.addOnDrawListener(this)
+    }
+
+    fun unbind(root: View?) {
+        root?.viewTreeObserver?.removeOnDrawListener(this)
+    }
+
+    fun close() {
+        unbind(rootView?.get())
+        rootView?.clear()
+        thread.quitSafely()
+    }
+
+    private fun ViewHierarchyNode.traverse(callback: (ViewHierarchyNode) -> Unit) {
+        callback(this)
+        if (this.children != null) {
+            this.children!!.forEach {
+                it.traverse(callback)
+            }
+        }
+    }
+
+    private fun View.traverse(parentNode: ViewHierarchyNode) {
+        if (this !is ViewGroup) {
+            return
+        }
+
+        if (this.childCount == 0) {
+            return
+        }
+
+        val childNodes = ArrayList<ViewHierarchyNode>(this.childCount)
+        for (i in 0 until childCount) {
+            val child = getChildAt(i)
+            if (child != null) {
+                val childNode = ViewHierarchyNode.fromView(child)
+                childNodes.add(childNode)
+                child.traverse(childNode)
+            }
+        }
+        parentNode.children = childNodes
+    }
+}

sentry-android-replay/src/main/java/io/sentry/android/replay/WindowRecorder.kt

@@ -0,0 +1,94 @@
+package io.sentry.android.replay
+
+import android.annotation.TargetApi
+import android.content.Context
+import android.graphics.Point
+import android.os.Build.VERSION
+import android.os.Build.VERSION_CODES
+import android.view.View
+import android.view.WindowManager
+import io.sentry.android.replay.video.MuxerConfig
+import io.sentry.android.replay.video.SimpleVideoEncoder
+import java.io.File
+import java.lang.ref.WeakReference
+import java.util.concurrent.atomic.AtomicBoolean
+import kotlin.LazyThreadSafetyMode.NONE
+import kotlin.math.roundToInt
+
+@TargetApi(26)
+class WindowRecorder {
+
+    private val rootViewsSpy by lazy(NONE) {
+        RootViewsSpy.install()
+    }
+
+    private var encoder: SimpleVideoEncoder? = null
+    private val isRecording = AtomicBoolean(false)
+    private val rootViews = ArrayList<WeakReference<View>>()
+    private var recorder: ScreenshotRecorder? = null
+
+    private val onRootViewsChangedListener = OnRootViewsChangedListener { root, added ->
+        if (added) {
+            rootViews.add(WeakReference(root))
+            recorder?.bind(root)
+        } else {
+            recorder?.unbind(root)
+            rootViews.removeAll { it.get() == root }
+
+            val newRoot = rootViews.lastOrNull()?.get()
+            if (newRoot != null && root != newRoot) {
+                recorder?.bind(newRoot)
+            }
+        }
+    }
+
+    fun startRecording(context: Context) {
+        if (isRecording.getAndSet(true)) {
+            return
+        }
+
+        val wm = context.getSystemService(Context.WINDOW_SERVICE) as WindowManager
+//    val (height, width) = (wm.currentWindowMetrics.bounds.bottom /
+//        context.resources.displayMetrics.density).roundToInt() to
+//        (wm.currentWindowMetrics.bounds.right /
+//            context.resources.displayMetrics.density).roundToInt()
+        // TODO: API level check
+        // PixelCopy takes screenshots including system bars, so we have to get the real size here
+        val height: Int
+        val aspectRatio = if (VERSION.SDK_INT >= VERSION_CODES.R) {
+            height = wm.currentWindowMetrics.bounds.bottom
+            height.toFloat() / wm.currentWindowMetrics.bounds.right.toFloat()
+        } else {
+            val screenResolution = Point()
+            @Suppress("DEPRECATION")
+            wm.defaultDisplay.getRealSize(screenResolution)
+            height = screenResolution.y
+            height.toFloat() / screenResolution.x.toFloat()
+        }
+
+        val videoFile = File(context.cacheDir, "sentry-sr.mp4")
+        encoder = SimpleVideoEncoder(
+            MuxerConfig(
+                videoFile,
+                videoWidth = (720 / aspectRatio).roundToInt(),
+                videoHeight = 720,
+                scaleFactor = 720f / height,
+                frameRate = 2f,
+                bitrate = 500 * 1000
+            )
+        ).also { it.start() }
+        recorder = ScreenshotRecorder(encoder!!)
+        rootViewsSpy.listeners += onRootViewsChangedListener
+    }
+
+    fun stopRecording() {
+        rootViewsSpy.listeners -= onRootViewsChangedListener
+        rootViews.forEach { recorder?.unbind(it.get()) }
+        recorder?.close()
+        rootViews.clear()
+        recorder = null
+        encoder?.startRelease()
+        encoder = null
+        isRecording.set(false)
+    }
+}