Skip to content

RUM-936: Serialize TextViews/Buttons to base64 #1592

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jonathanmos merged 1 commit into from
Sep 4, 2023
Merged

RUM-936: Serialize TextViews/Buttons to base64 #1592

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions features/dd-sdk-android-session-replay/api/apiSurface
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,9 @@ class com.datadog.android.sessionreplay.internal.recorder.mapper.MaskInputTextVi
constructor()
class com.datadog.android.sessionreplay.internal.recorder.mapper.MaskTextViewMapper : TextViewMapper
constructor()
open class com.datadog.android.sessionreplay.internal.recorder.mapper.TextViewMapper : BaseWireframeMapper<android.widget.TextView, com.datadog.android.sessionreplay.model.MobileSegment.Wireframe.TextWireframe>
open class com.datadog.android.sessionreplay.internal.recorder.mapper.TextViewMapper : BaseWireframeMapper<android.widget.TextView, com.datadog.android.sessionreplay.model.MobileSegment.Wireframe>
constructor()
override fun map(android.widget.TextView, com.datadog.android.sessionreplay.internal.recorder.MappingContext): List<com.datadog.android.sessionreplay.model.MobileSegment.Wireframe.TextWireframe>
override fun map(android.widget.TextView, com.datadog.android.sessionreplay.internal.recorder.MappingContext): List<com.datadog.android.sessionreplay.model.MobileSegment.Wireframe>
interface com.datadog.android.sessionreplay.internal.recorder.mapper.WireframeMapper<T: android.view.View, S: com.datadog.android.sessionreplay.model.MobileSegment.Wireframe>
fun map(T, com.datadog.android.sessionreplay.internal.recorder.MappingContext): List<S>
object com.datadog.android.sessionreplay.utils.StringUtils
Expand Down
18 changes: 15 additions & 3 deletions ...-session-replay/src/main/kotlin/com/datadog/android/sessionreplay/SessionReplayPrivacy.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,11 @@ enum class SessionReplayPrivacy {
when (this) {
ALLOW -> {
imageMapper = ViewScreenshotWireframeMapper(viewWireframeMapper)
textMapper = TextViewMapper()
textMapper = TextViewMapper(
base64Serializer = base64Serializer,
imageWireframeHelper = imageWireframeHelper,
uniqueIdentifierGenerator = uniqueIdentifierGenerator
)
buttonMapper = ButtonMapper(textMapper)
editTextViewMapper = EditTextViewMapper(textMapper)
checkedTextViewMapper = CheckedTextViewMapper(textMapper)
Expand All @@ -118,7 +122,11 @@ enum class SessionReplayPrivacy {
}
MASK -> {
imageMapper = ViewScreenshotWireframeMapper(viewWireframeMapper)
textMapper = MaskTextViewMapper()
textMapper = MaskTextViewMapper(
base64Serializer = base64Serializer,
imageWireframeHelper = imageWireframeHelper,
uniqueIdentifierGenerator = uniqueIdentifierGenerator
)
buttonMapper = ButtonMapper(textMapper)
editTextViewMapper = EditTextViewMapper(textMapper)
checkedTextViewMapper = MaskCheckedTextViewMapper(textMapper)
Expand All @@ -130,7 +138,11 @@ enum class SessionReplayPrivacy {
}
MASK_USER_INPUT -> {
imageMapper = ViewScreenshotWireframeMapper(viewWireframeMapper)
textMapper = MaskInputTextViewMapper()
textMapper = MaskInputTextViewMapper(
base64Serializer = base64Serializer,
imageWireframeHelper = imageWireframeHelper,
uniqueIdentifierGenerator = uniqueIdentifierGenerator
)
buttonMapper = ButtonMapper(textMapper)
editTextViewMapper = EditTextViewMapper(textMapper)
checkedTextViewMapper = MaskCheckedTextViewMapper(textMapper)
Expand Down
73 changes: 73 additions & 0 deletions .../src/main/kotlin/com/datadog/android/sessionreplay/internal/recorder/ViewUtilsInternal.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,12 @@

package com.datadog.android.sessionreplay.internal.recorder

import android.graphics.drawable.Drawable
import android.view.View
import android.view.ViewStub
import androidx.appcompat.widget.ActionBarContextView
import androidx.appcompat.widget.Toolbar
import com.datadog.android.sessionreplay.internal.recorder.base64.ImageWireframeHelper

internal class ViewUtilsInternal {

Expand All @@ -31,4 +33,75 @@ internal class ViewUtilsInternal {
return Toolbar::class.java.isAssignableFrom(view::class.java) ||
android.widget.Toolbar::class.java.isAssignableFrom(view::class.java)
}

internal fun resolveDrawableBounds(
view: View,
drawable: Drawable,
pixelsDensity: Float
): GlobalBounds {
val coordinates = IntArray(2)
// this will always have size >= 2
@Suppress("UnsafeThirdPartyFunctionCall")
view.getLocationOnScreen(coordinates)
val x = coordinates[0].densityNormalized(pixelsDensity).toLong()
val y = coordinates[1].densityNormalized(pixelsDensity).toLong()
val width = drawable.intrinsicWidth.densityNormalized(pixelsDensity).toLong()
val height = drawable.intrinsicHeight.densityNormalized(pixelsDensity).toLong()
return GlobalBounds(x = x, y = y, height = height, width = width)
}

internal fun resolveCompoundDrawableBounds(
view: View,
drawable: Drawable,
pixelsDensity: Float,
position: ImageWireframeHelper.CompoundDrawablePositions
): GlobalBounds {
val coordinates = IntArray(2)
// this will always have size >= 2
@Suppress("UnsafeThirdPartyFunctionCall")
view.getLocationOnScreen(coordinates)

val viewXPosition = coordinates[0].densityNormalized(pixelsDensity).toLong()
val viewYPosition = coordinates[1].densityNormalized(pixelsDensity).toLong()
val drawableWidth = drawable.intrinsicWidth.densityNormalized(pixelsDensity).toLong()
val drawableHeight = drawable.intrinsicHeight.densityNormalized(pixelsDensity).toLong()
val viewWidth = view.width.densityNormalized(pixelsDensity).toLong()
val viewHeight = view.height.densityNormalized(pixelsDensity).toLong()
val viewPaddingStart = view.paddingStart.densityNormalized(pixelsDensity).toLong()
val viewPaddingTop = view.paddingTop.densityNormalized(pixelsDensity).toLong()
val viewPaddingBottom = view.paddingBottom.densityNormalized(pixelsDensity).toLong()
val viewPaddingEnd = view.paddingEnd.densityNormalized(pixelsDensity).toLong()
var xPosition: Long
var yPosition: Long

when (position) {
ImageWireframeHelper.CompoundDrawablePositions.LEFT -> {
xPosition = viewPaddingStart
yPosition = getCenterVerticalOffset(viewHeight, drawableHeight)
}
ImageWireframeHelper.CompoundDrawablePositions.TOP -> {
xPosition = getCenterHorizontalOffset(viewWidth, drawableWidth)
yPosition = viewPaddingTop
}
ImageWireframeHelper.CompoundDrawablePositions.RIGHT -> {
xPosition = viewWidth - (drawableWidth + viewPaddingEnd)
yPosition = getCenterVerticalOffset(viewHeight, drawableHeight)
}
ImageWireframeHelper.CompoundDrawablePositions.BOTTOM -> {
xPosition = getCenterHorizontalOffset(viewWidth, drawableWidth)
yPosition = viewHeight - (drawableHeight + viewPaddingBottom)
}
}

xPosition += viewXPosition
yPosition += viewYPosition

return GlobalBounds(x = xPosition, y = yPosition, height = drawableHeight, width = drawableWidth)
}

private fun getCenterHorizontalOffset(viewWidth: Long, drawableWidth: Long): Long =
(viewWidth / 2) - (drawableWidth / 2)

private fun getCenterVerticalOffset(viewHeight: Long, drawableHeight: Long): Long =
(viewHeight / 2) - (drawableHeight / 2)
}
94 changes: 88 additions & 6 deletions ...kotlin/com/datadog/android/sessionreplay/internal/recorder/base64/ImageWireframeHelper.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,19 +8,25 @@ package com.datadog.android.sessionreplay.internal.recorder.base64

import android.graphics.drawable.Drawable
import android.view.View
import android.widget.TextView
import androidx.annotation.MainThread
import androidx.annotation.VisibleForTesting
import com.datadog.android.sessionreplay.internal.recorder.MappingContext
import com.datadog.android.sessionreplay.internal.recorder.ViewUtilsInternal
import com.datadog.android.sessionreplay.internal.recorder.densityNormalized
import com.datadog.android.sessionreplay.model.MobileSegment
import com.datadog.android.sessionreplay.utils.UniqueIdentifierGenerator

internal class ImageWireframeHelper(
private val imageCompression: ImageCompression = WebPImageCompression(),
private val uniqueIdentifierGenerator: UniqueIdentifierGenerator = UniqueIdentifierGenerator,
private val base64Serializer: Base64Serializer
private val base64Serializer: Base64Serializer,
private val viewUtilsInternal: ViewUtilsInternal = ViewUtilsInternal()
) {
@MainThread
internal fun createImageWireframe(
view: View,
index: Int,
currentWireframeIndex: Int,
x: Long,
y: Long,
width: Long,
Expand All @@ -30,10 +36,15 @@ internal class ImageWireframeHelper(
border: MobileSegment.ShapeBorder? = null,
prefix: String = DRAWABLE_CHILD_NAME
): MobileSegment.Wireframe.ImageWireframe? {
val id = uniqueIdentifierGenerator.resolveChildUniqueIdentifier(view, prefix + index)
val id = uniqueIdentifierGenerator.resolveChildUniqueIdentifier(view, prefix + currentWireframeIndex)

@Suppress("ComplexCondition")
if (drawable == null || id == null || drawable.intrinsicWidth < 0 || drawable.intrinsicHeight < 0) {
if (
drawable == null ||
id == null ||
drawable.intrinsicWidth <= 0 ||
drawable.intrinsicHeight <= 0
) {
return null
}

Expand Down Expand Up @@ -65,7 +76,78 @@ internal class ImageWireframeHelper(
return imageWireframe
}

private companion object {
private const val DRAWABLE_CHILD_NAME = "drawable"
@Suppress("NestedBlockDepth")

@mariusc83 mariusc83 Sep 4, 2023

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

much much better this way, tks !!

internal fun createCompoundDrawableWireframes(
view: TextView,
mappingContext: MappingContext,
prevWireframeIndex: Int
): MutableList<MobileSegment.Wireframe> {
val result = mutableListOf<MobileSegment.Wireframe>()
var wireframeIndex = prevWireframeIndex
val density = mappingContext.systemInformation.screenDensity

// CompoundDrawables returns an array of indexes in the following order:
// left, top, right, bottom
view.compoundDrawables.forEachIndexed { compoundDrawableIndex, _ ->
if (compoundDrawableIndex > CompoundDrawablePositions.values().size) {
return@forEachIndexed
}

val compoundDrawablePosition = convertIndexToCompoundDrawablePosition(
Comment thread
jonathanmos marked this conversation as resolved.
compoundDrawableIndex
) ?: return@forEachIndexed

val drawable = view.compoundDrawables[compoundDrawableIndex]

if (drawable != null) {
val drawableCoordinates = viewUtilsInternal.resolveCompoundDrawableBounds(
view = view,
drawable = drawable,
pixelsDensity = density,
position = compoundDrawablePosition
)

@Suppress("ThreadSafety") // TODO REPLAY-1861 caller thread of .map is unknown?
createImageWireframe(
view = view,
currentWireframeIndex = ++wireframeIndex,
x = drawableCoordinates.x,
y = drawableCoordinates.y,
width = drawable.intrinsicWidth
.densityNormalized(density).toLong(),
height = drawable.intrinsicHeight
.densityNormalized(density).toLong(),
drawable = drawable,
shapeStyle = null,
border = null
)?.let { resultWireframe ->
result.add(resultWireframe)
}
}
}

return result
}

@Suppress("MagicNumber")
private fun convertIndexToCompoundDrawablePosition(compoundDrawableIndex: Int): CompoundDrawablePositions? {
return when (compoundDrawableIndex) {
0 -> CompoundDrawablePositions.LEFT
1 -> CompoundDrawablePositions.TOP
2 -> CompoundDrawablePositions.RIGHT
3 -> CompoundDrawablePositions.BOTTOM
else -> null
}
}

internal enum class CompoundDrawablePositions {
LEFT,
TOP,
RIGHT,
BOTTOM
}

internal companion object {
@VisibleForTesting internal const val DRAWABLE_CHILD_NAME = "drawable"
}
}
19 changes: 10 additions & 9 deletions ...rc/main/kotlin/com/datadog/android/sessionreplay/internal/recorder/mapper/ButtonMapper.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,18 @@ import com.datadog.android.sessionreplay.model.MobileSegment
internal class ButtonMapper(
private val textWireframeMapper: TextViewMapper = TextViewMapper()
) :
WireframeMapper<Button, MobileSegment.Wireframe.TextWireframe> {
WireframeMapper<Button, MobileSegment.Wireframe> {
override fun map(view: Button, mappingContext: MappingContext):
List<MobileSegment.Wireframe.TextWireframe> {
return textWireframeMapper.map(view, mappingContext).map {
if (it.shapeStyle == null && it.border == null) {
// we were not able to resolve the background for this button so just add a border
it.copy(border = MobileSegment.ShapeBorder(BLACK_COLOR, 1))
} else {
it
List<MobileSegment.Wireframe> {
return textWireframeMapper.map(view, mappingContext)
.map {
if (it is MobileSegment.Wireframe.TextWireframe && it.shapeStyle == null && it.border == null) {
// we were not able to resolve the background for this button so just add a border
it.copy(border = MobileSegment.ShapeBorder(BLACK_COLOR, 1))
} else {
it
}
}
}
}

companion object {
Expand Down
2 changes: 1 addition & 1 deletion ...in/kotlin/com/datadog/android/sessionreplay/internal/recorder/mapper/ImageButtonMapper.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ internal class ImageButtonMapper(
@Suppress("ThreadSafety") // TODO REPLAY-1861 caller thread of .map is unknown?
imageWireframeHelper.createImageWireframe(
view = view,
index = wireframes.size,
currentWireframeIndex = wireframes.size,
x = centerX,
y = centerY,
width = scaledDrawableWidth,
Expand Down
17 changes: 12 additions & 5 deletions ...lin/com/datadog/android/sessionreplay/internal/recorder/mapper/MaskInputTextViewMapper.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,11 @@
package com.datadog.android.sessionreplay.internal.recorder.mapper

import android.widget.TextView
import androidx.annotation.VisibleForTesting
import com.datadog.android.sessionreplay.SessionReplayPrivacy
import com.datadog.android.sessionreplay.internal.recorder.base64.Base64Serializer
import com.datadog.android.sessionreplay.internal.recorder.base64.ImageWireframeHelper
import com.datadog.android.sessionreplay.internal.recorder.obfuscator.rules.MaskInputObfuscationRule
import com.datadog.android.sessionreplay.internal.recorder.obfuscator.rules.TextValueObfuscationRule
import com.datadog.android.sessionreplay.utils.UniqueIdentifierGenerator

/**
* A [WireframeMapper] implementation to map a [TextView] component and apply the
Expand All @@ -19,8 +20,14 @@ import com.datadog.android.sessionreplay.internal.recorder.obfuscator.rules.Text
class MaskInputTextViewMapper : TextViewMapper {
constructor() : super(textValueObfuscationRule = MaskInputObfuscationRule())

@VisibleForTesting
internal constructor(
textValueObfuscationRule: TextValueObfuscationRule
) : super(textValueObfuscationRule)
base64Serializer: Base64Serializer,
imageWireframeHelper: ImageWireframeHelper,
uniqueIdentifierGenerator: UniqueIdentifierGenerator
) : super(
base64Serializer = base64Serializer,
imageWireframeHelper = imageWireframeHelper,
uniqueIdentifierGenerator = uniqueIdentifierGenerator,
textValueObfuscationRule = MaskInputObfuscationRule()
)
}
19 changes: 18 additions & 1 deletion ...n/kotlin/com/datadog/android/sessionreplay/internal/recorder/mapper/MaskTextViewMapper.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,11 @@ package com.datadog.android.sessionreplay.internal.recorder.mapper
import android.widget.TextView
import androidx.annotation.VisibleForTesting
import com.datadog.android.sessionreplay.SessionReplayPrivacy
import com.datadog.android.sessionreplay.internal.recorder.base64.Base64Serializer
import com.datadog.android.sessionreplay.internal.recorder.base64.ImageWireframeHelper
import com.datadog.android.sessionreplay.internal.recorder.obfuscator.rules.MaskObfuscationRule
import com.datadog.android.sessionreplay.internal.recorder.obfuscator.rules.TextValueObfuscationRule
import com.datadog.android.sessionreplay.utils.UniqueIdentifierGenerator

/**
* A [WireframeMapper] implementation to map a [TextView] component and apply the
Expand All @@ -19,8 +22,22 @@ import com.datadog.android.sessionreplay.internal.recorder.obfuscator.rules.Text
class MaskTextViewMapper : TextViewMapper {
constructor() : super(textValueObfuscationRule = MaskObfuscationRule())

internal constructor(
base64Serializer: Base64Serializer,
imageWireframeHelper: ImageWireframeHelper,
uniqueIdentifierGenerator: UniqueIdentifierGenerator
) : super(
base64Serializer = base64Serializer,
imageWireframeHelper = imageWireframeHelper,
uniqueIdentifierGenerator = uniqueIdentifierGenerator,
textValueObfuscationRule = MaskObfuscationRule()
)

@VisibleForTesting
internal constructor(
base64Serializer: Base64Serializer,
imageWireframeHelper: ImageWireframeHelper,
uniqueIdentifierGenerator: UniqueIdentifierGenerator,
textValueObfuscationRule: TextValueObfuscationRule
) : super(textValueObfuscationRule)
) : super(base64Serializer, imageWireframeHelper, uniqueIdentifierGenerator, textValueObfuscationRule)
}
Loading