Skip to content

Commit ad4d00a

Browse files
0xnm0xnm
authored
Merge pull request #1678 from DataDog/nogorodnikov/rum-321/reduce-view-events-in-upload-pipeline
RUM-321: Introduce view event filtering in upload pipeline, remove view event throttling in write pipeline
2 parents 4744e9d + 4a3f24c commit ad4d00a

28 files changed

Lines changed: 759 additions & 555 deletions

‎detekt_custom.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -792,6 +792,7 @@ datadog:
792792
- "kotlin.collections.MutableMap.clear()"
793793
- "kotlin.collections.MutableMap.containsKey(android.view.Window)"
794794
- "kotlin.collections.MutableMap.containsKey(com.datadog.android.api.SdkCore)"
795+
- "kotlin.collections.MutableMap.containsKey(com.datadog.android.api.storage.RawBatchEvent)"
795796
- "kotlin.collections.MutableMap.containsKey(kotlin.Long)"
796797
- "kotlin.collections.MutableMap.containsKey(kotlin.String)"
797798
- "kotlin.collections.MutableMap.filterKeys(kotlin.Function1)"

‎features/dd-sdk-android-rum/src/main/kotlin/com/datadog/android/rum/internal/RumFeature.kt‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,10 @@ import com.datadog.android.rum.internal.anr.ANRDetectorRunnable
3535
import com.datadog.android.rum.internal.debug.UiRumDebugListener
3636
import com.datadog.android.rum.internal.domain.RumDataWriter
3737
import com.datadog.android.rum.internal.domain.event.RumEventMapper
38+
import com.datadog.android.rum.internal.domain.event.RumEventMetaDeserializer
39+
import com.datadog.android.rum.internal.domain.event.RumEventMetaSerializer
3840
import com.datadog.android.rum.internal.domain.event.RumEventSerializer
41+
import com.datadog.android.rum.internal.domain.event.RumViewEventFilter
3942
import com.datadog.android.rum.internal.instrumentation.MainLooperLongTaskStrategy
4043
import com.datadog.android.rum.internal.instrumentation.UserActionTrackingStrategyApi29
4144
import com.datadog.android.rum.internal.instrumentation.UserActionTrackingStrategyLegacy
@@ -182,7 +185,13 @@ internal class RumFeature constructor(
182185
}
183186

184187
override val requestFactory: RequestFactory by lazy {
185-
RumRequestFactory(configuration.customEndpointUrl, sdkCore.internalLogger)
188+
RumRequestFactory(
189+
customEndpointUrl = configuration.customEndpointUrl,
190+
viewEventFilter = RumViewEventFilter(
191+
eventMetaDeserializer = RumEventMetaDeserializer(sdkCore.internalLogger)
192+
),
193+
internalLogger = sdkCore.internalLogger
194+
)
186195
}
187196

188197
override val storageConfiguration: FeatureStorageConfiguration =
@@ -217,7 +226,7 @@ internal class RumFeature constructor(
217226
sdkCore: InternalSdkCore
218227
): DataWriter<Any> {
219228
return RumDataWriter(
220-
serializer = MapperSerializer(
229+
eventSerializer = MapperSerializer(
221230
RumEventMapper(
222231
sdkCore,
223232
viewEventMapper = configuration.viewEventMapper,
@@ -230,6 +239,7 @@ internal class RumFeature constructor(
230239
),
231240
RumEventSerializer(sdkCore.internalLogger)
232241
),
242+
eventMetaSerializer = RumEventMetaSerializer(),
233243
sdkCore = sdkCore
234244
)
235245
}

‎features/dd-sdk-android-rum/src/main/kotlin/com/datadog/android/rum/internal/domain/RumDataWriter.kt‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import com.datadog.android.core.InternalSdkCore
1414
import com.datadog.android.core.persistence.Serializer
1515
import com.datadog.android.core.persistence.serializeToByteArray
1616
import com.datadog.android.rum.GlobalRumMonitor
17+
import com.datadog.android.rum.internal.domain.event.RumEventMeta
1718
import com.datadog.android.rum.internal.monitor.AdvancedRumMonitor
1819
import com.datadog.android.rum.internal.monitor.StorageEvent
1920
import com.datadog.android.rum.model.ActionEvent
@@ -23,21 +24,38 @@ import com.datadog.android.rum.model.ResourceEvent
2324
import com.datadog.android.rum.model.ViewEvent
2425

2526
internal class RumDataWriter(
26-
internal val serializer: Serializer<Any>,
27+
internal val eventSerializer: Serializer<Any>,
28+
private val eventMetaSerializer: Serializer<RumEventMeta>,
2729
private val sdkCore: InternalSdkCore
2830
) : DataWriter<Any> {
2931

3032
// region DataWriter
3133

3234
@WorkerThread
3335
override fun write(writer: EventBatchWriter, element: Any): Boolean {
34-
val byteArray = serializer.serializeToByteArray(
36+
val byteArray = eventSerializer.serializeToByteArray(
3537
element,
3638
sdkCore.internalLogger
3739
) ?: return false
3840

41+
val batchEvent = if (element is ViewEvent) {
42+
val eventMeta = RumEventMeta.View(
43+
viewId = element.view.id,
44+
documentVersion = element.dd.documentVersion
45+
)
46+
val serializedEventMeta =
47+
eventMetaSerializer.serializeToByteArray(eventMeta, sdkCore.internalLogger)
48+
?: EMPTY_BYTE_ARRAY
49+
RawBatchEvent(
50+
data = byteArray,
51+
metadata = serializedEventMeta
52+
)
53+
} else {
54+
RawBatchEvent(data = byteArray)
55+
}
56+
3957
synchronized(this) {
40-
val result = writer.write(RawBatchEvent(data = byteArray), null)
58+
val result = writer.write(batchEvent, null)
4159
if (result) {
4260
onDataWritten(element, byteArray)
4361
}
@@ -86,4 +104,8 @@ internal class RumDataWriter(
86104
}
87105

88106
// endregion
107+
108+
companion object {
109+
val EMPTY_BYTE_ARRAY = ByteArray(0)
110+
}
89111
}

‎features/dd-sdk-android-rum/src/main/kotlin/com/datadog/android/rum/internal/domain/event/RumEventMeta.kt‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
/*
2+
* Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
3+
* This product includes software developed at Datadog (https://www.datadoghq.com/).
4+
* Copyright 2016-Present Datadog, Inc.
5+
*/
6+
7+
package com.datadog.android.rum.internal.domain.event
8+
9+
import com.datadog.android.api.InternalLogger
10+
import com.google.gson.JsonObject
11+
import com.google.gson.JsonParseException
12+
import com.google.gson.JsonParser
13+
import java.util.Locale
14+
import kotlin.jvm.Throws
15+
16+
internal sealed class RumEventMeta {
17+
18+
abstract val type: String
19+
20+
open fun toJson(): JsonObject {
21+
val model = JsonObject()
22+
23+
model.addProperty(TYPE_KEY, type)
24+
25+
return model
26+
}
27+
28+
data class View(
29+
val viewId: String,
30+
val documentVersion: Long
31+
) : RumEventMeta() {
32+
33+
override val type: String = VIEW_TYPE_VALUE
34+
35+
override fun toJson(): JsonObject {
36+
val model = super.toJson()
37+
38+
model.addProperty(VIEW_ID_KEY, viewId)
39+
model.addProperty(DOCUMENT_VERSION_KEY, documentVersion)
40+
41+
return model
42+
}
43+
}
44+
45+
companion object {
46+
47+
private const val UNKNOWN_RUM_EVENT_META_TYPE_ERROR = "Unknown RUM event meta type value [%s]"
48+
private const val UNABLE_TO_PARSE_JSON_INTO_META = "Unable to parse json into RUM event meta"
49+
50+
const val TYPE_KEY = "type"
51+
const val VIEW_TYPE_VALUE = "view"
52+
const val VIEW_ID_KEY = "viewId"
53+
const val DOCUMENT_VERSION_KEY = "documentVersion"
54+
55+
@Suppress("ThrowsCount", "ThrowingInternalException")
56+
@Throws(JsonParseException::class)
57+
fun fromJson(jsonString: String, internalLogger: InternalLogger): RumEventMeta? {
58+
return try {
59+
@Suppress("UnsafeThirdPartyFunctionCall") // JsonParseException is handled by the caller
60+
val model = JsonParser.parseString(jsonString).asJsonObject
61+
when (val type = model.get(TYPE_KEY).asString) {
62+
VIEW_TYPE_VALUE -> {
63+
val viewId = model.get(VIEW_ID_KEY).asString
64+
val docVersion = model.get(DOCUMENT_VERSION_KEY).asLong
65+
66+
View(viewId, docVersion)
67+
}
68+
69+
else -> {
70+
internalLogger.log(
71+
InternalLogger.Level.ERROR,
72+
InternalLogger.Target.USER,
73+
{ UNKNOWN_RUM_EVENT_META_TYPE_ERROR.format(Locale.US, type) }
74+
)
75+
null
76+
}
77+
}
78+
} catch (@Suppress("TooGenericExceptionCaught") e: NullPointerException) {
79+
throw JsonParseException(UNABLE_TO_PARSE_JSON_INTO_META, e)
80+
} catch (e: ClassCastException) {
81+
throw JsonParseException(UNABLE_TO_PARSE_JSON_INTO_META, e)
82+
} catch (e: IllegalStateException) {
83+
throw JsonParseException(UNABLE_TO_PARSE_JSON_INTO_META, e)
84+
} catch (e: NumberFormatException) {
85+
throw JsonParseException(UNABLE_TO_PARSE_JSON_INTO_META, e)
86+
}
87+
}
88+
}
89+
}

‎features/dd-sdk-android-rum/src/main/kotlin/com/datadog/android/rum/internal/domain/event/RumEventMetaDeserializer.kt‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/*
2+
* Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
3+
* This product includes software developed at Datadog (https://www.datadoghq.com/).
4+
* Copyright 2016-Present Datadog, Inc.
5+
*/
6+
7+
package com.datadog.android.rum.internal.domain.event
8+
9+
import com.datadog.android.api.InternalLogger
10+
import com.datadog.android.core.internal.persistence.Deserializer
11+
import com.google.gson.JsonParseException
12+
13+
internal class RumEventMetaDeserializer(
14+
private val internalLogger: InternalLogger
15+
) : Deserializer<ByteArray, RumEventMeta> {
16+
override fun deserialize(model: ByteArray): RumEventMeta? {
17+
if (model.isEmpty()) return null
18+
19+
return try {
20+
RumEventMeta.fromJson(String(model, Charsets.UTF_8), internalLogger)
21+
} catch (e: JsonParseException) {
22+
internalLogger.log(
23+
InternalLogger.Level.ERROR,
24+
InternalLogger.Target.USER,
25+
{ DESERIALIZATION_ERROR },
26+
e
27+
)
28+
null
29+
}
30+
}
31+
32+
companion object {
33+
const val DESERIALIZATION_ERROR = "Failed to deserialize RUM event meta"
34+
}
35+
}

‎features/dd-sdk-android-rum/src/main/kotlin/com/datadog/android/rum/internal/domain/event/RumEventMetaSerializer.kt‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
/*
2+
* Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
3+
* This product includes software developed at Datadog (https://www.datadoghq.com/).
4+
* Copyright 2016-Present Datadog, Inc.
5+
*/
6+
7+
package com.datadog.android.rum.internal.domain.event
8+
9+
import com.datadog.android.core.persistence.Serializer
10+
11+
internal class RumEventMetaSerializer : Serializer<RumEventMeta> {
12+
override fun serialize(model: RumEventMeta): String {
13+
return when (model) {
14+
is RumEventMeta.View -> model.toJson().toString()
15+
}
16+
}
17+
}

‎features/dd-sdk-android-rum/src/main/kotlin/com/datadog/android/rum/internal/domain/event/RumViewEventFilter.kt‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
/*
2+
* Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
3+
* This product includes software developed at Datadog (https://www.datadoghq.com/).
4+
* Copyright 2016-Present Datadog, Inc.
5+
*/
6+
7+
package com.datadog.android.rum.internal.domain.event
8+
9+
import com.datadog.android.api.storage.RawBatchEvent
10+
import com.datadog.android.core.internal.persistence.Deserializer
11+
import kotlin.math.max
12+
13+
internal class RumViewEventFilter(
14+
private val eventMetaDeserializer: Deserializer<ByteArray, RumEventMeta>
15+
) {
16+
17+
fun filterOutRedundantViewEvents(batch: List<RawBatchEvent>): List<RawBatchEvent> {
18+
val maxDocVersionByViewId = mutableMapOf<String, Long>()
19+
val viewMetaByEvent = mutableMapOf<RawBatchEvent, RumEventMeta.View>()
20+
21+
batch.forEach {
22+
val eventMeta = eventMetaDeserializer.deserialize(it.metadata)
23+
if (eventMeta is RumEventMeta.View) {
24+
viewMetaByEvent += it to eventMeta
25+
val viewId = eventMeta.viewId
26+
val documentVersion = eventMeta.documentVersion
27+
val maxDocVersionSeen = maxDocVersionByViewId[viewId]
28+
if (maxDocVersionSeen == null) {
29+
maxDocVersionByViewId[viewId] = documentVersion
30+
} else {
31+
maxDocVersionByViewId[viewId] = max(documentVersion, maxDocVersionSeen)
32+
}
33+
}
34+
}
35+
36+
return batch.filter {
37+
if (viewMetaByEvent.containsKey(it)) {
38+
@Suppress("UnsafeThirdPartyFunctionCall") // we checked the key before
39+
val viewMeta = viewMetaByEvent.getValue(it)
40+
// we need to leave only view event with a max doc version for a give viewId in the
41+
// batch, because backend will do the same during the reduce process
42+
@Suppress("UnsafeThirdPartyFunctionCall") // if there is a meta, there is a max doc version
43+
viewMeta.documentVersion == maxDocVersionByViewId.getValue(viewMeta.viewId)
44+
} else {
45+
true
46+
}
47+
}
48+
}
49+
}

‎features/dd-sdk-android-rum/src/main/kotlin/com/datadog/android/rum/internal/domain/scope/DefaultViewUpdatePredicate.kt‎

Lines changed: 0 additions & 32 deletions
This file was deleted.

‎features/dd-sdk-android-rum/src/main/kotlin/com/datadog/android/rum/internal/domain/scope/RumViewScope.kt‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,6 @@ internal open class RumViewScope(
5151
internal val cpuVitalMonitor: VitalMonitor,
5252
internal val memoryVitalMonitor: VitalMonitor,
5353
internal val frameRateVitalMonitor: VitalMonitor,
54-
private val viewUpdatePredicate: ViewUpdatePredicate = DefaultViewUpdatePredicate(),
5554
private val featuresContextResolver: FeaturesContextResolver = FeaturesContextResolver(),
5655
internal val type: RumViewType = RumViewType.FOREGROUND,
5756
private val trackFrustrations: Boolean,
@@ -653,9 +652,6 @@ internal open class RumViewScope(
653652
@Suppress("LongMethod", "ComplexMethod")
654653
private fun sendViewUpdate(event: RumRawEvent, writer: DataWriter<Any>) {
655654
val viewComplete = isViewComplete()
656-
if (!viewUpdatePredicate.canUpdateView(viewComplete, event)) {
657-
return
658-
}
659655
attributes.putAll(GlobalRumMonitor.get(sdkCore).getAttributes())
660656
version++
661657

‎features/dd-sdk-android-rum/src/main/kotlin/com/datadog/android/rum/internal/domain/scope/ViewUpdatePredicate.kt‎

Lines changed: 0 additions & 11 deletions
This file was deleted.

0 commit comments

Comments
 (0)