Add logs for creation/completion of datadog spark listener (#6943)

paul-laffon-dd · web-flow · commit 16c56bf74f74 · 2024-04-30T19:56:15.000+02:00
Easier troubleshooting to validate that everything is running as expected
diff --git a/dd-java-agent/instrumentation/spark/src/main/java/datadog/trace/instrumentation/spark/AbstractDatadogSparkListener.java b/dd-java-agent/instrumentation/spark/src/main/java/datadog/trace/instrumentation/spark/AbstractDatadogSparkListener.java
@@ -41,6 +41,8 @@
 import org.apache.spark.sql.streaming.StateOperatorProgress;
 import org.apache.spark.sql.streaming.StreamingQueryListener;
 import org.apache.spark.sql.streaming.StreamingQueryProgress;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import scala.Tuple2;
 import scala.collection.JavaConverters;
 
@@ -53,6 +55,7 @@
  * still needed
  */
 public abstract class AbstractDatadogSparkListener extends SparkListener {
+  private static final Logger log = LoggerFactory.getLogger(AbstractDatadogSparkListener.class);
   public static volatile AbstractDatadogSparkListener listener = null;
   public static volatile boolean finishTraceOnApplicationEnd = true;
   public static volatile boolean isPysparkShell = false;
@@ -121,6 +124,8 @@ public AbstractDatadogSparkListener(SparkConf sparkConf, String appId, String sp
     databricksClusterName = sparkConf.get("spark.databricks.clusterUsageTags.clusterName", null);
     databricksServiceName = getDatabricksServiceName(sparkConf, databricksClusterName);
     sparkServiceName = getSparkServiceName(sparkConf, isRunningOnDatabricks);
+
+    log.info("Created datadog spark listener: {}", this.getClass().getSimpleName());
   }
 
   /** Resource name of the spark job. Provide an implementation based on a specific scala version */
@@ -173,13 +178,19 @@ private void initApplicationSpanIfNotInitialized() {
 
   @Override
   public void onApplicationEnd(SparkListenerApplicationEnd applicationEnd) {
+    log.info(
+        "Received spark application end event, finish trace on this event: {}",
+        finishTraceOnApplicationEnd);
+
     if (finishTraceOnApplicationEnd) {
       finishApplication(applicationEnd.time(), null, 0, null);
     }
   }
 
   public synchronized void finishApplication(
       long time, Throwable throwable, int exitCode, String msg) {
+    log.info("Finishing spark application trace");
+
     if (applicationEnded) {
       return;
     }
