[cdc-base] Fix TM hangs caused by uncaught exception (#2511)

Jiabao-Sun · web-flow · commit 7172695471bc · 2023-09-21T16:59:56.000+08:00
diff --git a/flink-cdc-base/src/main/java/com/ververica/cdc/connectors/base/source/reader/external/IncrementalSourceScanFetcher.java b/flink-cdc-base/src/main/java/com/ververica/cdc/connectors/base/source/reader/external/IncrementalSourceScanFetcher.java
@@ -75,6 +75,8 @@ public IncrementalSourceScanFetcher(FetchTask.Context taskContext, int subtaskId
         ThreadFactory threadFactory =
                 new ThreadFactoryBuilder()
                         .setNameFormat("debezium-snapshot-reader-" + subtaskId)
+                        .setUncaughtExceptionHandler(
+                                (thread, throwable) -> setReadException(throwable))
                         .build();
         this.executorService = Executors.newSingleThreadExecutor(threadFactory);
         this.hasNextElement = new AtomicBoolean(false);
@@ -89,17 +91,13 @@ public void submitTask(FetchTask<SourceSplitBase> fetchTask) {
         this.queue = taskContext.getQueue();
         this.hasNextElement.set(true);
         this.reachEnd.set(false);
-        executorService.submit(
+
+        executorService.execute(
                 () -> {
                     try {
                         snapshotSplitReadTask.execute(taskContext);
                     } catch (Exception e) {
-                        LOG.error(
-                                String.format(
-                                        "Execute snapshot read task for snapshot split %s fail",
-                                        currentSnapshotSplit),
-                                e);
-                        readException = e;
+                        setReadException(e);
                     }
                 });
     }
@@ -186,6 +184,19 @@ private void checkReadException() {
         }
     }
 
+    private void setReadException(Throwable throwable) {
+        LOG.error(
+                String.format(
+                        "Execute snapshot read task for snapshot split %s fail",
+                        currentSnapshotSplit),
+                throwable);
+        if (readException == null) {
+            readException = throwable;
+        } else {
+            readException.addSuppressed(throwable);
+        }
+    }
+
     @Override
     public void close() {
         try {
