Fix ConcurrentModificationException when flushing async log messages

stalar · stalar · commit 18f5da992540 · 2017-07-03T11:49:47.000+02:00
A missing synchronized statement when adding to pendingWrites cause
ConcurrentModificationExceptions in the flush() method when the
pendingWrites are copied to writesToFlush.
diff --git a/google-cloud-logging/src/main/java/com/google/cloud/logging/LoggingImpl.java b/google-cloud-logging/src/main/java/com/google/cloud/logging/LoggingImpl.java
@@ -580,7 +580,9 @@ public void onFailure(Throwable t) {
                 }
               }
             });
-        pendingWrites.add(writeFuture);
+        synchronized (writeLock) {
+          pendingWrites.add(writeFuture);
+        }
         break;
     }
   }
diff --git a/google-cloud-logging/src/test/java/com/google/cloud/logging/LoggingImplTest.java b/google-cloud-logging/src/test/java/com/google/cloud/logging/LoggingImplTest.java
@@ -70,6 +70,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicInteger;
 import org.easymock.EasyMock;
 import org.junit.After;
 import org.junit.Before;
@@ -1448,5 +1449,44 @@ public void run() {
     flushWaiter.join(1000);
     assertFalse(flushWaiter.isAlive());
   }
+
+  @Test
+  public void testFlushStress() throws InterruptedException {
+    SettableApiFuture<WriteLogEntriesResponse> mockRpcResponse = SettableApiFuture.create();
+    mockRpcResponse.set(null);
+    replay(rpcFactoryMock);
+    logging = options.getService();
+    WriteLogEntriesRequest request = WriteLogEntriesRequest.newBuilder()
+        .addAllEntries(Iterables.transform(ImmutableList.of(LOG_ENTRY1),
+            LogEntry.toPbFunction(PROJECT)))
+        .build();
+
+    Thread[] threads = new Thread[100];
+    EasyMock.expect(loggingRpcMock.write(request)).andReturn(mockRpcResponse).times(threads.length);
+    EasyMock.replay(loggingRpcMock);
+
+    // log and flush concurrently in many threads to trigger a ConcurrentModificationException
+    final AtomicInteger exceptions = new AtomicInteger(0);
+    for (int i = 0; i < threads.length; i++) {
+      threads[i] = new Thread() {
+        @Override
+        public void run() {
+          try {
+            logging.write(ImmutableList.of(LOG_ENTRY1));
+            logging.flush();
+          } catch (Exception ex) {
+            ex.printStackTrace();
+            exceptions.incrementAndGet();
+          }
+        }
+      };
+      threads[i].start();
+    }
+    for (int i = 0; i < threads.length; i++) {
+      threads[i].join();
+    }
+    assertTrue(exceptions.get() == 0);
+  }
+
 }
 

Original file line number	Diff line number	Diff line change
`@@ -580,7 +580,9 @@ public void onFailure(Throwable t) {`
`580`	`580`	`}`
`581`	`581`	`}`
`582`	`582`	`});`
`583`		`- pendingWrites.add(writeFuture);`
	`583`	`+ synchronized (writeLock) {`
	`584`	`+ pendingWrites.add(writeFuture);`
	`585`	`+ }`
`584`	`586`	`break;`
`585`	`587`	`}`
`586`	`588`	`}`