Skip to content

Commit eb0c089

Browse files
jean-philippe-martinchingor13
jean-philippe-martin
authored and
chingor13
committed
---
yaml --- r: 11655 b: refs/heads/autosynth-pubsub c: b43b713 h: refs/heads/master i: 11653: a344062 11651: dffaa96 11647: fb83105
1 parent 03ef968 commit eb0c089

4 files changed

Lines changed: 89 additions & 11 deletions

File tree

‎[refs]‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ refs/heads/autosynth-compute: fa508ebab5b6a1db14949441c8de868cefbcf1a6
116116
refs/heads/autosynth-container: d0346e84b1f26e3dc10444450a998f357a43bcef
117117
refs/heads/autosynth-dataproc: c0f21dd811166848a60f810565a5614bb9c7f47e
118118
refs/heads/autosynth-monitoring: 28bf7ca67c0aab4815884e5d6253aaa8aa3baa1c
119-
refs/heads/autosynth-pubsub: 95d46417468f36a807aae83484e57a1efb6bd041
119+
refs/heads/autosynth-pubsub: b43b713ee426e71b9ddc7575dad12b191732bdd1
120120
refs/heads/autosynth-video-intelligence: 06fb1841ae847f4ef79a62ba995efbabc213edf8
121121
refs/heads/autosynth-vision: 6bece1ee6c1c41d5822c72bf0d6a82484166b402
122122
refs/heads/spanner: 54a5e197bfe0a004e13c190427f46c3413ab572d

‎branches/autosynth-pubsub/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/it/ITGcsNio.java‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -127,9 +127,13 @@ public static void beforeClass() throws IOException {
127127

128128
@AfterClass
129129
public static void afterClass() throws ExecutionException, InterruptedException {
130-
if (storage != null && !RemoteStorageHelper.forceDelete(storage, BUCKET, 5, TimeUnit.SECONDS) &&
131-
log.isLoggable(Level.WARNING)) {
132-
log.log(Level.WARNING, "Deletion of bucket {0} timed out, bucket is not empty", BUCKET);
130+
if (storage != null) {
131+
for (String bucket : new String[]{BUCKET, REQUESTER_PAYS_BUCKET}) {
132+
if (!RemoteStorageHelper.forceDelete(storage, bucket, 5, TimeUnit.SECONDS, project) &&
133+
log.isLoggable(Level.WARNING)) {
134+
log.log(Level.WARNING, "Deletion of bucket {0} timed out, bucket is not empty", bucket);
135+
}
136+
}
133137
}
134138
}
135139

‎branches/autosynth-pubsub/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/RemoteStorageHelper.java‎

Lines changed: 56 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import java.io.IOException;
3232
import java.io.InputStream;
3333
import java.util.ArrayList;
34+
import java.util.List;
3435
import java.util.UUID;
3536
import java.util.concurrent.Callable;
3637
import java.util.concurrent.ExecutionException;
@@ -41,6 +42,8 @@
4142
import java.util.concurrent.TimeoutException;
4243
import java.util.logging.Level;
4344
import java.util.logging.Logger;
45+
46+
import com.google.common.base.Strings;
4447
import org.threeten.bp.Duration;
4548

4649
/**
@@ -121,8 +124,29 @@ public void run() {
121124
*/
122125
public static Boolean forceDelete(Storage storage, String bucket, long timeout, TimeUnit unit)
123126
throws InterruptedException, ExecutionException {
127+
return forceDelete(storage, bucket, timeout, unit, "");
128+
}
129+
130+
/**
131+
* Deletes a bucket, even if non-empty. Objects in the bucket are listed and deleted until bucket
132+
* deletion succeeds or {@code timeout} expires. To allow for the timeout, this method uses a
133+
* separate thread to send the delete requests. Use
134+
* {@link #forceDelete(Storage storage, String bucket)} if spawning an additional thread is
135+
* undesirable, such as in the App Engine production runtime.
136+
*
137+
* @param storage the storage service to be used to issue requests
138+
* @param bucket the bucket to be deleted
139+
* @param timeout the maximum time to wait
140+
* @param unit the time unit of the timeout argument
141+
* @param userProject the project to bill for requester-pays buckets (or "")
142+
* @return true if deletion succeeded, false if timeout expired
143+
* @throws InterruptedException if the thread deleting the bucket is interrupted while waiting
144+
* @throws ExecutionException if an exception was thrown while deleting bucket or bucket objects
145+
*/
146+
public static Boolean forceDelete(Storage storage, String bucket, long timeout, TimeUnit unit, String userProject)
147+
throws InterruptedException, ExecutionException {
124148
ExecutorService executor = Executors.newSingleThreadExecutor();
125-
Future<Boolean> future = executor.submit(new DeleteBucketTask(storage, bucket));
149+
Future<Boolean> future = executor.submit(new DeleteBucketTask(storage, bucket, userProject));
126150
try {
127151
return future.get(timeout, unit);
128152
} catch (TimeoutException ex) {
@@ -210,26 +234,52 @@ private static RetrySettings retrySettings() {
210234

211235
private static class DeleteBucketTask implements Callable<Boolean> {
212236

213-
private Storage storage;
214-
private String bucket;
237+
private final Storage storage;
238+
private final String bucket;
239+
private final String userProject;
215240

216241
public DeleteBucketTask(Storage storage, String bucket) {
217242
this.storage = storage;
218243
this.bucket = bucket;
244+
this.userProject = "";
245+
}
246+
247+
public DeleteBucketTask(Storage storage, String bucket, String userProject) {
248+
this.storage = storage;
249+
this.bucket = bucket;
250+
this.userProject = userProject;
219251
}
220252

221253
@Override
222254
public Boolean call() {
223255
while (true) {
224256
ArrayList<BlobId> ids = new ArrayList<>();
225-
for (BlobInfo info : storage.list(bucket, BlobListOption.versions(true)).getValues()) {
257+
Page<Blob> listedBlobs;
258+
if (Strings.isNullOrEmpty(userProject)) {
259+
listedBlobs = storage.list(bucket, BlobListOption.versions(true));
260+
} else {
261+
listedBlobs = storage.list(bucket, BlobListOption.versions(true), BlobListOption.userProject(userProject));
262+
}
263+
for (BlobInfo info : listedBlobs.getValues()) {
226264
ids.add(info.getBlobId());
227265
}
228266
if (!ids.isEmpty()) {
229-
storage.delete(ids);
267+
List<Boolean> results = storage.delete(ids);
268+
if (!Strings.isNullOrEmpty(userProject)) {
269+
for (int i=0; i<results.size(); i++) {
270+
if (!results.get(i)) {
271+
// deleting that blob failed. Let's try in a different way.
272+
storage.delete(bucket, ids.get(i).getName(), Storage.BlobSourceOption.userProject(userProject));
273+
}
274+
}
275+
}
230276
}
231277
try {
232-
storage.delete(bucket);
278+
if (Strings.isNullOrEmpty(userProject)) {
279+
storage.delete(bucket);
280+
} else {
281+
storage.delete(bucket, Storage.BucketSourceOption.userProject(userProject));
282+
}
233283
return true;
234284
} catch (StorageException e) {
235285
if (e.getCode() == 409) {

‎branches/autosynth-pubsub/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/testing/RemoteStorageHelperTest.java‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,8 +76,9 @@ public class RemoteStorageHelperTest {
7676
private static final InputStream JSON_KEY_STREAM = new ByteArrayInputStream(JSON_KEY.getBytes());
7777
private static final StorageException RETRYABLE_EXCEPTION = new StorageException(409, "");
7878
private static final StorageException FATAL_EXCEPTION = new StorageException(500, "");
79+
private static final String BLOB_NAME2 ="n2";
7980
private static final BlobId BLOB_ID1 = BlobId.of(BUCKET_NAME, "n1");
80-
private static final BlobId BLOB_ID2 = BlobId.of(BUCKET_NAME, "n2");
81+
private static final BlobId BLOB_ID2 = BlobId.of(BUCKET_NAME, BLOB_NAME2);
8182

8283
private Blob blob1;
8384
private Blob blob2;
@@ -221,6 +222,29 @@ public void testForceDeleteNoTimeoutFail() {
221222
}
222223
}
223224

225+
@Test
226+
public void testForceDeleteRetriesWithUserProject() throws Exception {
227+
final String USER_PROJECT="user-project";
228+
Storage storageMock = EasyMock.createMock(Storage.class);
229+
EasyMock.expect(blob1.getBlobId()).andReturn(BLOB_ID1);
230+
EasyMock.expect(blob2.getBlobId()).andReturn(BLOB_ID2);
231+
EasyMock.expect(blob2.getName()).andReturn(BLOB_NAME2);
232+
ArrayList<BlobId> ids = new ArrayList<>();
233+
ids.add(BLOB_ID1);
234+
ids.add(BLOB_ID2);
235+
EasyMock.expect(storageMock.delete(ids)).andReturn(ImmutableList.of(Boolean.TRUE, Boolean.FALSE)).anyTimes();
236+
EasyMock.expect(storageMock.delete(BUCKET_NAME, BLOB_NAME2, Storage.BlobSourceOption.userProject(USER_PROJECT))).andReturn(true).anyTimes();
237+
EasyMock.expect(storageMock.list(BUCKET_NAME, BlobListOption.versions(true), BlobListOption.userProject(USER_PROJECT)))
238+
.andReturn(blobPage);
239+
EasyMock.expect(storageMock.delete(BUCKET_NAME, Storage.BucketSourceOption.userProject(USER_PROJECT))).andReturn(true);
240+
EasyMock.replay(storageMock, blob1, blob2);
241+
try {
242+
RemoteStorageHelper.forceDelete(storageMock, BUCKET_NAME, 5, TimeUnit.SECONDS, USER_PROJECT);
243+
} finally {
244+
EasyMock.verify(storageMock);
245+
}
246+
}
247+
224248
@Test
225249
public void testCreateFromStream() {
226250
RemoteStorageHelper helper = RemoteStorageHelper.create(PROJECT_ID, JSON_KEY_STREAM);

0 commit comments

Comments
 (0)