Skip to content

Commit 9ba6028

Browse files
droazenmichaelbausor
droazen
authored and
michaelbausor
committed
---
yaml --- r: 8371 b: refs/heads/snehashah-snippets c: 8556818 h: refs/heads/master i: 8369: 0428648 8367: da3e5b3
1 parent 3bee39a commit 9ba6028

5 files changed

Lines changed: 350 additions & 143 deletions

File tree

‎[refs]‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ refs/tags/v0.18.0: 9d193c4c4b9d1c6f21515dd8e50836b9194ec9bb
5656
refs/tags/v0.19.0: e67b56e4d8dad5f9a7b38c9b2107c23c828f2ed5
5757
refs/tags/v0.20.0: 839f7fb7156535146aa1cb2c5aadd8d375d854e8
5858
refs/tags/v0.20.1: 370471f437f1f4f68a11e068df5cd6bf39edb1fa
59-
refs/heads/snehashah-snippets: 4055f1ffeb78680120b4dab6bbda1f621eb5e2fc
59+
refs/heads/snehashah-snippets: 8556818dfbfcba96a17b212bd143e20bec1752a1
6060
refs/tags/v0.20.2: 5a53aa06f268b74dc192204f4f83e1a04d40f65d
6161
refs/tags/v0.20.3: 269025fdc14af0b68df214a4518be5379b2fe569
6262
refs/tags/v0.21.0: f88b200e00e41ba6262ee88a92abba38b1e2417e

‎branches/snehashah-snippets/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java‎

Lines changed: 133 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -269,7 +269,7 @@ public SeekableByteChannel newByteChannel(
269269
private SeekableByteChannel newReadChannel(Path path, Set<? extends OpenOption> options)
270270
throws IOException {
271271
initStorage();
272-
int maxChannelReopens = ((CloudStorageFileSystem) path.getFileSystem()).config().maxChannelReopens();
272+
int maxChannelReopens = CloudStorageUtil.getMaxChannelReopensFromPath(path);
273273
for (OpenOption option : options) {
274274
if (option instanceof StandardOpenOption) {
275275
switch ((StandardOpenOption) option) {
@@ -408,7 +408,18 @@ public boolean deleteIfExists(Path path) throws IOException {
408408
}
409409
throw new CloudStoragePseudoDirectoryException(cloudPath);
410410
}
411-
return storage.delete(cloudPath.getBlobId());
411+
412+
final CloudStorageRetryHandler retryHandler = new CloudStorageRetryHandler(CloudStorageUtil.getMaxChannelReopensFromPath(path));
413+
// Loop will terminate via an exception if all retries are exhausted
414+
while (true) {
415+
try {
416+
return storage.delete(cloudPath.getBlobId());
417+
} catch (StorageException exs) {
418+
// Will rethrow a StorageException if all retries/reopens are exhausted
419+
retryHandler.handleStorageException(exs);
420+
// we're being aggressive by retrying even on scenarios where we'd normally reopen.
421+
}
422+
}
412423
}
413424

414425
@Override
@@ -440,10 +451,11 @@ public void copy(Path source, Path target, CopyOption... options) throws IOExcep
440451
initStorage();
441452
boolean wantCopyAttributes = false;
442453
boolean wantReplaceExisting = false;
443-
boolean setContentType = false;
444-
boolean setCacheControl = false;
445-
boolean setContentEncoding = false;
446-
boolean setContentDisposition = false;
454+
// true if the option was set manually (so we shouldn't copy the parent's)
455+
boolean overrideContentType = false;
456+
boolean overrideCacheControl = false;
457+
boolean overrideContentEncoding = false;
458+
boolean overrideContentDisposition = false;
447459

448460
CloudStoragePath toPath = CloudStorageUtil.checkPath(target);
449461
BlobInfo.Builder tgtInfoBuilder = BlobInfo.newBuilder(toPath.getBlobId()).setContentType("");
@@ -467,17 +479,17 @@ public void copy(Path source, Path target, CopyOption... options) throws IOExcep
467479
blockSize = ((OptionBlockSize) option).size();
468480
} else if (option instanceof OptionMimeType) {
469481
tgtInfoBuilder.setContentType(((OptionMimeType) option).mimeType());
470-
setContentType = true;
482+
overrideContentType = true;
471483
} else if (option instanceof OptionCacheControl) {
472484
tgtInfoBuilder.setCacheControl(((OptionCacheControl) option).cacheControl());
473-
setCacheControl = true;
485+
overrideCacheControl = true;
474486
} else if (option instanceof OptionContentEncoding) {
475487
tgtInfoBuilder.setContentEncoding(((OptionContentEncoding) option).contentEncoding());
476-
setContentEncoding = true;
488+
overrideContentEncoding = true;
477489
} else if (option instanceof OptionContentDisposition) {
478490
tgtInfoBuilder.setContentDisposition(
479491
((OptionContentDisposition) option).contentDisposition());
480-
setContentDisposition = true;
492+
overrideContentDisposition = true;
481493
} else {
482494
throw new UnsupportedOperationException(option.toString());
483495
}
@@ -515,40 +527,51 @@ public void copy(Path source, Path target, CopyOption... options) throws IOExcep
515527
throw new CloudStoragePseudoDirectoryException(toPath);
516528
}
517529

518-
try {
519-
if (wantCopyAttributes) {
520-
BlobInfo blobInfo = storage.get(fromPath.getBlobId());
521-
if (null == blobInfo) {
522-
throw new NoSuchFileException(fromPath.toString());
523-
}
524-
if (!setCacheControl) {
525-
tgtInfoBuilder.setCacheControl(blobInfo.getCacheControl());
526-
}
527-
if (!setContentType) {
528-
tgtInfoBuilder.setContentType(blobInfo.getContentType());
530+
final CloudStorageRetryHandler retryHandler = new CloudStorageRetryHandler(CloudStorageUtil.getMaxChannelReopensFromPath(source));
531+
// Loop will terminate via an exception if all retries are exhausted
532+
while (true) {
533+
try {
534+
if ( wantCopyAttributes ) {
535+
BlobInfo blobInfo = storage.get(fromPath.getBlobId());
536+
if ( null == blobInfo ) {
537+
throw new NoSuchFileException(fromPath.toString());
538+
}
539+
if ( !overrideCacheControl ) {
540+
tgtInfoBuilder.setCacheControl(blobInfo.getCacheControl());
541+
}
542+
if ( !overrideContentType ) {
543+
tgtInfoBuilder.setContentType(blobInfo.getContentType());
544+
}
545+
if ( !overrideContentEncoding ) {
546+
tgtInfoBuilder.setContentEncoding(blobInfo.getContentEncoding());
547+
}
548+
if ( !overrideContentDisposition ) {
549+
tgtInfoBuilder.setContentDisposition(blobInfo.getContentDisposition());
550+
}
551+
tgtInfoBuilder.setAcl(blobInfo.getAcl());
552+
tgtInfoBuilder.setMetadata(blobInfo.getMetadata());
529553
}
530-
if (!setContentEncoding) {
531-
tgtInfoBuilder.setContentEncoding(blobInfo.getContentEncoding());
554+
555+
BlobInfo tgtInfo = tgtInfoBuilder.build();
556+
Storage.CopyRequest.Builder copyReqBuilder =
557+
Storage.CopyRequest.newBuilder().setSource(fromPath.getBlobId());
558+
if (wantReplaceExisting) {
559+
copyReqBuilder = copyReqBuilder.setTarget(tgtInfo);
560+
} else {
561+
copyReqBuilder = copyReqBuilder.setTarget(tgtInfo, Storage.BlobTargetOption.doesNotExist());
532562
}
533-
if (!setContentDisposition) {
534-
tgtInfoBuilder.setContentDisposition(blobInfo.getContentDisposition());
563+
CopyWriter copyWriter = storage.copy(copyReqBuilder.build());
564+
copyWriter.getResult();
565+
break;
566+
} catch ( StorageException oops ) {
567+
try {
568+
// Will rethrow a StorageException if all retries/reopens are exhausted
569+
retryHandler.handleStorageException(oops);
570+
// we're being aggressive by retrying even on scenarios where we'd normally reopen.
571+
} catch (StorageException retriesExhaustedException) {
572+
throw asIoException(retriesExhaustedException);
535573
}
536-
tgtInfoBuilder.setAcl(blobInfo.getAcl());
537-
tgtInfoBuilder.setMetadata(blobInfo.getMetadata());
538-
}
539-
540-
BlobInfo tgtInfo = tgtInfoBuilder.build();
541-
Storage.CopyRequest.Builder copyReqBuilder =
542-
Storage.CopyRequest.newBuilder().setSource(fromPath.getBlobId());
543-
if (wantReplaceExisting) {
544-
copyReqBuilder = copyReqBuilder.setTarget(tgtInfo);
545-
} else {
546-
copyReqBuilder = copyReqBuilder.setTarget(tgtInfo, Storage.BlobTargetOption.doesNotExist());
547574
}
548-
CopyWriter copyWriter = storage.copy(copyReqBuilder.build());
549-
copyWriter.getResult();
550-
} catch (StorageException oops) {
551-
throw asIoException(oops);
552575
}
553576
}
554577

@@ -579,13 +602,25 @@ public void checkAccess(Path path, AccessMode... modes) throws IOException {
579602
throw new UnsupportedOperationException(mode.toString());
580603
}
581604
}
582-
CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path);
583-
if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories()) {
584-
return;
585-
}
586-
if (storage.get(cloudPath.getBlobId(), Storage.BlobGetOption.fields(Storage.BlobField.ID))
587-
== null) {
588-
throw new NoSuchFileException(path.toString());
605+
606+
final CloudStorageRetryHandler retryHandler = new CloudStorageRetryHandler(CloudStorageUtil.getMaxChannelReopensFromPath(path));
607+
// Loop will terminate via an exception if all retries are exhausted
608+
while (true) {
609+
try {
610+
CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path);
611+
if ( cloudPath.seemsLikeADirectoryAndUsePseudoDirectories() ) {
612+
return;
613+
}
614+
if ( storage.get(cloudPath.getBlobId(), Storage.BlobGetOption.fields(Storage.BlobField.ID))
615+
== null ) {
616+
throw new NoSuchFileException(path.toString());
617+
}
618+
break;
619+
} catch (StorageException exs) {
620+
// Will rethrow a StorageException if all retries/reopens are exhausted
621+
retryHandler.handleStorageException(exs);
622+
// we're being aggressive by retrying even on scenarios where we'd normally reopen.
623+
}
589624
}
590625
}
591626

@@ -598,23 +633,34 @@ public <A extends BasicFileAttributes> A readAttributes(
598633
throw new UnsupportedOperationException(type.getSimpleName());
599634
}
600635
initStorage();
601-
CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path);
602-
if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories()) {
603-
@SuppressWarnings("unchecked")
604-
A result = (A) new CloudStoragePseudoDirectoryAttributes(cloudPath);
605-
return result;
606-
}
607-
BlobInfo blobInfo = storage.get(cloudPath.getBlobId());
608-
// null size indicate a file that we haven't closed yet, so GCS treats it as not there yet.
609-
if (null == blobInfo || blobInfo.getSize() == null) {
610-
throw new NoSuchFileException(
611-
"gs://" + cloudPath.getBlobId().getBucket() + "/" + cloudPath.getBlobId().getName());
636+
637+
final CloudStorageRetryHandler retryHandler = new CloudStorageRetryHandler(CloudStorageUtil.getMaxChannelReopensFromPath(path));
638+
// Loop will terminate via an exception if all retries are exhausted
639+
while (true) {
640+
try {
641+
CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path);
642+
if ( cloudPath.seemsLikeADirectoryAndUsePseudoDirectories() ) {
643+
@SuppressWarnings("unchecked")
644+
A result = (A) new CloudStoragePseudoDirectoryAttributes(cloudPath);
645+
return result;
646+
}
647+
BlobInfo blobInfo = storage.get(cloudPath.getBlobId());
648+
// null size indicate a file that we haven't closed yet, so GCS treats it as not there yet.
649+
if ( null == blobInfo || blobInfo.getSize() == null ) {
650+
throw new NoSuchFileException(
651+
"gs://" + cloudPath.getBlobId().getBucket() + "/" + cloudPath.getBlobId().getName());
652+
}
653+
CloudStorageObjectAttributes ret;
654+
ret = new CloudStorageObjectAttributes(blobInfo);
655+
@SuppressWarnings("unchecked")
656+
A result = (A) ret;
657+
return result;
658+
} catch (StorageException exs) {
659+
// Will rethrow a StorageException if all retries/reopens are exhausted
660+
retryHandler.handleStorageException(exs);
661+
// we're being aggressive by retrying even on scenarios where we'd normally reopen.
662+
}
612663
}
613-
CloudStorageObjectAttributes ret;
614-
ret = new CloudStorageObjectAttributes(blobInfo);
615-
@SuppressWarnings("unchecked")
616-
A result = (A) ret;
617-
return result;
618664
}
619665

620666
@Override
@@ -653,21 +699,32 @@ public DirectoryStream<Path> newDirectoryStream(Path dir, final Filter<? super P
653699
final CloudStoragePath cloudPath = CloudStorageUtil.checkPath(dir);
654700
checkNotNull(filter);
655701
initStorage();
656-
final String prefix = cloudPath.toRealPath().toString();
657-
final Iterator<Blob> blobIterator = storage.list(cloudPath.bucket(),
658-
Storage.BlobListOption.prefix(prefix), Storage.BlobListOption.currentDirectory(),
659-
Storage.BlobListOption.fields()).iterateAll().iterator();
660-
return new DirectoryStream<Path>() {
661-
@Override
662-
public Iterator<Path> iterator() {
663-
return new LazyPathIterator(cloudPath.getFileSystem(), prefix, blobIterator, filter);
664-
}
665702

666-
@Override
667-
public void close() throws IOException {
668-
// Does nothing since there's nothing to close. Commenting this method to quiet codacy.
703+
final CloudStorageRetryHandler retryHandler = new CloudStorageRetryHandler(CloudStorageUtil.getMaxChannelReopensFromPath(dir));
704+
// Loop will terminate via an exception if all retries are exhausted
705+
while (true) {
706+
try {
707+
final String prefix = cloudPath.toRealPath().toString();
708+
final Iterator<Blob> blobIterator = storage.list(cloudPath.bucket(),
709+
Storage.BlobListOption.prefix(prefix), Storage.BlobListOption.currentDirectory(),
710+
Storage.BlobListOption.fields()).iterateAll().iterator();
711+
return new DirectoryStream<Path>() {
712+
@Override
713+
public Iterator<Path> iterator() {
714+
return new LazyPathIterator(cloudPath.getFileSystem(), prefix, blobIterator, filter);
715+
}
716+
717+
@Override
718+
public void close() throws IOException {
719+
// Does nothing since there's nothing to close. Commenting this method to quiet codacy.
720+
}
721+
};
722+
} catch (StorageException exs) {
723+
// Will rethrow a StorageException if all retries/reopens are exhausted
724+
retryHandler.handleStorageException(exs);
725+
// we're being aggressive by retrying even on scenarios where we'd normally reopen.
669726
}
670-
};
727+
}
671728
}
672729

673730
/**

0 commit comments

Comments
 (0)