[infra] Add retry logic to find_base_commit.

sortie · sortie · commit 5e808aa04825 · 2019-01-04T13:11:35.000Z
The buildbucket API is not always reliable and responds within 30 seconds. Rather than spuriously failing, try again. This should improve the reliability of the base branch update bot. Change-Id: Iade2d78372743b8d843c0dead591c28f8ee8f3ae Reviewed-on: https://dart-review.googlesource.com/c/88445 Reviewed-by: William Hesse <whesse@google.com>
diff --git a/tools/bots/find_base_commit.dart b/tools/bots/find_base_commit.dart
@@ -5,6 +5,7 @@
 
 // Find the newest commit that has a full set of results on the bots.
 
+import 'dart:async';
 import 'dart:convert';
 import 'dart:io';
 import 'dart:math';
@@ -50,15 +51,31 @@ ${parser.usage}""");
       "&max_builds=$maxBuilds"
       "&status=COMPLETED"
       "&fields=builds(url%2Cparameters_json)");
-  final client = new HttpClient();
-  final request = await client.getUrl(url).timeout(const Duration(seconds: 30));
-  final response = await request.close().timeout(const Duration(seconds: 30));
-  final Map<String, dynamic> object = await response
-      .transform(new Utf8Decoder())
-      .transform(new JsonDecoder())
-      .first
-      .timeout(const Duration(seconds: 30));
-  client.close();
+  const maxRetries = 3;
+  const timeout = const Duration(seconds: 30);
+  Map<String, dynamic> object;
+  for (int i = 1; i <= maxRetries; i++) {
+    try {
+      final client = new HttpClient();
+      final request = await client.getUrl(url).timeout(timeout);
+      final response = await request.close().timeout(timeout);
+      object = await response
+          .transform(new Utf8Decoder())
+          .transform(new JsonDecoder())
+          .first
+          .timeout(timeout);
+      client.close();
+      break;
+    } on TimeoutException catch (e) {
+      final inSeconds = e.duration.inSeconds;
+      stderr.writeln(
+          "Attempt $i of $maxRetries timed out after $inSeconds seconds");
+      if (i == maxRetries) {
+        stderr.writeln("error: Failed to download $url");
+        exit(1);
+      }
+    }
+  }
 
   // Locate the builds we're interested in and map them to each commit. The
   // builds returned by the API are sorted with the newest first. Since bots
