Skip to content

Commit 79098d0

Browse files
author
Ajay Kannan
committed
---
yaml --- r: 1045 b: refs/heads/master c: 6e66dfd h: refs/heads/master i: 1043: cd4a639 v: v3
1 parent 06f0444 commit 79098d0

2 files changed

Lines changed: 114 additions & 67 deletions

File tree

‎[refs]‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
---
2-
refs/heads/master: 235da5aade459cd24f0b572a8cace8924f455352
2+
refs/heads/master: 6e66dfdac59750d66dde74e9272004a499cc7664
33
refs/heads/travis: 0fa997e2fc9c6b61b2d91e6d163655aae67d44b6
44
refs/heads/gh-pages: 5a10432ecc75f29812e33a8236c900379509fe99

‎trunk/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/testing/LocalGcdHelper.java‎

Lines changed: 113 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -54,23 +54,23 @@
5454
import java.util.List;
5555
import java.util.Locale;
5656
import java.util.Map;
57+
import java.util.logging.Level;
58+
import java.util.logging.Logger;
5759
import java.util.regex.Pattern;
5860
import java.util.zip.ZipEntry;
5961
import java.util.zip.ZipInputStream;
6062

61-
import java.util.logging.Level;
62-
import java.util.logging.Logger;
63-
6463
/**
6564
* Utility to start and stop local Google Cloud Datastore process.
6665
*/
6766
public class LocalGcdHelper {
68-
6967
private static final Logger log = Logger.getLogger(LocalGcdHelper.class.getName());
7068

7169
private final String projectId;
7270
private Path gcdPath;
71+
private Process startProcess;
7372
private ProcessStreamReader processReader;
73+
private ProcessErrorStreamReader processErrorReader;
7474
private final int port;
7575

7676
public static final String DEFAULT_PROJECT_ID = "projectid1";
@@ -179,91 +179,134 @@ private static Path executablePath(String cmd) {
179179
}
180180

181181
private static class ProcessStreamReader extends Thread {
182-
183-
private final Process process;
184182
private final BufferedReader reader;
185-
private final BufferedReader errorReader;
186183

187-
ProcessStreamReader(
188-
Process process, String blockUntil, boolean blockOnErrorStream) throws IOException {
184+
ProcessStreamReader(InputStream inputStream) {
189185
super("Local GCD InputStream reader");
190186
setDaemon(true);
191-
this.process = process;
192-
reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
193-
errorReader = new BufferedReader(new InputStreamReader(process.getErrorStream()));
187+
reader = new BufferedReader(new InputStreamReader(inputStream));
188+
}
189+
190+
void terminate() throws IOException {
191+
reader.close();
192+
}
193+
194+
@Override
195+
public void run() {
196+
try {
197+
while (!(reader.readLine() != null)) {
198+
// consume line
199+
}
200+
} catch (IOException e) {
201+
// ignore
202+
}
203+
}
204+
205+
public static ProcessStreamReader start(InputStream inputStream) {
206+
ProcessStreamReader thread = new ProcessStreamReader(inputStream);
207+
thread.start();
208+
return thread;
209+
}
210+
}
211+
212+
private static class ProcessErrorStreamReader extends Thread {
213+
private static final int LOG_LENGTH_LIMIT = 50000;
214+
private static final String GCD_LOGGING_CLASS =
215+
"com.google.apphosting.client.serviceapp.BaseApiServlet";
216+
217+
private final BufferedReader errorReader;
218+
private String currentLog = null;
219+
private Level currentLogLevel = null;
220+
221+
ProcessErrorStreamReader(InputStream errorStream, String blockUntil) throws IOException {
222+
super("Local GCD ErrorStream reader");
223+
setDaemon(true);
224+
errorReader = new BufferedReader(new InputStreamReader(errorStream));
194225
if (!Strings.isNullOrEmpty(blockUntil)) {
195226
String line;
196227
do {
197-
if (blockOnErrorStream) {
198-
line = errorReader.readLine();
199-
} else {
200-
line = reader.readLine();
201-
}
228+
line = errorReader.readLine();
202229
} while (line != null && !line.contains(blockUntil));
203230
}
204231
}
205232

206-
void terminate() throws InterruptedException, IOException {
207-
process.destroy();
208-
process.waitFor();
209-
reader.close();
233+
void terminate() throws IOException {
234+
writeLog(currentLogLevel, currentLog);
235+
errorReader.close();
210236
}
211237

212238
@Override
213239
public void run() {
214240
try {
215-
boolean readerDone = false;
216241
boolean errorReaderDone = false;
217-
String currentLog = null;
218-
while (!readerDone || !errorReaderDone) {
219-
if (!readerDone && reader.ready()) {
220-
readerDone = reader.readLine() == null;
221-
}
222-
if (!errorReaderDone && errorReader.ready()) {
223-
String errorOutput = errorReader.readLine();
224-
if (errorOutput == null) {
225-
errorReaderDone = true;
226-
} else {
227-
currentLog = processLog(errorOutput, currentLog);
228-
}
242+
String previousLine = "";
243+
String currentLine = "";
244+
while (!errorReaderDone) {
245+
previousLine = currentLine;
246+
currentLine = errorReader.readLine();
247+
if (currentLine == null) {
248+
errorReaderDone = true;
249+
} else {
250+
processLogLine(previousLine, currentLine);
229251
}
230252
}
231253
} catch (IOException e) {
232254
// ignore
233255
}
234256
}
235257

236-
private static boolean isNewGcdLog(String line) {
237-
return line.contains("com.google.apphosting.client.serviceapp.BaseApiServlet")
238-
&& !line.trim().startsWith("at ");
258+
private void processLogLine(String previousLine, String currentLine) {
259+
// Each gcd log is two lines with the following format:
260+
// [Date] [Time] [GCD_LOGGING_CLASS] [method]
261+
// [LEVEL]: error message
262+
// Exceptions and stack traces are included in gcd error stream, separated by a newline
263+
Level nextLogLevel = getLevel(currentLine);
264+
if (previousLine.contains(GCD_LOGGING_CLASS) && nextLogLevel != null) {
265+
writeLog(currentLogLevel, currentLog);
266+
if (currentLine.startsWith("SEVERE: ")) {
267+
// don't show duplicate error messages from gcd.sh (see issue #258)
268+
currentLog = null;
269+
currentLogLevel = null;
270+
} else {
271+
currentLog = "GCD" + currentLine.split(":", 2)[1] + System.getProperty("line.separator");
272+
currentLogLevel = nextLogLevel;
273+
}
274+
} else if (currentLog != null && currentLog.length() > LOG_LENGTH_LIMIT) {
275+
// log processing may be off, so drop some logs before the string becomes too big
276+
currentLog = null;
277+
currentLogLevel = null;
278+
} else if (currentLog != null && isUsefulLogInfo(currentLine)) {
279+
currentLog += currentLine + System.getProperty("line.separator");
280+
}
239281
}
240282

241-
private static String processLog(String currentLine, String currentLog) {
242-
if (isNewGcdLog(currentLine)) {
243-
if (currentLog != null) {
244-
log.info(currentLog.trim());
245-
}
246-
return "GCD ";
247-
} else if (currentLine.startsWith("SEVERE: ")) {
248-
return null; // Don't show duplicate error messages from gcd.sh
249-
} else if (currentLog != null && currentLine.startsWith("INFO: ")) {
250-
return currentLog + currentLine.substring("INFO: ".length()) + "\n";
251-
} else if (currentLog != null && !currentLine.trim().startsWith("at ")) {
252-
return currentLog + currentLine + "\n";
283+
private static void writeLog(Level level, String msg) {
284+
if (level != null && !Strings.isNullOrEmpty(msg)) {
285+
log.log(level, msg.trim());
286+
}
287+
}
288+
289+
private static boolean isUsefulLogInfo(String line) {
290+
return !line.trim().startsWith("at ") && !line.contains(GCD_LOGGING_CLASS);
291+
}
292+
293+
private static Level getLevel(String line) {
294+
try {
295+
return Level.parse(line.split(":")[0]);
296+
} catch (IllegalArgumentException e) {
297+
return null; // level wasn't supplied in this log line
253298
}
254-
return currentLog;
255299
}
256300

257-
public static ProcessStreamReader start(
258-
Process process, String blockUntil, boolean blockOnErrorStream) throws IOException {
259-
ProcessStreamReader thread = new ProcessStreamReader(process, blockUntil, blockOnErrorStream);
301+
public static ProcessErrorStreamReader start(InputStream errorStream, String blockUntil)
302+
throws IOException {
303+
ProcessErrorStreamReader thread = new ProcessErrorStreamReader(errorStream, blockUntil);
260304
thread.start();
261305
return thread;
262306
}
263307
}
264308

265309
private static class CommandWrapper {
266-
267310
private final List<String> prefix;
268311
private List<String> command;
269312
private String nullFilename;
@@ -434,12 +477,15 @@ private void startGcd(Path executablePath) throws IOException, InterruptedExcept
434477
if (log.isLoggable(Level.FINE)) {
435478
log.log(Level.FINE, "Starting datastore emulator for the project: {0}", projectId);
436479
}
437-
Process startProcess = CommandWrapper.create()
438-
.command(gcdAbsolutePath.toString(), "start", "--testing", "--allow_remote_shutdown",
439-
"--port=" + Integer.toString(port), projectId)
440-
.directory(gcdPath)
441-
.start();
442-
processReader = ProcessStreamReader.start(startProcess, "Dev App Server is now running", true);
480+
startProcess =
481+
CommandWrapper.create()
482+
.command(gcdAbsolutePath.toString(), "start", "--testing", "--allow_remote_shutdown",
483+
"--port=" + Integer.toString(port), projectId)
484+
.directory(gcdPath)
485+
.start();
486+
processReader = ProcessStreamReader.start(startProcess.getInputStream());
487+
processErrorReader = ProcessErrorStreamReader.start(
488+
startProcess.getErrorStream(), "Dev App Server is now running");
443489
}
444490

445491
private static String md5(File gcdZipFile) throws IOException {
@@ -495,6 +541,9 @@ public void stop() throws IOException, InterruptedException {
495541
sendQuitRequest(port);
496542
if (processReader != null) {
497543
processReader.terminate();
544+
processErrorReader.terminate();
545+
startProcess.destroy();
546+
startProcess.waitFor();
498547
}
499548
if (gcdPath != null) {
500549
deleteRecurse(gcdPath);
@@ -506,7 +555,6 @@ private static void deleteRecurse(Path path) throws IOException {
506555
return;
507556
}
508557
Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
509-
510558
@Override
511559
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
512560
Files.delete(dir);
@@ -521,7 +569,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO
521569
});
522570
}
523571

524-
public static LocalGcdHelper start(String projectId, int port)
572+
public static LocalGcdHelper start(String projectId, int port)
525573
throws IOException, InterruptedException {
526574
LocalGcdHelper helper = new LocalGcdHelper(projectId, port);
527575
helper.start();
@@ -531,15 +579,14 @@ public static LocalGcdHelper start(String projectId, int port)
531579
public static void main(String... args) throws IOException, InterruptedException {
532580
Map<String, String> parsedArgs = parseArgs(args);
533581
String action = parsedArgs.get("action");
534-
int port = (parsedArgs.get("port") == null) ? DEFAULT_PORT
535-
: Integer.parseInt(parsedArgs.get("port"));
582+
int port =
583+
(parsedArgs.get("port") == null) ? DEFAULT_PORT : Integer.parseInt(parsedArgs.get("port"));
536584
switch (action) {
537585
case "START":
538586
if (!isActive(DEFAULT_PROJECT_ID, port)) {
539587
LocalGcdHelper helper = start(DEFAULT_PROJECT_ID, port);
540588
try (FileWriter writer = new FileWriter(".local_gcd_helper")) {
541-
writer.write(
542-
helper.gcdPath.toAbsolutePath().toString() + System.lineSeparator());
589+
writer.write(helper.gcdPath.toAbsolutePath().toString() + System.lineSeparator());
543590
writer.write(Integer.toString(port));
544591
}
545592
}

0 commit comments

Comments
 (0)