Skip to content

Commit cd4a639

Browse files
author
Ajay Kannan
committed
---
yaml --- r: 1043 b: refs/heads/master c: 39e3d3a h: refs/heads/master i: 1041: 7a4d00a 1039: f892325 v: v3
1 parent bca32ac commit cd4a639

2 files changed

Lines changed: 119 additions & 52 deletions

File tree

‎[refs]‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
---
2-
refs/heads/master: 33ddb90dec5aa31e527a2dc9af949cd6189857ec
2+
refs/heads/master: 39e3d3a18429c814cbd5911bd13caa92510dafdf
33
refs/heads/travis: 0fa997e2fc9c6b61b2d91e6d163655aae67d44b6
44
refs/heads/gh-pages: 5a10432ecc75f29812e33a8236c900379509fe99

‎trunk/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/testing/LocalGcdHelper.java‎

Lines changed: 118 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -54,23 +54,23 @@
5454
import java.util.List;
5555
import java.util.Locale;
5656
import java.util.Map;
57+
import java.util.logging.Level;
58+
import java.util.logging.Logger;
5759
import java.util.regex.Pattern;
5860
import java.util.zip.ZipEntry;
5961
import java.util.zip.ZipInputStream;
6062

61-
import java.util.logging.Level;
62-
import java.util.logging.Logger;
63-
6463
/**
6564
* Utility to start and stop local Google Cloud Datastore process.
6665
*/
6766
public class LocalGcdHelper {
68-
6967
private static final Logger log = Logger.getLogger(LocalGcdHelper.class.getName());
7068

7169
private final String projectId;
7270
private Path gcdPath;
71+
private Process startProcess;
7372
private ProcessStreamReader processReader;
73+
private ProcessErrorStreamReader processErrorReader;
7474
private final int port;
7575

7676
public static final String DEFAULT_PROJECT_ID = "projectid1";
@@ -179,71 +179,134 @@ private static Path executablePath(String cmd) {
179179
}
180180

181181
private static class ProcessStreamReader extends Thread {
182-
183-
private final Process process;
184182
private final BufferedReader reader;
185-
private final BufferedReader errorReader;
186183

187-
ProcessStreamReader(
188-
Process process, String blockUntil, boolean blockOnErrorStream) throws IOException {
184+
ProcessStreamReader(InputStream inputStream) {
189185
super("Local GCD InputStream reader");
190186
setDaemon(true);
191-
this.process = process;
192-
reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
193-
errorReader = new BufferedReader(new InputStreamReader(process.getErrorStream()));
187+
reader = new BufferedReader(new InputStreamReader(inputStream));
188+
}
189+
190+
void terminate() throws IOException {
191+
reader.close();
192+
}
193+
194+
@Override
195+
public void run() {
196+
try {
197+
while (!(reader.readLine() != null)) {
198+
// consume line
199+
}
200+
} catch (IOException e) {
201+
// ignore
202+
}
203+
}
204+
205+
public static ProcessStreamReader start(InputStream inputStream) {
206+
ProcessStreamReader thread = new ProcessStreamReader(inputStream);
207+
thread.start();
208+
return thread;
209+
}
210+
}
211+
212+
private static class ProcessErrorStreamReader extends Thread {
213+
private static final int LOG_LENGTH_LIMIT = 50000;
214+
private static final String GCD_LOGGING_CLASS =
215+
"com.google.apphosting.client.serviceapp.BaseApiServlet";
216+
217+
private final BufferedReader errorReader;
218+
private String currentLog = null;
219+
private Level currentLogLevel = null;
220+
221+
ProcessErrorStreamReader(InputStream errorStream, String blockUntil) throws IOException {
222+
super("Local GCD ErrorStream reader");
223+
setDaemon(true);
224+
errorReader = new BufferedReader(new InputStreamReader(errorStream));
194225
if (!Strings.isNullOrEmpty(blockUntil)) {
195226
String line;
196227
do {
197-
if (blockOnErrorStream) {
198-
line = errorReader.readLine();
199-
} else {
200-
line = reader.readLine();
201-
}
228+
line = errorReader.readLine();
202229
} while (line != null && !line.contains(blockUntil));
203230
}
204231
}
205232

206-
void terminate() throws InterruptedException, IOException {
207-
process.destroy();
208-
process.waitFor();
209-
reader.close();
233+
void terminate() throws IOException {
234+
writeLog(currentLogLevel, currentLog);
235+
errorReader.close();
210236
}
211237

212238
@Override
213239
public void run() {
214240
try {
215-
boolean readerDone = false;
216241
boolean errorReaderDone = false;
217-
while (!readerDone || !errorReaderDone) {
218-
if (!readerDone && reader.ready()) {
219-
readerDone = reader.readLine() != null;
220-
}
221-
if (!errorReaderDone && errorReader.ready()) {
222-
String errorOutput = errorReader.readLine();
223-
if (errorOutput == null) {
224-
errorReaderDone = true;
225-
} else {
226-
if (errorOutput.startsWith("SEVERE")) {
227-
System.err.println(errorOutput);
228-
}
229-
}
242+
String previousLine = "";
243+
String currentLine = "";
244+
while (!errorReaderDone) {
245+
previousLine = currentLine;
246+
currentLine = errorReader.readLine();
247+
if (currentLine == null) {
248+
errorReaderDone = true;
249+
} else {
250+
processLogLine(previousLine, currentLine);
230251
}
231252
}
232253
} catch (IOException e) {
233254
// ignore
234255
}
235256
}
236257

237-
public static ProcessStreamReader start(
238-
Process process, String blockUntil, boolean blockOnErrorStream) throws IOException {
239-
ProcessStreamReader thread = new ProcessStreamReader(process, blockUntil, blockOnErrorStream);
258+
private void processLogLine(String previousLine, String currentLine) {
259+
// Each gcd log is two lines with the following format:
260+
// [Date] [Time] [GCD_LOGGING_CLASS] [method]
261+
// [LEVEL]: error message
262+
// Exceptions and stack traces are included in gcd error stream, separated by a newline
263+
Level nextLogLevel = getLevel(currentLine);
264+
if (previousLine.contains(GCD_LOGGING_CLASS) && nextLogLevel != null) {
265+
writeLog(currentLogLevel, currentLog);
266+
if (currentLine.startsWith("SEVERE: ")) {
267+
// don't show duplicate error messages from gcd.sh (see issue #258)
268+
currentLog = null;
269+
currentLogLevel = null;
270+
} else {
271+
currentLog = "GCD" + currentLine.split(":", 2)[1] + System.getProperty("line.separator");
272+
currentLogLevel = nextLogLevel;
273+
}
274+
} else if (currentLog != null && currentLog.length() > LOG_LENGTH_LIMIT) {
275+
// log processing may be off, so drop some logs before the string becomes too big
276+
currentLog = null;
277+
currentLogLevel = null;
278+
} else if (currentLog != null && isUsefulLogInfo(currentLine)) {
279+
currentLog += currentLine + System.getProperty("line.separator");
280+
}
281+
}
282+
283+
private static void writeLog(Level level, String msg) {
284+
if (level != null && !Strings.isNullOrEmpty(msg)) {
285+
log.log(level, msg.trim());
286+
}
287+
}
288+
289+
private static boolean isUsefulLogInfo(String line) {
290+
return !line.trim().startsWith("at ") && !line.contains(GCD_LOGGING_CLASS);
291+
}
292+
293+
private static Level getLevel(String line) {
294+
try {
295+
return Level.parse(line.split(":")[0]);
296+
} catch (IllegalArgumentException e) {
297+
return null; // level wasn't supplied in this log line
298+
}
299+
}
300+
301+
public static ProcessErrorStreamReader start(InputStream errorStream, String blockUntil)
302+
throws IOException {
303+
ProcessErrorStreamReader thread = new ProcessErrorStreamReader(errorStream, blockUntil);
240304
thread.start();
241305
return thread;
242306
}
243307
}
244308

245309
private static class CommandWrapper {
246-
247310
private final List<String> prefix;
248311
private List<String> command;
249312
private String nullFilename;
@@ -414,12 +477,15 @@ private void startGcd(Path executablePath) throws IOException, InterruptedExcept
414477
if (log.isLoggable(Level.FINE)) {
415478
log.log(Level.FINE, "Starting datastore emulator for the project: {0}", projectId);
416479
}
417-
Process startProcess = CommandWrapper.create()
418-
.command(gcdAbsolutePath.toString(), "start", "--testing", "--allow_remote_shutdown",
419-
"--port=" + Integer.toString(port), projectId)
420-
.directory(gcdPath)
421-
.start();
422-
processReader = ProcessStreamReader.start(startProcess, "Dev App Server is now running", true);
480+
startProcess =
481+
CommandWrapper.create()
482+
.command(gcdAbsolutePath.toString(), "start", "--testing", "--allow_remote_shutdown",
483+
"--port=" + Integer.toString(port), projectId)
484+
.directory(gcdPath)
485+
.start();
486+
processReader = ProcessStreamReader.start(startProcess.getInputStream());
487+
processErrorReader = ProcessErrorStreamReader.start(
488+
startProcess.getErrorStream(), "Dev App Server is now running");
423489
}
424490

425491
private static String md5(File gcdZipFile) throws IOException {
@@ -475,6 +541,9 @@ public void stop() throws IOException, InterruptedException {
475541
sendQuitRequest(port);
476542
if (processReader != null) {
477543
processReader.terminate();
544+
processErrorReader.terminate();
545+
startProcess.destroy();
546+
startProcess.waitFor();
478547
}
479548
if (gcdPath != null) {
480549
deleteRecurse(gcdPath);
@@ -486,7 +555,6 @@ private static void deleteRecurse(Path path) throws IOException {
486555
return;
487556
}
488557
Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
489-
490558
@Override
491559
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
492560
Files.delete(dir);
@@ -501,7 +569,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO
501569
});
502570
}
503571

504-
public static LocalGcdHelper start(String projectId, int port)
572+
public static LocalGcdHelper start(String projectId, int port)
505573
throws IOException, InterruptedException {
506574
LocalGcdHelper helper = new LocalGcdHelper(projectId, port);
507575
helper.start();
@@ -511,15 +579,14 @@ public static LocalGcdHelper start(String projectId, int port)
511579
public static void main(String... args) throws IOException, InterruptedException {
512580
Map<String, String> parsedArgs = parseArgs(args);
513581
String action = parsedArgs.get("action");
514-
int port = (parsedArgs.get("port") == null) ? DEFAULT_PORT
515-
: Integer.parseInt(parsedArgs.get("port"));
582+
int port =
583+
(parsedArgs.get("port") == null) ? DEFAULT_PORT : Integer.parseInt(parsedArgs.get("port"));
516584
switch (action) {
517585
case "START":
518586
if (!isActive(DEFAULT_PROJECT_ID, port)) {
519587
LocalGcdHelper helper = start(DEFAULT_PROJECT_ID, port);
520588
try (FileWriter writer = new FileWriter(".local_gcd_helper")) {
521-
writer.write(
522-
helper.gcdPath.toAbsolutePath().toString() + System.lineSeparator());
589+
writer.write(helper.gcdPath.toAbsolutePath().toString() + System.lineSeparator());
523590
writer.write(Integer.toString(port));
524591
}
525592
}

0 commit comments

Comments
 (0)