Skip to content

Commit fa36a8d

Browse files
xiangfu0xiangfu0
committed
Adding endpoint to download local log files for each component
1 parent 022fd37 commit fa36a8d

File tree

13 files changed

+446
-1
lines changed

13 files changed

+446
-1
lines changed

‎pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotBrokerLogger.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,11 @@
2525
import io.swagger.annotations.Authorization;
2626
import io.swagger.annotations.SecurityDefinition;
2727
import io.swagger.annotations.SwaggerDefinition;
28+
import java.io.IOException;
2829
import java.util.List;
2930
import java.util.Map;
31+
import java.util.Set;
32+
import javax.inject.Inject;
3033
import javax.ws.rs.GET;
3134
import javax.ws.rs.PUT;
3235
import javax.ws.rs.Path;
@@ -37,6 +40,7 @@
3740
import javax.ws.rs.core.HttpHeaders;
3841
import javax.ws.rs.core.MediaType;
3942
import javax.ws.rs.core.Response;
43+
import org.apache.pinot.common.utils.LoggerFileServer;
4044
import org.apache.pinot.common.utils.LoggerUtils;
4145

4246
import static org.apache.pinot.spi.utils.CommonConstants.SWAGGER_AUTHORIZATION_KEY;
@@ -51,6 +55,9 @@
5155
@Path("/")
5256
public class PinotBrokerLogger {
5357

58+
@Inject
59+
private LoggerFileServer _loggerFileServer;
60+
5461
@GET
5562
@Path("/loggers")
5663
@Produces(MediaType.APPLICATION_JSON)
@@ -80,4 +87,32 @@ public Map<String, String> setLoggerLevel(@ApiParam(value = "Logger name") @Path
8087
@ApiParam(value = "Logger level") @QueryParam("level") String level) {
8188
return LoggerUtils.setLoggerLevel(loggerName, level);
8289
}
90+
91+
@GET
92+
@Path("/loggers/files")
93+
@Produces(MediaType.APPLICATION_JSON)
94+
@ApiOperation(value = "Get all local log files")
95+
public Set<String> getLocalLogFiles() {
96+
try {
97+
if (_loggerFileServer == null) {
98+
throw new WebApplicationException("Root log directory doesn't exist", Response.Status.INTERNAL_SERVER_ERROR);
99+
}
100+
return _loggerFileServer.getAllPaths();
101+
} catch (IOException e) {
102+
throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
103+
}
104+
}
105+
106+
@GET
107+
@Path("/loggers/download")
108+
@Produces(MediaType.APPLICATION_OCTET_STREAM)
109+
@ApiOperation(value = "Download a log file")
110+
public Response downloadLogFile(
111+
@ApiParam(value = "Log file path", required = true) @QueryParam("filePath") String filePath) {
112+
if (_loggerFileServer == null) {
113+
throw new WebApplicationException("Root log directory is not configured",
114+
Response.Status.INTERNAL_SERVER_ERROR);
115+
}
116+
return _loggerFileServer.downloadLogFile(filePath);
117+
}
83118
}

‎pinot-broker/src/main/java/org/apache/pinot/broker/broker/BrokerAdminApiApplication.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import org.apache.pinot.broker.requesthandler.BrokerRequestHandler;
3333
import org.apache.pinot.broker.routing.BrokerRoutingManager;
3434
import org.apache.pinot.common.metrics.BrokerMetrics;
35+
import org.apache.pinot.common.utils.LoggerFileServer;
3536
import org.apache.pinot.core.api.ServiceAutoDiscoveryFeature;
3637
import org.apache.pinot.core.query.executor.sql.SqlQueryExecutor;
3738
import org.apache.pinot.core.transport.ListenerConfig;
@@ -51,6 +52,7 @@ public class BrokerAdminApiApplication extends ResourceConfig {
5152
private static final String RESOURCE_PACKAGE = "org.apache.pinot.broker.api.resources";
5253
public static final String PINOT_CONFIGURATION = "pinotConfiguration";
5354
public static final String BROKER_INSTANCE_ID = "brokerInstanceId";
55+
5456
private final boolean _useHttps;
5557

5658
private HttpServer _httpServer;
@@ -78,6 +80,10 @@ protected void configure() {
7880
bind(routingManager).to(BrokerRoutingManager.class);
7981
bind(brokerRequestHandler).to(BrokerRequestHandler.class);
8082
bind(brokerMetrics).to(BrokerMetrics.class);
83+
String loggerRootDir = brokerConf.getProperty(CommonConstants.Broker.CONFIG_OF_LOGGER_ROOT_DIR);
84+
if (loggerRootDir != null) {
85+
bind(new LoggerFileServer(loggerRootDir)).to(LoggerFileServer.class);
86+
}
8187
bind(brokerConf.getProperty(CommonConstants.Broker.CONFIG_OF_BROKER_ID)).named(BROKER_INSTANCE_ID);
8288
}
8389
});

‎pinot-common/src/main/java/org/apache/pinot/common/utils/LoggerFileServer.java‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.pinot.common.utils;
20+
21+
import com.google.common.base.Preconditions;
22+
import java.io.File;
23+
import java.io.IOException;
24+
import java.nio.file.Files;
25+
import java.nio.file.Path;
26+
import java.nio.file.Paths;
27+
import java.util.Set;
28+
import java.util.TreeSet;
29+
import javax.ws.rs.WebApplicationException;
30+
import javax.ws.rs.core.HttpHeaders;
31+
import javax.ws.rs.core.Response;
32+
import javax.ws.rs.core.StreamingOutput;
33+
34+
35+
/**
36+
* Logger file server.
37+
*/
38+
public class LoggerFileServer {
39+
private final File _loggerRootDir;
40+
private final Path _loggerRootDirPath;
41+
42+
public LoggerFileServer(String loggerRootDir) {
43+
Preconditions.checkNotNull(loggerRootDir, "Logger root directory is null");
44+
_loggerRootDir = new File(loggerRootDir);
45+
Preconditions.checkState(_loggerRootDir.exists(), "Logger directory doesn't exists");
46+
_loggerRootDirPath = Paths.get(_loggerRootDir.getAbsolutePath());
47+
}
48+
49+
public Set<String> getAllPaths()
50+
throws IOException {
51+
Set<String> allFiles = new TreeSet<>();
52+
Files.walk(_loggerRootDirPath).filter(Files::isRegularFile).forEach(
53+
f -> allFiles.add(f.toAbsolutePath().toString().replace(_loggerRootDirPath.toAbsolutePath() + "/", "")));
54+
return allFiles;
55+
}
56+
57+
public Response downloadLogFile(String filePath) {
58+
try {
59+
if (!getAllPaths().contains(filePath)) {
60+
throw new WebApplicationException("Invalid file path: " + filePath, Response.Status.FORBIDDEN);
61+
}
62+
File logFile = new File(_loggerRootDir, filePath);
63+
if (!logFile.exists()) {
64+
throw new WebApplicationException("File: " + filePath + " doesn't exists", Response.Status.NOT_FOUND);
65+
}
66+
Response.ResponseBuilder builder = Response.ok();
67+
builder.entity(logFile);
68+
builder.entity((StreamingOutput) output -> Files.copy(logFile.toPath(), output));
69+
builder.header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + logFile.getName());
70+
builder.header(HttpHeaders.CONTENT_LENGTH, logFile.length());
71+
return builder.build();
72+
} catch (IOException e) {
73+
throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
74+
}
75+
}
76+
}

‎pinot-common/src/main/java/org/apache/pinot/common/utils/config/InstanceUtils.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.apache.commons.collections.MapUtils;
2626
import org.apache.helix.model.InstanceConfig;
2727
import org.apache.helix.zookeeper.datamodel.ZNRecord;
28+
import org.apache.pinot.common.helix.ExtraInstanceConfig;
2829
import org.apache.pinot.spi.config.instance.Instance;
2930
import org.apache.pinot.spi.utils.CommonConstants;
3031
import org.apache.pinot.spi.utils.CommonConstants.Helix;
@@ -157,4 +158,24 @@ public static void updateHelixInstanceConfig(InstanceConfig instanceConfig, Inst
157158
mapFields.remove(POOL_KEY);
158159
}
159160
}
161+
162+
public static String getInstanceBaseUri(InstanceConfig instanceConfig) {
163+
Map<String, String> fieldMap = instanceConfig.getRecord().getSimpleFields();
164+
String hostName = instanceConfig.getHostName();
165+
String adminPort;
166+
String scheme;
167+
if (fieldMap.containsKey(CommonConstants.Helix.Instance.ADMIN_HTTPS_PORT_KEY)) {
168+
// For Pinot Server admin https port
169+
adminPort = fieldMap.get(CommonConstants.Helix.Instance.ADMIN_HTTPS_PORT_KEY);
170+
scheme = "https";
171+
} else if (fieldMap.containsKey(ExtraInstanceConfig.PinotInstanceConfigProperty.PINOT_TLS_PORT.toString())) {
172+
// For Pinot Controller/Broker TLS port
173+
adminPort = fieldMap.get(ExtraInstanceConfig.PinotInstanceConfigProperty.PINOT_TLS_PORT.toString());
174+
scheme = "https";
175+
} else {
176+
adminPort = fieldMap.getOrDefault(CommonConstants.Helix.Instance.ADMIN_PORT_KEY, instanceConfig.getPort());
177+
scheme = "http";
178+
}
179+
return String.format("%s://%s:%s", scheme, hostName, adminPort);
180+
}
160181
}

‎pinot-common/src/main/java/org/apache/pinot/common/utils/http/HttpClient.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -288,7 +288,7 @@ public SimpleHttpResponse sendRequest(HttpUriRequest request)
288288
}
289289
}
290290

291-
protected CloseableHttpResponse execute(HttpUriRequest request)
291+
public CloseableHttpResponse execute(HttpUriRequest request)
292292
throws IOException {
293293
return _httpClient.execute(request);
294294
}

‎pinot-common/src/test/java/org/apache/pinot/common/utils/LoggerFileServerTest.java‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.pinot.common.utils;
20+
21+
import java.io.File;
22+
import java.io.IOException;
23+
import java.nio.charset.Charset;
24+
import javax.ws.rs.WebApplicationException;
25+
import javax.ws.rs.core.Response;
26+
import org.apache.commons.io.FileUtils;
27+
import org.testng.Assert;
28+
import org.testng.annotations.Test;
29+
30+
import static org.testng.Assert.assertEquals;
31+
import static org.testng.Assert.assertNotNull;
32+
33+
34+
public class LoggerFileServerTest {
35+
36+
@Test
37+
public void testLoggerFileServer()
38+
throws IOException {
39+
File logRootDir = new File(FileUtils.getTempDirectory(), "testGetAllLoggers-" + System.currentTimeMillis());
40+
try {
41+
logRootDir.mkdirs();
42+
LoggerFileServer loggerFileServer = new LoggerFileServer(logRootDir.getAbsolutePath());
43+
44+
// Empty root log directory
45+
assertEquals(loggerFileServer.getAllPaths().size(), 0);
46+
try {
47+
loggerFileServer.downloadLogFile("log1");
48+
Assert.fail("Shouldn't reach here");
49+
} catch (WebApplicationException e1) {
50+
assertEquals(e1.getResponse().getStatus(), Response.Status.FORBIDDEN.getStatusCode());
51+
}
52+
53+
// 1 file: [ log1 ] in root log directory
54+
FileUtils.writeStringToFile(new File(logRootDir, "log1"), "mylog1", Charset.defaultCharset());
55+
assertEquals(loggerFileServer.getAllPaths().size(), 1);
56+
assertNotNull(loggerFileServer.downloadLogFile("log1"));
57+
try {
58+
loggerFileServer.downloadLogFile("log2");
59+
Assert.fail("Shouldn't reach here");
60+
} catch (WebApplicationException e1) {
61+
assertEquals(e1.getResponse().getStatus(), Response.Status.FORBIDDEN.getStatusCode());
62+
}
63+
64+
// 2 files: [ log1, log2 ] in root log directory
65+
FileUtils.writeStringToFile(new File(logRootDir, "log2"), "mylog2", Charset.defaultCharset());
66+
assertEquals(loggerFileServer.getAllPaths().size(), 2);
67+
assertNotNull(loggerFileServer.downloadLogFile("log1"));
68+
assertNotNull(loggerFileServer.downloadLogFile("log2"));
69+
try {
70+
loggerFileServer.downloadLogFile("log3");
71+
Assert.fail("Shouldn't reach here");
72+
} catch (WebApplicationException e1) {
73+
assertEquals(e1.getResponse().getStatus(), Response.Status.FORBIDDEN.getStatusCode());
74+
}
75+
} finally {
76+
FileUtils.deleteQuietly(logRootDir);
77+
}
78+
}
79+
}

‎pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@
5959
import org.apache.pinot.common.minion.InMemoryTaskManagerStatusCache;
6060
import org.apache.pinot.common.minion.TaskGeneratorMostRecentRunInfo;
6161
import org.apache.pinot.common.minion.TaskManagerStatusCache;
62+
import org.apache.pinot.common.utils.LoggerFileServer;
6263
import org.apache.pinot.common.utils.ServiceStartableUtils;
6364
import org.apache.pinot.common.utils.ServiceStatus;
6465
import org.apache.pinot.common.utils.TlsUtils;
@@ -472,6 +473,10 @@ protected void configure() {
472473
bind(_periodicTaskScheduler).to(PeriodicTaskScheduler.class);
473474
bind(_sqlQueryExecutor).to(SqlQueryExecutor.class);
474475
bind(_pinotLLCRealtimeSegmentManager).to(PinotLLCRealtimeSegmentManager.class);
476+
String loggerRootDir = _config.getProperty(CommonConstants.Controller.CONFIG_OF_LOGGER_ROOT_DIR);
477+
if (loggerRootDir != null) {
478+
bind(new LoggerFileServer(loggerRootDir)).to(LoggerFileServer.class);
479+
}
475480
}
476481
});
477482

0 commit comments

Comments
 (0)