Skip to content

Commit 427ea99

Browse files
jayeshchoudharyjayeshchoudhary
committed
Merge branch 'master' into persist-oidc-auth
2 parents 2f93887 + d475712 commit 427ea99

File tree

264 files changed

+9012
-2868
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

264 files changed

+9012
-2868
lines changed

kubernetes/helm/pinot/templates/broker/ingress.yaml renamed to kubernetes/helm/pinot/templates/broker/ingress-v1.yaml

Lines changed: 0 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,3 @@
1-
{{- if .Values.broker.ingress.v1beta1.enabled -}}
2-
{{- $ingressPath := .Values.broker.ingress.v1beta1.path -}}
3-
{{- $serviceName := include "pinot.broker.fullname" . -}}
4-
{{- $servicePort := .Values.broker.service.port -}}
5-
apiVersion: extensions/v1beta1
6-
kind: Ingress
7-
metadata:
8-
name: {{ $serviceName }}
9-
{{- if .Values.broker.ingress.v1beta1.annotations }}
10-
annotations:
11-
{{ toYaml .Values.broker.ingress.v1beta1.annotations | indent 4 }}
12-
{{- end }}
13-
labels:
14-
{{- include "pinot.brokerLabels" . | nindent 4 }}
15-
spec:
16-
{{- if .Values.broker.ingress.v1beta1.tls }}
17-
tls:
18-
{{ toYaml .Values.broker.ingress.v1beta1.tls | indent 4 }}
19-
{{- end }}
20-
rules:
21-
{{- range .Values.broker.ingress.v1beta1.hosts }}
22-
- host: {{ . }}
23-
http:
24-
paths:
25-
- path: {{ $ingressPath }}
26-
backend:
27-
serviceName: {{ $serviceName }}
28-
servicePort: {{ $servicePort }}
29-
{{- end }}
30-
{{- end }}
31-
321
{{- if .Values.broker.ingress.v1.enabled -}}
332
{{- $ingressPath := .Values.broker.ingress.v1.path -}}
343
{{- $serviceName := include "pinot.broker.fullname" . -}}

‎kubernetes/helm/pinot/templates/broker/ingress-v1beta1.yaml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
{{- if .Values.broker.ingress.v1beta1.enabled -}}
2+
{{- $ingressPath := .Values.broker.ingress.v1beta1.path -}}
3+
{{- $serviceName := include "pinot.broker.fullname" . -}}
4+
{{- $servicePort := .Values.broker.service.port -}}
5+
apiVersion: extensions/v1beta1
6+
kind: Ingress
7+
metadata:
8+
name: {{ $serviceName }}
9+
{{- if .Values.broker.ingress.v1beta1.annotations }}
10+
annotations:
11+
{{ toYaml .Values.broker.ingress.v1beta1.annotations | indent 4 }}
12+
{{- end }}
13+
labels:
14+
{{- include "pinot.brokerLabels" . | nindent 4 }}
15+
spec:
16+
{{- if .Values.broker.ingress.v1beta1.tls }}
17+
tls:
18+
{{ toYaml .Values.broker.ingress.v1beta1.tls | indent 4 }}
19+
{{- end }}
20+
rules:
21+
{{- range .Values.broker.ingress.v1beta1.hosts }}
22+
- host: {{ . }}
23+
http:
24+
paths:
25+
- path: {{ $ingressPath }}
26+
backend:
27+
serviceName: {{ $serviceName }}
28+
servicePort: {{ $servicePort }}
29+
{{- end }}
30+
{{- end }}

kubernetes/helm/pinot/templates/controller/ingress.yaml renamed to kubernetes/helm/pinot/templates/controller/ingress-v1.yaml

Lines changed: 0 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,3 @@
1-
{{- if .Values.controller.ingress.v1beta1.enabled -}}
2-
{{- $ingressPath := .Values.controller.ingress.v1beta1.path -}}
3-
{{- $serviceName := include "pinot.controller.fullname" . -}}
4-
{{- $servicePort := .Values.controller.service.port -}}
5-
apiVersion: extensions/v1beta1
6-
kind: Ingress
7-
metadata:
8-
name: {{ $serviceName }}
9-
{{- if .Values.controller.ingress.v1beta1.annotations }}
10-
annotations:
11-
{{ toYaml .Values.controller.ingress.v1beta1.annotations | indent 4 }}
12-
{{- end }}
13-
labels:
14-
{{- include "pinot.controllerLabels" . | nindent 4 }}
15-
spec:
16-
{{- if .Values.controller.ingress.v1beta1.tls }}
17-
tls:
18-
{{ toYaml .Values.controller.ingress.v1beta1.tls | indent 4 }}
19-
{{- end }}
20-
rules:
21-
{{- range .Values.controller.ingress.v1beta1.hosts }}
22-
- host: {{ . }}
23-
http:
24-
paths:
25-
- path: {{ $ingressPath }}
26-
backend:
27-
serviceName: {{ $serviceName }}
28-
servicePort: {{ $servicePort }}
29-
{{- end }}
30-
{{- end }}
31-
321
{{- if .Values.controller.ingress.v1.enabled -}}
332
{{- $ingressPath := .Values.controller.ingress.v1.path -}}
343
{{- $serviceName := include "pinot.controller.fullname" . -}}

‎kubernetes/helm/pinot/templates/controller/ingress-v1beta1.yaml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
{{- if .Values.controller.ingress.v1beta1.enabled -}}
2+
{{- $ingressPath := .Values.controller.ingress.v1beta1.path -}}
3+
{{- $serviceName := include "pinot.controller.fullname" . -}}
4+
{{- $servicePort := .Values.controller.service.port -}}
5+
apiVersion: extensions/v1beta1
6+
kind: Ingress
7+
metadata:
8+
name: {{ $serviceName }}
9+
{{- if .Values.controller.ingress.v1beta1.annotations }}
10+
annotations:
11+
{{ toYaml .Values.controller.ingress.v1beta1.annotations | indent 4 }}
12+
{{- end }}
13+
labels:
14+
{{- include "pinot.controllerLabels" . | nindent 4 }}
15+
spec:
16+
{{- if .Values.controller.ingress.v1beta1.tls }}
17+
tls:
18+
{{ toYaml .Values.controller.ingress.v1beta1.tls | indent 4 }}
19+
{{- end }}
20+
rules:
21+
{{- range .Values.controller.ingress.v1beta1.hosts }}
22+
- host: {{ . }}
23+
http:
24+
paths:
25+
- path: {{ $ingressPath }}
26+
backend:
27+
serviceName: {{ $serviceName }}
28+
servicePort: {{ $servicePort }}
29+
{{- end }}
30+
{{- end }}

‎pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/MultiStageBrokerRequestHandler.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@
5454
import org.apache.pinot.query.routing.WorkerManager;
5555
import org.apache.pinot.query.runtime.blocks.TransferableBlock;
5656
import org.apache.pinot.query.service.QueryConfig;
57-
import org.apache.pinot.query.service.QueryDispatcher;
57+
import org.apache.pinot.query.service.dispatch.QueryDispatcher;
5858
import org.apache.pinot.query.type.TypeFactory;
5959
import org.apache.pinot.query.type.TypeSystem;
6060
import org.apache.pinot.spi.env.PinotConfiguration;
@@ -169,10 +169,14 @@ private BrokerResponse handleRequest(long requestId, String query,
169169
return new BrokerResponseNative(QueryException.getException(QueryException.SQL_PARSING_ERROR, e));
170170
}
171171

172+
boolean traceEnabled = Boolean.parseBoolean(
173+
request.has(CommonConstants.Broker.Request.TRACE) ? request.get(CommonConstants.Broker.Request.TRACE).asText()
174+
: "false");
175+
172176
ResultTable queryResults;
173177
Map<Integer, ExecutionStatsAggregator> stageIdStatsMap = new HashMap<>();
174178
for (Integer stageId: queryPlan.getStageMetadataMap().keySet()) {
175-
stageIdStatsMap.put(stageId, new ExecutionStatsAggregator(false));
179+
stageIdStatsMap.put(stageId, new ExecutionStatsAggregator(traceEnabled));
176180
}
177181

178182
try {

‎pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/PinotConnection.java‎

Lines changed: 45 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,9 @@
2424
import java.sql.SQLException;
2525
import java.sql.Statement;
2626
import java.util.Arrays;
27+
import java.util.HashMap;
2728
import java.util.List;
29+
import java.util.Map;
2830
import java.util.Properties;
2931
import org.apache.pinot.client.base.AbstractBaseConnection;
3032
import org.apache.pinot.client.controller.PinotControllerTransport;
@@ -38,11 +40,16 @@
3840
public class PinotConnection extends AbstractBaseConnection {
3941

4042
private static final Logger LOGGER = LoggerFactory.getLogger(Connection.class);
43+
protected static final String[] POSSIBLE_QUERY_OPTIONS = {
44+
QueryOptionKey.ENABLE_NULL_HANDLING,
45+
QueryOptionKey.USE_MULTISTAGE_ENGINE
46+
};
4147
private org.apache.pinot.client.Connection _session;
4248
private boolean _closed;
4349
private String _controllerURL;
4450
private PinotControllerTransport _controllerTransport;
45-
private final boolean _enableNullHandling;
51+
private final Map<String, Object> _queryOptions = new HashMap<String, Object>();
52+
4653
public static final String BROKER_LIST = "brokers";
4754

4855
PinotConnection(String controllerURL, PinotClientTransport transport, String tenant,
@@ -67,15 +74,49 @@ public class PinotConnection extends AbstractBaseConnection {
6774
}
6875
_session = new org.apache.pinot.client.Connection(properties, brokers, transport);
6976

70-
_enableNullHandling = Boolean.parseBoolean(properties.getProperty(QueryOptionKey.ENABLE_NULL_HANDLING));
77+
for (String possibleQueryOption: POSSIBLE_QUERY_OPTIONS) {
78+
Object property = properties.getProperty(possibleQueryOption);
79+
if (property != null) {
80+
_queryOptions.put(possibleQueryOption, parseOptionValue(property));
81+
}
82+
}
83+
}
84+
85+
private Object parseOptionValue(Object value) {
86+
if (value instanceof String) {
87+
String str = (String) value;
88+
89+
try {
90+
Long numVal = Long.valueOf(str);
91+
if (numVal != null) {
92+
return numVal;
93+
}
94+
} catch (NumberFormatException e) {
95+
}
96+
97+
try {
98+
Double numVal = Double.valueOf(str);
99+
if (numVal != null) {
100+
return numVal;
101+
}
102+
} catch (NumberFormatException e) {
103+
}
104+
105+
Boolean boolVal = Boolean.valueOf(str.toLowerCase());
106+
if (boolVal != null) {
107+
return boolVal;
108+
}
109+
}
110+
111+
return value;
71112
}
72113

73114
public org.apache.pinot.client.Connection getSession() {
74115
return _session;
75116
}
76117

77-
public boolean isNullHandlingEnabled() {
78-
return _enableNullHandling;
118+
public Map<String, Object> getQueryOptions() {
119+
return _queryOptions;
79120
}
80121

81122
private List<String> getBrokerList(String controllerURL, String tenant) {

‎pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/PinotPreparedStatement.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ public PinotPreparedStatement(PinotConnection connection, String query) {
5151
if (!DriverUtils.queryContainsLimitStatement(_query)) {
5252
_query += " " + LIMIT_STATEMENT + " " + _maxRows;
5353
}
54-
_query = DriverUtils.enableNullHandling(_connection, _query);
54+
_query = DriverUtils.enableQueryOptions(_query, _connection.getQueryOptions());
5555
_preparedStatement = new PreparedStatement(_session, _query);
5656
}
5757

@@ -177,7 +177,7 @@ public ResultSet executeQuery(String sql)
177177
throws SQLException {
178178
validateState();
179179
try {
180-
_resultSetGroup = _session.execute(DriverUtils.enableNullHandling(_connection, sql));
180+
_resultSetGroup = _session.execute(DriverUtils.enableQueryOptions(sql, _connection.getQueryOptions()));
181181
if (_resultSetGroup.getResultSetCount() == 0) {
182182
_resultSet = PinotResultSet.empty();
183183
return _resultSet;

‎pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/PinotStatement.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ public ResultSet executeQuery(String sql)
6363
if (!DriverUtils.queryContainsLimitStatement(sql)) {
6464
sql += " " + LIMIT_STATEMENT + " " + _maxRows;
6565
}
66-
String enabledSql = DriverUtils.enableNullHandling(_connection, sql);
66+
String enabledSql = DriverUtils.enableQueryOptions(sql, _connection.getQueryOptions());
6767
ResultSetGroup resultSetGroup = _session.execute(enabledSql);
6868
if (resultSetGroup.getResultSetCount() == 0) {
6969
_resultSet = PinotResultSet.empty();

‎pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/utils/DriverUtils.java‎

Lines changed: 30 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -36,12 +36,10 @@
3636
import org.apache.commons.lang3.StringUtils;
3737
import org.apache.http.NameValuePair;
3838
import org.apache.http.client.utils.URLEncodedUtils;
39-
import org.apache.pinot.client.PinotConnection;
4039
import org.apache.pinot.common.config.TlsConfig;
4140
import org.apache.pinot.common.utils.TlsUtils;
4241
import org.apache.pinot.core.auth.BasicAuthUtils;
4342
import org.apache.pinot.spi.env.PinotConfiguration;
44-
import org.apache.pinot.spi.utils.CommonConstants.Broker.Request.QueryOptionKey;
4543
import org.slf4j.Logger;
4644
import org.slf4j.LoggerFactory;
4745

@@ -220,13 +218,37 @@ public static boolean queryContainsLimitStatement(String query) {
220218
return matcher.find();
221219
}
222220

223-
public static String enableNullHandling(PinotConnection connection, String query) {
224-
if (query.contains(QueryOptionKey.ENABLE_NULL_HANDLING)) {
225-
return query;
221+
public static String enableQueryOptions(String sql, Map<String, Object> options) {
222+
StringBuilder optionsBuilder = new StringBuilder();
223+
for (Map.Entry<String, Object> optionEntry: options.entrySet()) {
224+
if (!sql.contains(optionEntry.getKey())) {
225+
optionsBuilder.append(DriverUtils.createSetQueryOptionString(optionEntry.getKey(), optionEntry.getValue()));
226+
}
227+
}
228+
optionsBuilder.append(sql);
229+
return optionsBuilder.toString();
230+
}
231+
232+
public static String createSetQueryOptionString(String optionKey, Object optionValue) {
233+
StringBuilder optionBuilder = new StringBuilder();
234+
optionBuilder.append("SET ").append(optionKey);
235+
236+
if (optionValue != null) {
237+
optionBuilder.append('=');
238+
239+
if (optionValue instanceof Boolean) {
240+
optionBuilder.append(((Boolean) optionValue).booleanValue());
241+
} else if (optionValue instanceof Integer || optionValue instanceof Long) {
242+
optionBuilder.append(((Number) optionValue).longValue());
243+
} else if (optionValue instanceof Float || optionValue instanceof Double) {
244+
optionBuilder.append(((Number) optionValue).doubleValue());
245+
} else {
246+
throw new IllegalArgumentException(
247+
"Option Type " + optionValue.getClass().getSimpleName() + " is not supported.");
248+
}
226249
}
227250

228-
return connection.isNullHandlingEnabled()
229-
? String.format("SET %s = true; %s", QueryOptionKey.ENABLE_NULL_HANDLING, query)
230-
: query;
251+
optionBuilder.append(";\n");
252+
return optionBuilder.toString();
231253
}
232254
}

‎pinot-clients/pinot-jdbc-client/src/test/java/org/apache/pinot/client/PinotPreparedStatementTest.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@
2626
import java.util.Properties;
2727
import org.apache.commons.codec.binary.Hex;
2828
import org.apache.pinot.client.utils.DateTimeUtils;
29+
import org.apache.pinot.client.utils.DriverUtils;
30+
import org.apache.pinot.spi.utils.CommonConstants.Broker.Request.QueryOptionKey;
2931
import org.testng.Assert;
3032
import org.testng.annotations.Test;
3133

@@ -35,6 +37,7 @@ public class PinotPreparedStatementTest {
3537
"SELECT * FROM dummy WHERE name = ? and age = ? and score = ? and ts = ? and eligible = ? and sub_score = ?";
3638
public static final String DATE_QUERY = "SELECT * FROM dummy WHERE date = ? and updated_at = ? and created_at = ?";
3739
public static final String SINGLE_STRING_QUERY = "SELECT * FROM dummy WHERE value = ?";
40+
private static final String BASIC_TEST_QUERY = "SELECT * FROM dummy";
3841
private DummyPinotClientTransport _dummyPinotClientTransport = new DummyPinotClientTransport();
3942
private DummyPinotControllerTransport _dummyPinotControllerTransport = DummyPinotControllerTransport.create();
4043

@@ -120,4 +123,32 @@ public void testSetAdditionalDataTypes()
120123
Assert.assertEquals(lastExecutedQuery.substring(0, lastExecutedQuery.indexOf("LIMIT")).trim(),
121124
String.format("SELECT * FROM dummy WHERE value = '%s'", Hex.encodeHexString(value.getBytes())));
122125
}
126+
127+
@Test
128+
public void testSetEnableNullHandling()
129+
throws Exception {
130+
Properties props = new Properties();
131+
props.put(QueryOptionKey.ENABLE_NULL_HANDLING, "true");
132+
PinotConnection pinotConnection =
133+
new PinotConnection(props, "dummy", _dummyPinotClientTransport, "dummy", _dummyPinotControllerTransport);
134+
PreparedStatement preparedStatement = pinotConnection.prepareStatement(BASIC_TEST_QUERY);
135+
preparedStatement.executeQuery();
136+
String expectedSql =
137+
DriverUtils.createSetQueryOptionString(QueryOptionKey.ENABLE_NULL_HANDLING, true) + BASIC_TEST_QUERY;
138+
Assert.assertEquals(_dummyPinotClientTransport.getLastQuery().substring(0, expectedSql.length()), expectedSql);
139+
}
140+
141+
@Test
142+
public void testSetEnableNullHandling2()
143+
throws Exception {
144+
Properties props = new Properties();
145+
props.put(QueryOptionKey.ENABLE_NULL_HANDLING, "true");
146+
PinotConnection pinotConnection =
147+
new PinotConnection(props, "dummy", _dummyPinotClientTransport, "dummy", _dummyPinotControllerTransport);
148+
PreparedStatement preparedStatement = pinotConnection.prepareStatement("");
149+
preparedStatement.executeQuery(BASIC_TEST_QUERY);
150+
String expectedSql =
151+
DriverUtils.createSetQueryOptionString(QueryOptionKey.ENABLE_NULL_HANDLING, true) + BASIC_TEST_QUERY;
152+
Assert.assertEquals(_dummyPinotClientTransport.getLastQuery().substring(0, expectedSql.length()), expectedSql);
153+
}
123154
}

0 commit comments

Comments
 (0)