Skip to content

Commit ef1a234

Browse files
Jackie-JiangJackie-Jiang
committed
Do not implicit cast for BOOLEAN and TIMESTAMP
1 parent 6bf11c8 commit ef1a234

29 files changed

+1081
-973
lines changed

‎pinot-common/src/main/java/org/apache/pinot/common/function/scalar/DataTypeConversionFunctions.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ private DataTypeConversionFunctions() {
4343
public static Object cast(Object value, String targetTypeLiteral) {
4444
try {
4545
Class<?> clazz = value.getClass();
46+
// TODO: Support cast for MV
4647
Preconditions.checkArgument(!clazz.isArray() | clazz == byte[].class, "%s must not be an array type", clazz);
4748
PinotDataType sourceType = PinotDataType.getSingleValueType(clazz);
4849
String transformed = targetTypeLiteral.toUpperCase();

‎pinot-core/src/main/java/org/apache/pinot/core/common/DataFetcher.java‎

Lines changed: 42 additions & 109 deletions
Large diffs are not rendered by default.

‎pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunction.java‎

Lines changed: 77 additions & 99 deletions
Large diffs are not rendered by default.

‎pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CaseTransformFunction.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -436,7 +436,7 @@ public byte[][] transformToBytesValuesSV(ProjectionBlock projectionBlock) {
436436
TransformFunction transformFunction = _elseThenStatements.get(i);
437437
byte[][] bytesValues = transformFunction.transformToBytesValuesSV(projectionBlock);
438438
if (_numSelections == 1) {
439-
System.arraycopy(bytesValues, 0, _byteValuesSV, 0, numDocs);
439+
System.arraycopy(bytesValues, 0, _bytesValuesSV, 0, numDocs);
440440
} else {
441441
for (int j = 0; j < numDocs; j++) {
442442
if (selected[j] == i) {

‎pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CastTransformFunction.java‎

Lines changed: 286 additions & 77 deletions
Large diffs are not rendered by default.

‎pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LogicalOperatorTransformFunction.java‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
2525
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
2626
import org.apache.pinot.segment.spi.datasource.DataSource;
27-
import org.apache.pinot.spi.utils.ArrayCopyUtils;
2827

2928

3029
/**
@@ -33,15 +32,15 @@
3332
*/
3433
public abstract class LogicalOperatorTransformFunction extends BaseTransformFunction {
3534
protected List<TransformFunction> _arguments;
36-
protected int[] _results;
3735

3836
@Override
3937
public void init(List<TransformFunction> arguments, Map<String, DataSource> dataSourceMap) {
4038
_arguments = arguments;
4139
int numArguments = arguments.size();
4240
if (numArguments <= 1) {
43-
throw new IllegalArgumentException("Expect more than 1 argument for logical operator [" + getName() + "], args ["
44-
+ Arrays.toString(arguments.toArray()) + "].");
41+
throw new IllegalArgumentException(
42+
"Expect more than 1 argument for logical operator [" + getName() + "], args [" + Arrays.toString(
43+
arguments.toArray()) + "].");
4544
}
4645
for (int i = 0; i < numArguments; i++) {
4746
TransformResultMetadata argumentMetadata = arguments.get(i).getResultMetadata();
@@ -60,20 +59,19 @@ public TransformResultMetadata getResultMetadata() {
6059
@Override
6160
public int[] transformToIntValuesSV(ProjectionBlock projectionBlock) {
6261
int numDocs = projectionBlock.getNumDocs();
63-
64-
if (_results == null || _results.length < numDocs) {
65-
_results = new int[numDocs];
62+
if (_intValuesSV == null) {
63+
_intValuesSV = new int[numDocs];
6664
}
67-
ArrayCopyUtils.copy(_arguments.get(0).transformToIntValuesSV(projectionBlock), _results, numDocs);
65+
System.arraycopy(_arguments.get(0).transformToIntValuesSV(projectionBlock), 0, _intValuesSV, 0, numDocs);
6866
int numArguments = _arguments.size();
6967
for (int i = 1; i < numArguments; i++) {
7068
TransformFunction transformFunction = _arguments.get(i);
7169
int[] results = transformFunction.transformToIntValuesSV(projectionBlock);
7270
for (int j = 0; j < numDocs; j++) {
73-
_results[j] = getLogicalFuncResult(_results[j], results[j]);
71+
_intValuesSV[j] = getLogicalFuncResult(_intValuesSV[j], results[j]);
7472
}
7573
}
76-
return _results;
74+
return _intValuesSV;
7775
}
7876

7977
abstract int getLogicalFuncResult(int left, int right);

‎pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LookupTransformFunction.java‎

Lines changed: 48 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -227,8 +227,11 @@ private void lookup(ProjectionBlock projectionBlock, ValueAcceptor valueAcceptor
227227

228228
@Override
229229
public int[] transformToIntValuesSV(ProjectionBlock projectionBlock) {
230+
if (_lookupColumnFieldSpec.getDataType().getStoredType() != DataType.INT) {
231+
return super.transformToIntValuesSV(projectionBlock);
232+
}
230233
int numDocs = projectionBlock.getNumDocs();
231-
if (_intValuesSV == null || _intValuesSV.length < numDocs) {
234+
if (_intValuesSV == null) {
232235
_intValuesSV = new int[numDocs];
233236
}
234237
lookup(projectionBlock, this::setIntSV);
@@ -237,8 +240,11 @@ public int[] transformToIntValuesSV(ProjectionBlock projectionBlock) {
237240

238241
@Override
239242
public long[] transformToLongValuesSV(ProjectionBlock projectionBlock) {
243+
if (_lookupColumnFieldSpec.getDataType().getStoredType() != DataType.LONG) {
244+
return super.transformToLongValuesSV(projectionBlock);
245+
}
240246
int numDocs = projectionBlock.getNumDocs();
241-
if (_longValuesSV == null || _longValuesSV.length < numDocs) {
247+
if (_longValuesSV == null) {
242248
_longValuesSV = new long[numDocs];
243249
}
244250
lookup(projectionBlock, this::setLongSV);
@@ -247,8 +253,11 @@ public long[] transformToLongValuesSV(ProjectionBlock projectionBlock) {
247253

248254
@Override
249255
public float[] transformToFloatValuesSV(ProjectionBlock projectionBlock) {
256+
if (_lookupColumnFieldSpec.getDataType().getStoredType() != DataType.FLOAT) {
257+
return super.transformToFloatValuesSV(projectionBlock);
258+
}
250259
int numDocs = projectionBlock.getNumDocs();
251-
if (_floatValuesSV == null || _floatValuesSV.length < numDocs) {
260+
if (_floatValuesSV == null) {
252261
_floatValuesSV = new float[numDocs];
253262
}
254263
lookup(projectionBlock, this::setFloatSV);
@@ -257,8 +266,11 @@ public float[] transformToFloatValuesSV(ProjectionBlock projectionBlock) {
257266

258267
@Override
259268
public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) {
269+
if (_lookupColumnFieldSpec.getDataType().getStoredType() != DataType.DOUBLE) {
270+
return super.transformToDoubleValuesSV(projectionBlock);
271+
}
260272
int numDocs = projectionBlock.getNumDocs();
261-
if (_doubleValuesSV == null || _doubleValuesSV.length < numDocs) {
273+
if (_doubleValuesSV == null) {
262274
_doubleValuesSV = new double[numDocs];
263275
}
264276
lookup(projectionBlock, this::setDoubleSV);
@@ -267,8 +279,11 @@ public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) {
267279

268280
@Override
269281
public String[] transformToStringValuesSV(ProjectionBlock projectionBlock) {
282+
if (_lookupColumnFieldSpec.getDataType().getStoredType() != DataType.STRING) {
283+
return super.transformToStringValuesSV(projectionBlock);
284+
}
270285
int numDocs = projectionBlock.getNumDocs();
271-
if (_stringValuesSV == null || _stringValuesSV.length < numDocs) {
286+
if (_stringValuesSV == null) {
272287
_stringValuesSV = new String[numDocs];
273288
}
274289
lookup(projectionBlock, this::setStringSV);
@@ -277,18 +292,24 @@ public String[] transformToStringValuesSV(ProjectionBlock projectionBlock) {
277292

278293
@Override
279294
public byte[][] transformToBytesValuesSV(ProjectionBlock projectionBlock) {
295+
if (_lookupColumnFieldSpec.getDataType().getStoredType() != DataType.BYTES) {
296+
return super.transformToBytesValuesSV(projectionBlock);
297+
}
280298
int numDocs = projectionBlock.getNumDocs();
281-
if (_byteValuesSV == null || _byteValuesSV.length < numDocs) {
282-
_byteValuesSV = new byte[numDocs][];
299+
if (_bytesValuesSV == null) {
300+
_bytesValuesSV = new byte[numDocs][];
283301
}
284302
lookup(projectionBlock, this::setBytesSV);
285-
return _byteValuesSV;
303+
return _bytesValuesSV;
286304
}
287305

288306
@Override
289307
public int[][] transformToIntValuesMV(ProjectionBlock projectionBlock) {
308+
if (_lookupColumnFieldSpec.getDataType().getStoredType() != DataType.INT) {
309+
return super.transformToIntValuesMV(projectionBlock);
310+
}
290311
int numDocs = projectionBlock.getNumDocs();
291-
if (_intValuesMV == null || _intValuesMV.length < numDocs) {
312+
if (_intValuesMV == null) {
292313
_intValuesMV = new int[numDocs][];
293314
}
294315
lookup(projectionBlock, this::setIntMV);
@@ -297,8 +318,11 @@ public int[][] transformToIntValuesMV(ProjectionBlock projectionBlock) {
297318

298319
@Override
299320
public long[][] transformToLongValuesMV(ProjectionBlock projectionBlock) {
321+
if (_lookupColumnFieldSpec.getDataType().getStoredType() != DataType.LONG) {
322+
return super.transformToLongValuesMV(projectionBlock);
323+
}
300324
int numDocs = projectionBlock.getNumDocs();
301-
if (_longValuesMV == null || _longValuesMV.length < numDocs) {
325+
if (_longValuesMV == null) {
302326
_longValuesMV = new long[numDocs][];
303327
}
304328
lookup(projectionBlock, this::setLongMV);
@@ -307,8 +331,11 @@ public long[][] transformToLongValuesMV(ProjectionBlock projectionBlock) {
307331

308332
@Override
309333
public float[][] transformToFloatValuesMV(ProjectionBlock projectionBlock) {
334+
if (_lookupColumnFieldSpec.getDataType().getStoredType() != DataType.FLOAT) {
335+
return super.transformToFloatValuesMV(projectionBlock);
336+
}
310337
int numDocs = projectionBlock.getNumDocs();
311-
if (_floatValuesMV == null || _floatValuesMV.length < numDocs) {
338+
if (_floatValuesMV == null) {
312339
_floatValuesMV = new float[numDocs][];
313340
}
314341
lookup(projectionBlock, this::setFloatMV);
@@ -317,8 +344,11 @@ public float[][] transformToFloatValuesMV(ProjectionBlock projectionBlock) {
317344

318345
@Override
319346
public double[][] transformToDoubleValuesMV(ProjectionBlock projectionBlock) {
347+
if (_lookupColumnFieldSpec.getDataType().getStoredType() != DataType.DOUBLE) {
348+
return super.transformToDoubleValuesMV(projectionBlock);
349+
}
320350
int numDocs = projectionBlock.getNumDocs();
321-
if (_doubleValuesMV == null || _doubleValuesMV.length < numDocs) {
351+
if (_doubleValuesMV == null) {
322352
_doubleValuesMV = new double[numDocs][];
323353
}
324354
lookup(projectionBlock, this::setDoubleMV);
@@ -327,8 +357,11 @@ public double[][] transformToDoubleValuesMV(ProjectionBlock projectionBlock) {
327357

328358
@Override
329359
public String[][] transformToStringValuesMV(ProjectionBlock projectionBlock) {
360+
if (_lookupColumnFieldSpec.getDataType().getStoredType() != DataType.STRING) {
361+
return super.transformToStringValuesMV(projectionBlock);
362+
}
330363
int numDocs = projectionBlock.getNumDocs();
331-
if (_stringValuesMV == null || _stringValuesMV.length < numDocs) {
364+
if (_stringValuesMV == null) {
332365
_stringValuesMV = new String[numDocs][];
333366
}
334367
lookup(projectionBlock, this::setStringMV);
@@ -377,9 +410,9 @@ private void setStringSV(int index, Object value) {
377410

378411
private void setBytesSV(int index, Object value) {
379412
if (value instanceof byte[]) {
380-
_byteValuesSV[index] = (byte[]) value;
413+
_bytesValuesSV[index] = (byte[]) value;
381414
} else {
382-
_byteValuesSV[index] = EMPTY_BYTES;
415+
_bytesValuesSV[index] = EMPTY_BYTES;
383416
}
384417
}
385418

‎pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ValueInTransformFunction.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@
4343
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
4444
import org.apache.pinot.segment.spi.datasource.DataSource;
4545
import org.apache.pinot.segment.spi.index.reader.Dictionary;
46-
import org.apache.pinot.spi.data.FieldSpec;
46+
import org.apache.pinot.spi.data.FieldSpec.DataType;
4747

4848

4949
public class ValueInTransformFunction extends BaseTransformFunction {
@@ -132,7 +132,7 @@ public int[][] transformToDictIdsMV(ProjectionBlock projectionBlock) {
132132

133133
@Override
134134
public int[][] transformToIntValuesMV(ProjectionBlock projectionBlock) {
135-
if (_dictionary != null || _resultMetadata.getDataType() != FieldSpec.DataType.INT) {
135+
if (_dictionary != null || _resultMetadata.getDataType().getStoredType() != DataType.INT) {
136136
return super.transformToIntValuesMV(projectionBlock);
137137
}
138138

@@ -155,7 +155,7 @@ public int[][] transformToIntValuesMV(ProjectionBlock projectionBlock) {
155155

156156
@Override
157157
public long[][] transformToLongValuesMV(ProjectionBlock projectionBlock) {
158-
if (_dictionary != null || _resultMetadata.getDataType() != FieldSpec.DataType.LONG) {
158+
if (_dictionary != null || _resultMetadata.getDataType().getStoredType() != DataType.LONG) {
159159
return super.transformToLongValuesMV(projectionBlock);
160160
}
161161

@@ -178,7 +178,7 @@ public long[][] transformToLongValuesMV(ProjectionBlock projectionBlock) {
178178

179179
@Override
180180
public float[][] transformToFloatValuesMV(ProjectionBlock projectionBlock) {
181-
if (_dictionary != null || _resultMetadata.getDataType() != FieldSpec.DataType.FLOAT) {
181+
if (_dictionary != null || _resultMetadata.getDataType().getStoredType() != DataType.FLOAT) {
182182
return super.transformToFloatValuesMV(projectionBlock);
183183
}
184184

@@ -201,7 +201,7 @@ public float[][] transformToFloatValuesMV(ProjectionBlock projectionBlock) {
201201

202202
@Override
203203
public double[][] transformToDoubleValuesMV(ProjectionBlock projectionBlock) {
204-
if (_dictionary != null || _resultMetadata.getDataType() != FieldSpec.DataType.DOUBLE) {
204+
if (_dictionary != null || _resultMetadata.getDataType().getStoredType() != DataType.DOUBLE) {
205205
return super.transformToDoubleValuesMV(projectionBlock);
206206
}
207207

@@ -224,7 +224,7 @@ public double[][] transformToDoubleValuesMV(ProjectionBlock projectionBlock) {
224224

225225
@Override
226226
public String[][] transformToStringValuesMV(ProjectionBlock projectionBlock) {
227-
if (_dictionary != null || _resultMetadata.getDataType() != FieldSpec.DataType.STRING) {
227+
if (_dictionary != null || _resultMetadata.getDataType().getStoredType() != DataType.STRING) {
228228
return super.transformToStringValuesMV(projectionBlock);
229229
}
230230

‎pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/AndOperatorTransformFunctionTest.java‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,17 +18,18 @@
1818
*/
1919
package org.apache.pinot.core.operator.transform.function;
2020

21+
import org.apache.pinot.common.function.TransformFunctionType;
22+
23+
2124
public class AndOperatorTransformFunctionTest extends LogicalOperatorTransformFunctionTest {
25+
2226
@Override
23-
int getExpectedValue(boolean left, boolean right) {
24-
if (left && right) {
25-
return 1;
26-
}
27-
return 0;
27+
boolean getExpectedValue(boolean left, boolean right) {
28+
return left && right;
2829
}
2930

3031
@Override
31-
String getFuncName() {
32-
return new AndOperatorTransformFunction().getName();
32+
String getFunctionName() {
33+
return TransformFunctionType.AND.getName();
3334
}
3435
}

0 commit comments

Comments
 (0)