Skip to content

Commit 276abe3

Browse files
vitaliili-dbMaxGekk
authored andcommitted
[SPARK-40924][SQL] Fix for Unhex when input has odd number of symbols
### What changes were proposed in this pull request? Fix for a bug in `Unhex` function when there is an odd number of symbols in the input string. ### Why are the changes needed? `Unhex` function and other functions depending on it (e.g. `ToBinary`) produce incorrect output. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Unit tests Closes #38402 from vitaliili-db/unhex. Authored-by: Vitalii Li <[email protected]> Signed-off-by: Max Gekk <[email protected]>
1 parent 5b5eb23 commit 276abe3

7 files changed

Lines changed: 113 additions & 3 deletions

File tree

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1036,6 +1036,7 @@ object Hex {
10361036
def unhex(bytes: Array[Byte]): Array[Byte] = {
10371037
val out = new Array[Byte]((bytes.length + 1) >> 1)
10381038
var i = 0
1039+
var oddShift = 0
10391040
if ((bytes.length & 0x01) != 0) {
10401041
// padding with '0'
10411042
if (bytes(0) < 0) {
@@ -1047,6 +1048,7 @@ object Hex {
10471048
}
10481049
out(0) = v
10491050
i += 1
1051+
oddShift = 1
10501052
}
10511053
// two characters form the hex value.
10521054
while (i < bytes.length) {
@@ -1058,7 +1060,7 @@ object Hex {
10581060
if (first == -1 || second == -1) {
10591061
return null
10601062
}
1061-
out(i / 2) = (((first << 4) | second) & 0xFF).toByte
1063+
out(i / 2 + oddShift) = (((first << 4) | second) & 0xFF).toByte
10621064
i += 2
10631065
}
10641066
out

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MathExpressionsSuite.scala‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -590,6 +590,8 @@ class MathExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
590590
checkEvaluation(Unhex(Literal("F")), Array[Byte](15))
591591
checkEvaluation(Unhex(Literal("ff")), Array[Byte](-1))
592592
checkEvaluation(Unhex(Literal("GG")), null)
593+
checkEvaluation(Unhex(Literal("123")), Array[Byte](1, 35))
594+
checkEvaluation(Unhex(Literal("12345")), Array[Byte](1, 35, 69))
593595
// scalastyle:off
594596
// Turn off scala style for non-ascii chars
595597
checkEvaluation(Unhex(Literal("E4B889E9878DE79A84")), "三重的".getBytes(StandardCharsets.UTF_8))

‎sql/core/src/test/resources/sql-tests/inputs/string-functions.sql‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,8 @@ SELECT rpad('hi', 'invalid_length');
8383
SELECT hex(lpad(unhex(''), 5));
8484
SELECT hex(lpad(unhex('aabb'), 5));
8585
SELECT hex(lpad(unhex('aabbcc'), 2));
86+
SELECT hex(lpad(unhex('123'), 2));
87+
SELECT hex(lpad(unhex('12345'), 2));
8688
SELECT hex(lpad(unhex(''), 5, unhex('1f')));
8789
SELECT hex(lpad(unhex('aa'), 5, unhex('1f')));
8890
SELECT hex(lpad(unhex('aa'), 6, unhex('1f')));
@@ -97,6 +99,8 @@ SELECT hex(lpad(unhex('aabbcc'), 2, unhex('ff')));
9799
SELECT hex(rpad(unhex(''), 5));
98100
SELECT hex(rpad(unhex('aabb'), 5));
99101
SELECT hex(rpad(unhex('aabbcc'), 2));
102+
SELECT hex(rpad(unhex('123'), 2));
103+
SELECT hex(rpad(unhex('12345'), 2));
100104
SELECT hex(rpad(unhex(''), 5, unhex('1f')));
101105
SELECT hex(rpad(unhex('aa'), 5, unhex('1f')));
102106
SELECT hex(rpad(unhex('aa'), 6, unhex('1f')));
@@ -202,6 +206,8 @@ select to_binary('737472696E67', 'hex');
202206
select to_binary('');
203207
select to_binary('1', 'hex');
204208
select to_binary('FF');
209+
select to_binary('123', 'hex');
210+
select to_binary('12345', 'hex');
205211
-- hex invalid
206212
select to_binary('GG');
207213
select to_binary('01 AF', 'hex');

‎sql/core/src/test/resources/sql-tests/inputs/try-string-functions.sql‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ select try_to_binary('737472696E67', 'hex');
2727
select try_to_binary('');
2828
select try_to_binary('1', 'hex');
2929
select try_to_binary('FF');
30+
select try_to_binary('123');
31+
select try_to_binary('12345');
3032
-- hex invalid
3133
select try_to_binary('GG');
3234
select try_to_binary('01 AF', 'hex');

‎sql/core/src/test/resources/sql-tests/results/ansi/string-functions.sql.out‎

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -552,6 +552,22 @@ struct<hex(lpad(unhex(aabbcc), 2, X'00')):string>
552552
AABB
553553

554554

555+
-- !query
556+
SELECT hex(lpad(unhex('123'), 2))
557+
-- !query schema
558+
struct<hex(lpad(unhex(123), 2, X'00')):string>
559+
-- !query output
560+
0123
561+
562+
563+
-- !query
564+
SELECT hex(lpad(unhex('12345'), 2))
565+
-- !query schema
566+
struct<hex(lpad(unhex(12345), 2, X'00')):string>
567+
-- !query output
568+
0123
569+
570+
555571
-- !query
556572
SELECT hex(lpad(unhex(''), 5, unhex('1f')))
557573
-- !query schema
@@ -648,6 +664,22 @@ struct<hex(rpad(unhex(aabbcc), 2, X'00')):string>
648664
AABB
649665

650666

667+
-- !query
668+
SELECT hex(rpad(unhex('123'), 2))
669+
-- !query schema
670+
struct<hex(rpad(unhex(123), 2, X'00')):string>
671+
-- !query output
672+
0123
673+
674+
675+
-- !query
676+
SELECT hex(rpad(unhex('12345'), 2))
677+
-- !query schema
678+
struct<hex(rpad(unhex(12345), 2, X'00')):string>
679+
-- !query output
680+
0123
681+
682+
651683
-- !query
652684
SELECT hex(rpad(unhex(''), 5, unhex('1f')))
653685
-- !query schema
@@ -1408,6 +1440,22 @@ struct<to_binary(FF):binary>
14081440
�
14091441

14101442

1443+
-- !query
1444+
select to_binary('123', 'hex')
1445+
-- !query schema
1446+
struct<to_binary(123, hex):binary>
1447+
-- !query output
1448+
#
1449+
1450+
1451+
-- !query
1452+
select to_binary('12345', 'hex')
1453+
-- !query schema
1454+
struct<to_binary(12345, hex):binary>
1455+
-- !query output
1456+
#E
1457+
1458+
14111459
-- !query
14121460
select to_binary('GG')
14131461
-- !query schema
@@ -1489,7 +1537,8 @@ select to_binary('abc', 'Hex')
14891537
-- !query schema
14901538
struct<to_binary(abc, Hex):binary>
14911539
-- !query output
1492-
�
1540+
1541+
�
14931542

14941543

14951544
-- !query

‎sql/core/src/test/resources/sql-tests/results/string-functions.sql.out‎

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -484,6 +484,22 @@ struct<hex(lpad(unhex(aabbcc), 2, X'00')):string>
484484
AABB
485485

486486

487+
-- !query
488+
SELECT hex(lpad(unhex('123'), 2))
489+
-- !query schema
490+
struct<hex(lpad(unhex(123), 2, X'00')):string>
491+
-- !query output
492+
0123
493+
494+
495+
-- !query
496+
SELECT hex(lpad(unhex('12345'), 2))
497+
-- !query schema
498+
struct<hex(lpad(unhex(12345), 2, X'00')):string>
499+
-- !query output
500+
0123
501+
502+
487503
-- !query
488504
SELECT hex(lpad(unhex(''), 5, unhex('1f')))
489505
-- !query schema
@@ -580,6 +596,22 @@ struct<hex(rpad(unhex(aabbcc), 2, X'00')):string>
580596
AABB
581597

582598

599+
-- !query
600+
SELECT hex(rpad(unhex('123'), 2))
601+
-- !query schema
602+
struct<hex(rpad(unhex(123), 2, X'00')):string>
603+
-- !query output
604+
0123
605+
606+
607+
-- !query
608+
SELECT hex(rpad(unhex('12345'), 2))
609+
-- !query schema
610+
struct<hex(rpad(unhex(12345), 2, X'00')):string>
611+
-- !query output
612+
0123
613+
614+
583615
-- !query
584616
SELECT hex(rpad(unhex(''), 5, unhex('1f')))
585617
-- !query schema
@@ -1340,6 +1372,22 @@ struct<to_binary(FF):binary>
13401372
�
13411373

13421374

1375+
-- !query
1376+
select to_binary('123', 'hex')
1377+
-- !query schema
1378+
struct<to_binary(123, hex):binary>
1379+
-- !query output
1380+
#
1381+
1382+
1383+
-- !query
1384+
select to_binary('12345', 'hex')
1385+
-- !query schema
1386+
struct<to_binary(12345, hex):binary>
1387+
-- !query output
1388+
#E
1389+
1390+
13431391
-- !query
13441392
select to_binary('GG')
13451393
-- !query schema
@@ -1421,7 +1469,8 @@ select to_binary('abc', 'Hex')
14211469
-- !query schema
14221470
struct<to_binary(abc, Hex):binary>
14231471
-- !query output
1424-
�
1472+
1473+
�
14251474

14261475

14271476
-- !query
Binary file not shown.

0 commit comments

Comments
 (0)