[SPARK-35381][R] Fix lambda variable name issues in nested higher order functions at R APIs

HyukjinKwon · HyukjinKwon · commit ecb48ccb7db1 · 2021-05-12T16:52:39.000+09:00
### What changes were proposed in this pull request? This PR fixes the same issue as #32424 ```r df <- sql("SELECT array(1, 2, 3) as numbers, array('a', 'b', 'c') as letters") collect(select( df, array_transform("numbers", function(number) { array_transform("letters", function(latter) { struct(alias(number, "n"), alias(latter, "l")) }) }) )) ``` **Before:** ``` ... a, a, b, b, c, c, a, a, b, b, c, c, a, a, b, b, c, c ``` **After:** ``` ... 1, a, 1, b, 1, c, 2, a, 2, b, 2, c, 3, a, 3, b, 3, c ``` ### Why are the changes needed? To produce the correct results. ### Does this PR introduce _any_ user-facing change? Yes, it fixes the results to be correct as mentioned above. ### How was this patch tested? Manually tested as above, and unit test was added. Closes #32517 from HyukjinKwon/SPARK-35381. Authored-by: Hyukjin Kwon <gurwls223@apache.org> Signed-off-by: Hyukjin Kwon <gurwls223@apache.org>
diff --git a/R/pkg/R/functions.R b/R/pkg/R/functions.R
@@ -3670,7 +3670,12 @@ unresolved_named_lambda_var <- function(...) {
     "org.apache.spark.sql.Column",
     newJObject(
       "org.apache.spark.sql.catalyst.expressions.UnresolvedNamedLambdaVariable",
-      list(...)
+      lapply(list(...), function(x) {
+        handledCallJStatic(
+          "org.apache.spark.sql.catalyst.expressions.UnresolvedNamedLambdaVariable",
+          "freshVarName",
+          x)
+      })
     )
   )
   column(jc)
diff --git a/R/pkg/tests/fulltests/test_sparkSQL.R b/R/pkg/tests/fulltests/test_sparkSQL.R
@@ -2161,6 +2161,20 @@ test_that("higher order functions", {
   expect_error(array_transform("xs", function(...) 42))
 })
 
+test_that("SPARK-34794: lambda vars must be resolved properly in nested higher order functions", {
+  df <- sql("SELECT array(1, 2, 3) as numbers, array('a', 'b', 'c') as letters")
+  ret <- first(select(
+    df,
+    array_transform("numbers", function(number) {
+      array_transform("letters", function(latter) {
+        struct(alias(number, "n"), alias(latter, "l"))
+      })
+    })
+  ))
+
+  expect_equal(1, ret[[1]][[1]][[1]][[1]]$n)
+})
+
 test_that("group by, agg functions", {
   df <- read.json(jsonPath)
   df1 <- agg(df, name = "max", age = "sum")

Original file line number	Diff line number	Diff line change
`@@ -3670,7 +3670,12 @@ unresolved_named_lambda_var <- function(...) {`
`3670`	`3670`	`"org.apache.spark.sql.Column",`
`3671`	`3671`	`newJObject(`
`3672`	`3672`	`"org.apache.spark.sql.catalyst.expressions.UnresolvedNamedLambdaVariable",`
`3673`		`- list(...)`
	`3673`	`+ lapply(list(...), function(x) {`
	`3674`	`+ handledCallJStatic(`
	`3675`	`+ "org.apache.spark.sql.catalyst.expressions.UnresolvedNamedLambdaVariable",`
	`3676`	`+ "freshVarName",`
	`3677`	`+ x)`
	`3678`	`+ })`
`3674`	`3679`	`)`
`3675`	`3680`	`)`
`3676`	`3681`	`column(jc)`