Skip to content

Commit 6d8624b

Browse files
authored
ARROW-16690: [R][FlightRPC] Additional max_chunksize parameter in do_put method (#13267)
**Summary** An additional parameter in Flight do_put to specify chunk size in R. **Problem** Currently, all data is sent through in a single message. It's a likely scenario that users will want the ability to control the batch sizes without building a custom do_put method. **Solution** Additional (optional) parameter to specify chunk size. Lead-authored-by: Christopher.Dunderdale <[email protected]> Co-authored-by: Christopher Dunderdale <[email protected]> Signed-off-by: Dewey Dunnington <[email protected]>
1 parent 5f84335 commit 6d8624b

3 files changed

Lines changed: 27 additions & 3 deletions

File tree

‎r/R/flight.R‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,9 +56,11 @@ flight_disconnect <- function(client) {
5656
#' @param overwrite logical: if `path` exists on `client` already, should we
5757
#' replace it with the contents of `data`? Default is `TRUE`; if `FALSE` and
5858
#' `path` exists, the function will error.
59+
#' @param max_chunksize integer: Maximum size for RecordBatch chunks when a `data.frame` is sent.
60+
#' Individual chunks may be smaller depending on the chunk layout of individual columns.
5961
#' @return `client`, invisibly.
6062
#' @export
61-
flight_put <- function(client, data, path, overwrite = TRUE) {
63+
flight_put <- function(client, data, path, overwrite = TRUE, max_chunksize = NULL) {
6264
assert_is(data, c("data.frame", "Table", "RecordBatch"))
6365

6466
if (!overwrite && flight_path_exists(client, path)) {
@@ -70,8 +72,13 @@ flight_put <- function(client, data, path, overwrite = TRUE) {
7072

7173
py_data <- reticulate::r_to_py(data)
7274
writer <- client$do_put(descriptor_for_path(path), py_data$schema)[[1]]
73-
if (inherits(data, "RecordBatch")) {
75+
if (inherits(data, "RecordBatch") && !is.null(max_chunksize)) {
76+
warning("`max_chunksize` is not supported for flight_put with RecordBatch")
7477
writer$write_batch(py_data)
78+
} else if (inherits(data, "RecordBatch")) {
79+
writer$write_batch(py_data)
80+
} else if (!is.null(max_chunksize)) {
81+
writer$write_table(py_data, max_chunksize)
7582
} else {
7683
writer$write_table(py_data)
7784
}

‎r/man/flight_put.Rd‎

Lines changed: 4 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎r/tests/testthat/test-python-flight.R‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,20 @@ if (process_is_running("demo_flight_server")) {
3737
regexp = 'data must be a "data.frame", "Table", or "RecordBatch"'
3838
)
3939
})
40+
41+
test_that("flight_put with max_chunksize", {
42+
flight_put(client, example_data, path = flight_obj, max_chunksize = 1)
43+
expect_true(flight_path_exists(client, flight_obj))
44+
expect_true(flight_obj %in% list_flights(client))
45+
expect_warning(
46+
flight_put(client, record_batch(example_data), path = flight_obj, max_chunksize = 123),
47+
regexp = "`max_chunksize` is not supported for flight_put with RecordBatch"
48+
)
49+
expect_error(
50+
flight_put(client, Array$create(c(1:3)), path = flight_obj),
51+
regexp = 'data must be a "data.frame", "Table", or "RecordBatch"'
52+
)
53+
})
4054

4155
test_that("flight_get", {
4256
expect_identical(as.data.frame(flight_get(client, flight_obj)), example_data)

0 commit comments

Comments
 (0)