Skip to content

[WIP] froll more robust, no nested parallelism #3394

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jangorecki wants to merge 1 commit into from
Closed

[WIP] froll more robust, no nested parallelism #3394

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 14 additions & 3 deletions R/test.data.table.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -280,9 +280,20 @@ test <- function(num,x,y=TRUE,error=NULL,warning=NULL,output=NULL,message=NULL)
}
if (!fail && !length(error) && length(output)) {
if (out[length(out)] == "NULL") out = out[-length(out)]
out = paste(out, collapse="\n")
output = paste(output, collapse="\n") # so that output= can be either a \n separated string, or a vector of strings.
if (!string_match(output, out)) {
output_mismatch = FALSE
if (length(output)>1L && length(output)==length(out)) { # allow matching which would fail due to newline symbol in string
if (!all(mapply(string_match, output, out, USE.NAMES=FALSE))) {
output_mismatch = TRUE
}
out = paste(out, collapse="\n")
output = paste(output, collapse="\n")
} else {
out = paste(out, collapse="\n")
output = paste(output, collapse="\n") # so that output= can be either a \n separated string, or a vector of strings.
if (!string_match(output, out))
output_mismatch = TRUE
}
if (output_mismatch) {
# nocov start
cat("Test",num,"didn't produce correct output:\n")
cat("Expected: <<",gsub("\n","\\\\n",output),">>\n",sep="") # \n printed as '\\n' so the two lines of output can be compared vertically
Expand Down
204 changes: 154 additions & 50 deletions inst/tests/froll.Rraw
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -299,28 +299,44 @@ test(9999.067, ans2, expected)
#### early stopping NAs in leading k obs
test(9999.0671, frollmean(c(1:2,NA,4:10), 4, verbose=TRUE), c(rep(NA_real_, 6), 5.5, 6.5, 7.5, 8.5), output=c(
"frollfunR: allocating memory for results 1x1",
"frollfunR: single column and single window, parallel processing by multiple answer vectors skipped",
"frollfunR: 1 column(s) and 1 window(s), if product > 1 then entering parallel execution",
"frollfunR: 1:",
"frollmeanFast: running for input length 10, window 4, hasna 0, narm 0",
"frollmeanFast: NA (or other non-finite) value(s) are present in input, skip non-NA attempt and run with extra care for NAs"
"frollmeanFast: NA (or other non-finite) value(s) are present in input, skip non-NA attempt and run with extra care for NAs",
"frollmean: processing.*took.*s",
"frollfunR: processing.*took.*s",
".*[[]1[]] .*"
))
test(9999.0672, frollmean(c(1:2,NA,4:10), 4, hasNA=FALSE, verbose=TRUE), c(rep(NA_real_, 6), 5.5, 6.5, 7.5, 8.5), output=c(
"frollfunR: allocating memory for results 1x1",
"frollfunR: single column and single window, parallel processing by multiple answer vectors skipped",
"frollfunR: 1 column(s) and 1 window(s), if product > 1 then entering parallel execution",
"frollfunR: 1:",
"frollmeanFast: running for input length 10, window 4, hasna -1, narm 0",
"frollmeanFast: NA (or other non-finite) value(s) are present in input, skip non-NA attempt and run with extra care for NAs"
"frollmeanFast: NA (or other non-finite) value(s) are present in input, skip non-NA attempt and run with extra care for NAs",
"frollmean: processing.*took.*s",
"frollfunR: processing.*took.*s",
".*[[]1[]] .*"
), warning="hasNA=FALSE used but NA (or other non-finite) value(s) are present in input, use default hasNA=NA to avoid this warning")
test(9999.0673, ans<-frollmean(c(1:2,NA,4:10), 2, hasNA=FALSE, verbose=TRUE), c(NA, 1.5, NA, NA, 4.5, 5.5, 6.5, 7.5, 8.5, 9.5), output=c(
test(9999.0673, frollmean(c(1:2,NA,4:10), 2, hasNA=FALSE, verbose=TRUE), c(NA, 1.5, NA, NA, 4.5, 5.5, 6.5, 7.5, 8.5, 9.5), output=c(
"frollfunR: allocating memory for results 1x1",
"frollfunR: single column and single window, parallel processing by multiple answer vectors skipped",
"frollfunR: 1 column(s) and 1 window(s), if product > 1 then entering parallel execution",
"frollfunR: 1:",
"frollmeanFast: running for input length 10, window 2, hasna -1, narm 0",
"frollmeanFast: NA (or other non-finite) value(s) are present in input, re-running with extra care for NAs"
"frollmeanFast: NA (or other non-finite) value(s) are present in input, re-running with extra care for NAs",
"frollmean: processing.*took.*s",
"frollfunR: processing.*took.*s",
".*[[]1[]] .*"
), warning="hasNA=FALSE used but NA (or other non-finite) value(s) are present in input, use default hasNA=NA to avoid this warning")
test(9999.0674, frollmean(c(1:2,NA,4:10), 4, verbose=TRUE, align="center"), c(rep(NA_real_, 4), 5.5, 6.5, 7.5, 8.5, NA, NA), output=c(
"frollfunR: allocating memory for results 1x1",
"frollfunR: single column and single window, parallel processing by multiple answer vectors skipped",
"frollfunR: 1 column(s) and 1 window(s), if product > 1 then entering parallel execution",
"frollfunR: 1:",
"frollmeanFast: running for input length 10, window 4, hasna 0, narm 0",
"frollmeanFast: NA (or other non-finite) value(s) are present in input, skip non-NA attempt and run with extra care for NAs",
"frollmean: align 0, shift answer by -2"
"frollmean: align 0, shift answer by -2",
"frollmean: processing.*took.*s",
"frollfunR: processing.*took.*s",
".*[[]1[]] .*"
))

#### fill constant
Expand Down Expand Up @@ -414,16 +430,24 @@ test(9999.1195, frollmean(1:5, 2, na.rm=TRUE, hasNA=FALSE), error="using hasNA F
#### exact na.rm=TRUE adaptive=TRUE verbose=TRUE
test(9999.1196, frollmean(c(1:5,NA), 1:6, algo="exact", na.rm=TRUE, adaptive=TRUE, verbose=TRUE), output=c(
"frollfunR: allocating memory for results 1x1",
"frollfunR: single column and single window, parallel processing by multiple answer vectors skipped but 'exact' version of rolling function will compute results in parallel",
"fadaptiverollmeanExact: running for input length 6, hasna 0, narm 1",
"fadaptiverollmeanExact: NA (or other non-finite) value(s) are present in input, re-running with extra care for NAs"
"frollfunR: 1 column(s) and 1 window(s), not entering parallel execution here because algo='exact' will compute results in parallel",
"frollfunR: 1:",
"fadaptiverollmeanExact: running in parallel for input length 6, hasna 0, narm 1",
"fadaptiverollmeanExact: NA (or other non-finite) value(s) are present in input, re-running with extra care for NAs",
"fadaptiverollmean: processing.*took.*s",
"frollfunR: processing.*took.*s",
".*[[]1[]] .*"
))
#### exact na.rm=TRUE verbose=TRUE
test(9999.1197, frollmean(c(1:5,NA), 2, algo="exact", na.rm=TRUE, verbose=TRUE), output=c(
"frollfunR: allocating memory for results 1x1",
"frollfunR: single column and single window, parallel processing by multiple answer vectors skipped but 'exact' version of rolling function will compute results in parallel",
"frollmeanExact: running for input length 6, window 2, hasna 0, narm 1",
"frollmeanExact: NA (or other non-finite) value(s) are present in input, re-running with extra care for NAs"
"frollfunR: 1 column(s) and 1 window(s), not entering parallel execution here because algo='exact' will compute results in parallel",
"frollfunR: 1:",
"frollmeanExact: running in parallel for input length 6, window 2, hasna 0, narm 1",
"frollmeanExact: NA (or other non-finite) value(s) are present in input, re-running with extra care for NAs",
"frollmean: processing.*took.*s",
"frollfunR: processing.*took.*s",
".*[[]1[]] .*"
))
#### adaptive=TRUE n=character
test(9999.1198, frollmean(1:5, n=letters[1:5], adaptive=TRUE), error="n must be integer vector or list of integer vectors")
Expand Down Expand Up @@ -633,82 +657,162 @@ x = 1:10
n = 3
test(9999.171, frollmean(x, n, verbose=TRUE), output=c(
"frollfunR: allocating memory for results 1x1",
"frollfunR: single column and single window, parallel processing by multiple answer vectors skipped",
"frollmeanFast: running for input length 10, window 3, hasna 0, narm 0"))
"frollfunR: 1 column(s) and 1 window(s), if product > 1 then entering parallel execution",
"frollfunR: 1:",
"frollmeanFast: running for input length 10, window 3, hasna 0, narm 0",
"frollmean: processing.*took.*s",
"frollfunR: processing.*took.*s",
".*[[]1[]] .*"))
test(9999.172, frollmean(list(x, x+1), n, verbose=TRUE), output=c(
"frollfunR: allocating memory for results 2x1",
"frollfunR: 2 column(s) and 1 window(s), entering parallel execution, but actually single threaded due to enabled verbose which is not thread safe",
"frollfunR: 2 column(s) and 1 window(s), if product > 1 then entering parallel execution",
"frollfunR: 1:",
"frollmeanFast: running for input length 10, window 3, hasna 0, narm 0",
"frollmeanFast: running for input length 10, window 3, hasna 0, narm 0"))
"frollmean: processing.*took.*s",
"frollfunR: 2:",
"frollmeanFast: running for input length 10, window 3, hasna 0, narm 0",
"frollmean: processing.*took.*s",
"frollfunR: processing.*took.*s",
"[[][[]1[]][]]",".*[[]1[]] .*", "", "[[][[]2[]][]]",".*[[]1[]] .*", ""))
test(9999.173, frollmean(x, c(n, n+1), verbose=TRUE), output=c(
"frollfunR: allocating memory for results 1x2",
"frollfunR: 1 column(s) and 2 window(s), entering parallel execution, but actually single threaded due to enabled verbose which is not thread safe",
"frollfunR: 1 column(s) and 2 window(s), if product > 1 then entering parallel execution",
"frollfunR: 1:",
"frollmeanFast: running for input length 10, window 3, hasna 0, narm 0",
"frollmeanFast: running for input length 10, window 4, hasna 0, narm 0"))
"frollmean: processing.*took.*s",
"frollfunR: 2:",
"frollmeanFast: running for input length 10, window 4, hasna 0, narm 0",
"frollmean: processing.*took.*s",
"frollfunR: processing.*took.*s",
"[[][[]1[]][]]",".*[[]1[]] .*", "", "[[][[]2[]][]]",".*[[]1[]] .*", ""))
test(9999.174, frollmean(list(x, x+1), c(n, n+1), verbose=TRUE), output=c(
"frollfunR: allocating memory for results 2x2",
"frollfunR: 2 column(s) and 2 window(s), entering parallel execution, but actually single threaded due to enabled verbose which is not thread safe",
"frollfunR: 2 column(s) and 2 window(s), if product > 1 then entering parallel execution",
"frollfunR: 1:",
"frollmeanFast: running for input length 10, window 3, hasna 0, narm 0",
"frollmeanFast: running for input length 10, window 4, hasna 0, narm 0"))
"frollmean: processing.*took.*s",
"frollfunR: 2:",
"frollmeanFast: running for input length 10, window 4, hasna 0, narm 0",
"frollmean: processing.*took.*s",
"frollfunR: 3:",
"frollmeanFast: running for input length 10, window 3, hasna 0, narm 0",
"frollmean: processing.*took.*s",
"frollfunR: 4:",
"frollmeanFast: running for input length 10, window 4, hasna 0, narm 0",
"frollmean: processing.*took.*s",
"frollfunR: processing.*took.*s",
"[[][[]1[]][]]",".*[[]1[]] .*", "", "[[][[]2[]][]]",".*[[]1[]] .*", "",
"[[][[]3[]][]]",".*[[]1[]] .*", "", "[[][[]4[]][]]",".*[[]1[]] .*", ""))
test(9999.175, frollmean(x, n, algo="exact", verbose=TRUE), output=c(
"frollfunR: allocating memory for results 1x1",
"frollfunR: single column and single window, parallel processing by multiple answer vectors skipped but 'exact' version of rolling function will compute results in parallel",
"frollmeanExact: running for input length 10, window 3, hasna 0, narm 0"))
"frollfunR: 1 column(s) and 1 window(s), not entering parallel execution here because algo='exact' will compute results in parallel",
"frollfunR: 1:",
"frollmeanExact: running in parallel for input length 10, window 3, hasna 0, narm 0",
"frollmean: processing.*took.*s",
"frollfunR: processing.*took.*s",
".*[[]1[]] .*"))
test(9999.176, frollmean(x, n, align="center", verbose=TRUE), output=c(
"frollfunR: allocating memory for results 1x1",
"frollfunR: single column and single window, parallel processing by multiple answer vectors skipped",
"frollfunR: 1 column(s) and 1 window(s), if product > 1 then entering parallel execution",
"frollfunR: 1:",
"frollmeanFast: running for input length 10, window 3, hasna 0, narm 0",
"frollmean: align 0, shift answer by -1"))
"frollmean: align 0, shift answer by -1",
"frollmean: processing.*took.*s",
"frollfunR: processing.*took.*s",
".*[[]1[]] .*"))
test(9999.177, frollmean(x, n, align="left", verbose=TRUE), output=c(
"frollfunR: allocating memory for results 1x1",
"frollfunR: single column and single window, parallel processing by multiple answer vectors skipped",
"frollfunR: 1 column(s) and 1 window(s), if product > 1 then entering parallel execution",
"frollfunR: 1:",
"frollmeanFast: running for input length 10, window 3, hasna 0, narm 0",
"frollmean: align -1, shift answer by -2"))
"frollmean: align -1, shift answer by -2",
"frollmean: processing.*took.*s",
"frollfunR: processing.*took.*s",
".*[[]1[]] .*"))
nn = c(1:4,2:3,1:4)
test(9999.178, frollmean(x, nn, adaptive=TRUE, verbose=TRUE), output=c(
"frollfunR: allocating memory for results 1x1",
"frollfunR: single column and single window, parallel processing by multiple answer vectors skipped",
"fadaptiverollmeanFast: running for input length 10, hasna 0, narm 0"))
"frollfunR: 1 column(s) and 1 window(s), if product > 1 then entering parallel execution",
"frollfunR: 1:",
"fadaptiverollmeanFast: running for input length 10, hasna 0, narm 0",
"fadaptiverollmean: processing.*took.*s",
"frollfunR: processing.*took.*s",
".*[[]1[]] .*"))
test(9999.179, frollmean(x, nn, algo="exact", adaptive=TRUE, verbose=TRUE), output=c(
"frollfunR: allocating memory for results 1x1",
"frollfunR: single column and single window, parallel processing by multiple answer vectors skipped but 'exact' version of rolling function will compute results in parallel",
"fadaptiverollmeanExact: running for input length 10, hasna 0, narm 0"))
"frollfunR: 1 column(s) and 1 window(s), not entering parallel execution here because algo='exact' will compute results in parallel",
"frollfunR: 1:",
"fadaptiverollmeanExact: running in parallel for input length 10, hasna 0, narm 0",
"fadaptiverollmean: processing.*took.*s",
"frollfunR: processing.*took.*s",
".*[[]1[]] .*"))

x[8] = NA
test(9999.180, frollmean(x, n, verbose=TRUE), output=c(
"frollfunR: allocating memory for results 1x1",
"frollfunR: single column and single window, parallel processing by multiple answer vectors skipped",
"frollfunR: 1 column(s) and 1 window(s), if product > 1 then entering parallel execution",
"frollfunR: 1:",
"frollmeanFast: running for input length 10, window 3, hasna 0, narm 0",
"frollmeanFast: NA (or other non-finite) value(s) are present in input, re-running with extra care for NAs"))
"frollmeanFast: NA (or other non-finite) value(s) are present in input, re-running with extra care for NAs",
"frollmean: processing.*took.*s",
"frollfunR: processing.*took.*s",
".*[[]1[]] .*"))
test(9999.181, frollmean(x, n, algo="exact", verbose=TRUE), output=c(
"frollfunR: allocating memory for results 1x1",
"frollfunR: single column and single window, parallel processing by multiple answer vectors skipped but 'exact' version of rolling function will compute results in parallel",
"frollmeanExact: running for input length 10, window 3, hasna 0, narm 0",
"frollmeanExact: NA (or other non-finite) value(s) are present in input, na.rm was FALSE so in 'exact' implementation NAs were handled already, no need to re-run"))
"frollfunR: 1 column(s) and 1 window(s), not entering parallel execution here because algo='exact' will compute results in parallel",
"frollfunR: 1:",
"frollmeanExact: running in parallel for input length 10, window 3, hasna 0, narm 0",
"frollmeanExact: NA (or other non-finite) value(s) are present in input, na.rm was FALSE so in 'exact' implementation NAs were handled already, no need to re-run",
"frollmean: processing.*took.*s",
"frollfunR: processing.*took.*s",
".*[[]1[]] .*"))
test(9999.182, frollmean(x, nn, adaptive=TRUE, verbose=TRUE), output=c(
"frollfunR: allocating memory for results 1x1",
"frollfunR: single column and single window, parallel processing by multiple answer vectors skipped",
"frollfunR: 1 column(s) and 1 window(s), if product > 1 then entering parallel execution",
"frollfunR: 1:",
"fadaptiverollmeanFast: running for input length 10, hasna 0, narm 0",
"fadaptiverollmeanFast: NA (or other non-finite) value(s) are present in input, re-running with extra care for NAs"))
"fadaptiverollmeanFast: NA (or other non-finite) value(s) are present in input, re-running with extra care for NAs",
"fadaptiverollmean: processing.*took.*s",
"frollfunR: processing.*took.*s",
".*[[]1[]] .*"))
test(9999.183, frollmean(x, nn, algo="exact", adaptive=TRUE, verbose=TRUE), output=c(
"frollfunR: allocating memory for results 1x1",
"frollfunR: single column and single window, parallel processing by multiple answer vectors skipped but 'exact' version of rolling function will compute results in parallel",
"fadaptiverollmeanExact: running for input length 10, hasna 0, narm 0",
"fadaptiverollmeanExact: NA (or other non-finite) value(s) are present in input, na.rm was FALSE so in 'exact' implementation NAs were handled already, no need to re-run"))
"frollfunR: 1 column(s) and 1 window(s), not entering parallel execution here because algo='exact' will compute results in parallel",
"frollfunR: 1:",
"fadaptiverollmeanExact: running in parallel for input length 10, hasna 0, narm 0",
"fadaptiverollmeanExact: NA (or other non-finite) value(s) are present in input, na.rm was FALSE so in 'exact' implementation NAs were handled already, no need to re-run",
"fadaptiverollmean: processing.*took.*s",
"frollfunR: processing.*took.*s",
".*[[]1[]] .*"))

d = as.data.table(list(1:10/2, 10:1/4))
test(9999.184, frollmean(d[,1], 3, algo="exact", verbose=TRUE), output=c(
"frollfunR: allocating memory for results 1x1",
"frollfunR: single column and single window, parallel processing by multiple answer vectors skipped but 'exact' version of rolling function will compute results in parallel",
"frollmeanExact: running for input length 10, window 3, hasna 0, narm 0"
"frollfunR: 1 column(s) and 1 window(s), not entering parallel execution here because algo='exact' will compute results in parallel",
"frollfunR: 1:",
"frollmeanExact: running in parallel for input length 10, window 3, hasna 0, narm 0",
"frollmean: processing.*took.*s",
"frollfunR: processing.*took.*s",
"[[][[]1[]][]]",".*[[]1[]] .*",""
))
test(9999.185, frollmean(d, 3:4, algo="exact", verbose=TRUE), output=c(
"frollfunR: allocating memory for results 2x2",
"frollfunR: 2 column(s) and 2 window(s), entering parallel execution, but actually single threaded due to enabled verbose which is not thread safe, 'exact' version of rolling function will compute results in parallel anyway as it does not print with verbose",
"frollmeanExact: running for input length 10, window 3, hasna 0, narm 0",
"frollmeanExact: running for input length 10, window 4, hasna 0, narm 0",
"frollmeanExact: running for input length 10, window 3, hasna 0, narm 0",
"frollmeanExact: running for input length 10, window 4, hasna 0, narm 0"
"frollfunR: 2 column(s) and 2 window(s), not entering parallel execution here because algo='exact' will compute results in parallel",
"frollfunR: 1:",
"frollmeanExact: running in parallel for input length 10, window 3, hasna 0, narm 0",
"frollmean: processing.*took.*s",
"frollfunR: 2:",
"frollmeanExact: running in parallel for input length 10, window 4, hasna 0, narm 0",
"frollmean: processing.*took.*s",
"frollfunR: 3:",
"frollmeanExact: running in parallel for input length 10, window 3, hasna 0, narm 0",
"frollmean: processing.*took.*s",
"frollfunR: 4:",
"frollmeanExact: running in parallel for input length 10, window 4, hasna 0, narm 0",
"frollmean: processing.*took.*s",
"frollfunR: processing.*took.*s",
"[[][[]1[]][]]",".*[[]1[]] .*", "", "[[][[]2[]][]]",".*[[]1[]] .*", "",
"[[][[]3[]][]]",".*[[]1[]] .*", "", "[[][[]4[]][]]",".*[[]1[]] .*", ""
))

## test warnings
Expand Down
Loading