Skip to content

Commit 698d586

Browse files
MichaelChiricoMichaelChirico
committed
Merge branch 'master' into ms-to-sec
2 parents f002fa3 + a507ed5 commit 698d586

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+828
-425
lines changed

‎.dev/CRAN_Release.cmd‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -530,6 +530,7 @@ sudo apt-get -y install libquantlib0-dev # for RQuantLib
530530
sudo apt-get -y install cargo # for gifski, a suggest of nasoi
531531
sudo apt-get -y install libgit2-dev # for gert
532532
sudo apt-get -y install cmake # for symengine for RxODE
533+
sudo apt-get -y install libxslt1-dev # for xslt
533534
sudo R CMD javareconf
534535
# ENDIF
535536

‎.gitlab-ci.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ variables:
77
## Setting TZ for all GLCI jobs to isolate them from timezone. We could have a new GLCI job to test under
88
## a non-UTC timezone, although, that's what we do routinely in dev.
99
R_REL_VERSION: "4.0"
10-
R_DEVEL_VERSION: "4.1"
10+
R_DEVEL_VERSION: "4.2"
1111
R_OLDREL_VERSION: "3.6"
1212

1313
stages:
@@ -61,7 +61,7 @@ build: ## build data.table sources as tar.gz archive
6161
image: registry.gitlab.com/jangorecki/dockerfiles/r-builder
6262
needs: ["mirror-packages"]
6363
before_script:
64-
- Rscript -e 'install.packages("knitr", repos=file.path("file:",normalizePath("bus/mirror-packages/cran")), quiet=TRUE)'
64+
- Rscript -e 'install.packages(c("knitr","rmarkdown"), repos=file.path("file:",normalizePath("bus/mirror-packages/cran")), quiet=TRUE)'
6565
- rm -r bus
6666
- echo "Revision:" $CI_BUILD_REF >> ./DESCRIPTION
6767
script:

‎DESCRIPTION‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ Authors@R: c(
6464
person("Ben","Schwen", role="ctb"))
6565
Depends: R (>= 3.1.0)
6666
Imports: methods
67-
Suggests: bit64 (>= 4.0.0), bit (>= 4.0.4), curl, R.utils, xts, nanotime, zoo (>= 1.8-1), yaml, knitr, rmarkdown
67+
Suggests: bit64 (>= 4.0.0), bit (>= 4.0.4), curl, R.utils, xts, nanotime, zoo (>= 1.8-1), yaml, knitr, rmarkdown, markdown
6868
SystemRequirements: zlib
6969
Description: Fast aggregation of large data (e.g. 100GB in RAM), fast ordered joins, fast add/modify/delete of columns by group using no copies at all, list columns, friendly and fast character-separated-value read/write. Offers a natural and flexible syntax, for faster development.
7070
License: MPL-2.0 | file LICENSE

‎NEWS.md‎

Lines changed: 78 additions & 6 deletions
Large diffs are not rendered by default.

‎R/as.data.table.R‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,7 @@ as.data.table.list = function(x,
129129
eachncol = integer(n)
130130
missing.check.names = missing(check.names)
131131
origListNames = if (missing(.named)) names(x) else NULL # as.data.table called directly, not from inside data.table() which provides .named, #3854
132+
empty_atomic = FALSE
132133
for (i in seq_len(n)) {
133134
xi = x[[i]]
134135
if (is.null(xi)) next # eachncol already initialized to 0 by integer() above
@@ -148,10 +149,13 @@ as.data.table.list = function(x,
148149
}
149150
eachnrow[i] = NROW(xi) # for a vector (including list() columns) returns the length
150151
eachncol[i] = NCOL(xi) # for a vector returns 1
152+
if (is.atomic(xi) && length(xi)==0L && !is.null(xi)) {
153+
empty_atomic = TRUE # any empty atomic (not empty list()) should result in nrows=0L, #3727
154+
}
151155
}
152156
ncol = sum(eachncol) # hence removes NULL items silently (no error or warning), #842.
153157
if (ncol==0L) return(null.data.table())
154-
nrow = max(eachnrow)
158+
nrow = if (empty_atomic) 0L else max(eachnrow)
155159
ans = vector("list",ncol) # always return a new VECSXP
156160
recycle = function(x, nrow) {
157161
if (length(x)==nrow) {
@@ -173,8 +177,6 @@ as.data.table.list = function(x,
173177
if (is.null(xi)) { n_null = n_null+1L; next }
174178
if (eachnrow[i]>1L && nrow%%eachnrow[i]!=0L) # in future: eachnrow[i]!=nrow
175179
warning("Item ", i, " has ", eachnrow[i], " rows but longest item has ", nrow, "; recycled with remainder.")
176-
if (eachnrow[i]==0L && nrow>0L && is.atomic(xi)) # is.atomic to ignore list() since list() is a common way to initialize; let's not insist on list(NULL)
177-
warning("Item ", i, " has 0 rows but longest item has ", nrow, "; filled with NA") # the rep() in recycle() above creates the NA vector
178180
if (is.data.table(xi)) { # matrix and data.frame were coerced to data.table above
179181
prefix = if (!isFALSE(.named[i]) && isTRUE(nchar(names(x)[i])>0L)) paste0(names(x)[i],".") else "" # test 2058.12
180182
for (j in seq_along(xi)) {
@@ -219,7 +221,8 @@ as.data.table.data.frame = function(x, keep.rownames=FALSE, key=NULL, ...) {
219221
}
220222
if (any(vapply_1i(x, function(xi) length(dim(xi))))) { # not is.atomic because is.atomic(matrix) is true
221223
# a data.frame with a column that is data.frame needs to be expanded; test 2013.4
222-
return(as.data.table.list(x, keep.rownames=keep.rownames, ...))
224+
# x may be a class with [[ method that behaves differently, so as.list first for default [[, #4526
225+
return(as.data.table.list(as.list(x), keep.rownames=keep.rownames, ...))
223226
}
224227
ans = copy(x) # TO DO: change this deep copy to be shallow.
225228
setattr(ans, "row.names", .set_row_names(nrow(x)))

‎R/data.table.R‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,8 @@ replace_dot_alias = function(e) {
141141
return(ans)
142142
}
143143
if (!missing(verbose)) {
144-
stopifnot(isTRUEorFALSE(verbose))
144+
if (!is.integer(verbose) && !is.logical(verbose)) stop("verbose must be logical or integer")
145+
if (length(verbose)!=1 || anyNA(verbose)) stop("verbose must be length 1 non-NA")
145146
# set the global verbose option because that is fetched from C code without having to pass it through
146147
oldverbose = options(datatable.verbose=verbose)
147148
on.exit(options(oldverbose))
@@ -552,6 +553,11 @@ replace_dot_alias = function(e) {
552553
# i is not a data.table
553554
if (!is.logical(i) && !is.numeric(i)) stop("i has evaluated to type ", typeof(i), ". Expecting logical, integer or double.")
554555
if (is.logical(i)) {
556+
if (is.na(which)) { # #4411 i filter not optimized to join: DT[A > 1, which = NA]
557+
## we need this branch here, not below next to which=TRUE because irows=i=which(i) will filter out NAs: DT[A > 10, which = NA] will be incorrect
558+
if (notjoin) stop("internal error: notjoin and which=NA (non-matches), huh? please provide reproducible example to issue tracker") # nocov
559+
return(which(is.na(i) | !i))
560+
}
555561
if (length(i)==1L # to avoid unname copy when length(i)==nrow (normal case we don't want to slow down)
556562
&& isTRUE(unname(i))) { irows=i=NULL } # unname() for #2152 - length 1 named logical vector.
557563
# NULL is efficient signal to avoid creating 1:nrow(x) but still return all rows, fixes #1249
@@ -1339,7 +1345,9 @@ replace_dot_alias = function(e) {
13391345
}
13401346

13411347
if (is.data.table(jval)) {
1342-
setattr(jval, 'class', class(x)) # fix for #64
1348+
# should set the parent class only when jval is a plain data.table #4324
1349+
if (identical(class(jval), c('data.table', 'data.frame')))
1350+
setattr(jval, 'class', class(x)) # fix for #64
13431351
if (haskey(x) && all(key(x) %chin% names(jval)) && is.sorted(jval, by=key(x)))
13441352
setattr(jval, 'sorted', key(x))
13451353
if (any(sapply(jval, is.null))) stop("Internal error: j has created a data.table result containing a NULL column") # nocov
@@ -1384,7 +1392,8 @@ replace_dot_alias = function(e) {
13841392
byval = i
13851393
bynames = if (missing(on)) head(key(x),length(leftcols)) else names(on)
13861394
allbyvars = NULL
1387-
bysameorder = haskey(i) || (is.sorted(f__) && ((roll == FALSE) || length(f__) == 1L)) # Fix for #1010
1395+
bysameorder = (haskey(i) && identical(leftcols, chmatch(head(key(i),length(leftcols)), names(i)))) || # leftcols leading subset of key(i); see #4917
1396+
(roll==FALSE && is.sorted(f__)) # roll==FALSE is fix for #1010
13881397
## 'av' correct here ?? *** TO DO ***
13891398
xjisvars = intersect(av, names_x[rightcols]) # no "x." for xvars.
13901399
# if 'get' is in 'av' use all cols in 'i', fix for bug #34
@@ -2855,7 +2864,7 @@ ghead = function(x, n) .Call(Cghead, x, as.integer(n)) # n is not used at the mo
28552864
gtail = function(x, n) .Call(Cgtail, x, as.integer(n)) # n is not used at the moment
28562865
gfirst = function(x) .Call(Cgfirst, x)
28572866
glast = function(x) .Call(Cglast, x)
2858-
gsum = function(x, na.rm=FALSE) .Call(Cgsum, x, na.rm, TRUE) # warnOverflow=TRUE, #986
2867+
gsum = function(x, na.rm=FALSE) .Call(Cgsum, x, na.rm)
28592868
gmean = function(x, na.rm=FALSE) .Call(Cgmean, x, na.rm)
28602869
gprod = function(x, na.rm=FALSE) .Call(Cgprod, x, na.rm)
28612870
gmedian = function(x, na.rm=FALSE) .Call(Cgmedian, x, na.rm)
@@ -3117,7 +3126,7 @@ isReallyReal = function(x) {
31173126
}
31183127
idx_op = match(operators, ops, nomatch=0L)
31193128
if (any(idx_op %in% c(0L, 6L)))
3120-
stop("Invalid operators ", paste(operators[idx_op %in% c(0L, 6L)], collapse=","), ". Only allowed operators are ", paste(ops[1:5], collapse=""), ".")
3129+
stop(gettextf("Invalid join operators %s. Only allowed operators are %s.", brackify(operators[idx_op %in% c(0L, 6L)]), brackify(ops[1:5]), domain="R-data.table"), domain=NA)
31213130
## the final on will contain the xCol as name, the iCol as value
31223131
on = iCols
31233132
names(on) = xCols

‎R/fcast.R‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@ dcast <- function(
1717
else {
1818
data_name = deparse(substitute(data))
1919
ns = tryCatch(getNamespace("reshape2"), error=function(e)
20-
stop("The dcast generic in data.table has been passed a ",class(data)[1L],", but data.table::dcast currently only has a method for data.tables. Please confirm your input is a data.table, with setDT(", data_name, ") or as.data.table(", data_name, "). If you intend to use a reshape2::dcast, try installing that package first, but do note that reshape2 is deprecated and you should be migrating your code away from using it."))
21-
warning("The dcast generic in data.table has been passed a ", class(data)[1L], " and will attempt to redirect to the reshape2::dcast; please note that reshape2 is deprecated, and this redirection is now deprecated as well. Please do this redirection yourself like reshape2::dcast(", data_name, "). In the next version, this warning will become an error.")
20+
stop("The dcast generic in data.table has been passed a ",class(data)[1L],", but data.table::dcast currently only has a method for data.tables. Please confirm your input is a data.table, with setDT(", data_name, ") or as.data.table(", data_name, "). If you intend to use a reshape2::dcast, try installing that package first, but do note that reshape2 is superseded and is no longer actively developed."))
21+
warning("The dcast generic in data.table has been passed a ", class(data)[1L], " and will attempt to redirect to the reshape2::dcast; please note that reshape2 is superseded and is no longer actively developed, and this redirection is now deprecated. Please do this redirection yourself like reshape2::dcast(", data_name, "). In the next version, this warning will become an error.")
2222
ns$dcast(data, formula, fun.aggregate = fun.aggregate, ..., margins = margins,
2323
subset = subset, fill = fill, value.var = value.var)
2424
}

‎R/fmelt.R‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@ melt <- function(data, ..., na.rm = FALSE, value.name = "value") {
1212
} else {
1313
data_name = deparse(substitute(data))
1414
ns = tryCatch(getNamespace("reshape2"), error=function(e)
15-
stop("The melt generic in data.table has been passed a ",class(data)[1L],", but data.table::melt currently only has a method for data.tables. Please confirm your input is a data.table, with setDT(", data_name, ") or as.data.table(", data_name, "). If you intend to use a method from reshape2, try installing that package first, but do note that reshape2 is deprecated and you should be migrating your code away from using it."))
16-
warning("The melt generic in data.table has been passed a ", class(data)[1L], " and will attempt to redirect to the relevant reshape2 method; please note that reshape2 is deprecated, and this redirection is now deprecated as well. To continue using melt methods from reshape2 while both libraries are attached, e.g. melt.list, you can prepend the namespace like reshape2::melt(", data_name, "). In the next version, this warning will become an error.")
15+
stop("The melt generic in data.table has been passed a ",class(data)[1L],", but data.table::melt currently only has a method for data.tables. Please confirm your input is a data.table, with setDT(", data_name, ") or as.data.table(", data_name, "). If you intend to use a method from reshape2, try installing that package first, but do note that reshape2 is superseded and is no longer actively developed."))
16+
warning("The melt generic in data.table has been passed a ", class(data)[1L], " and will attempt to redirect to the relevant reshape2 method; please note that reshape2 is superseded and is no longer actively developed, and this redirection is now deprecated. To continue using melt methods from reshape2 while both libraries are attached, e.g. melt.list, you can prepend the namespace like reshape2::melt(", data_name, "). In the next version, this warning will become an error.")
1717
ns$melt(data, ..., na.rm=na.rm, value.name=value.name)
1818
}
1919
# nocov end

‎R/frank.R‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,13 @@ frankv = function(x, cols=seq_along(x), order=1L, na.last=TRUE, ties.method=c("a
2222
if (!length(cols))
2323
stop("x is a list, 'cols' can not be 0-length")
2424
}
25-
x = .shallow(x, cols) # shallow copy even if list..
25+
# need to unlock for #4429
26+
x = .shallow(x, cols, unlock = TRUE) # shallow copy even if list..
2627
setDT(x)
2728
cols = seq_along(cols)
2829
if (is.na(na.last)) {
30+
if ("..na_prefix.." %chin% names(x))
31+
stop("Input column '..na_prefix..' conflicts with data.table internal usage; please rename")
2932
set(x, j = "..na_prefix..", value = is_na(x, cols))
3033
order = if (length(order) == 1L) c(1L, rep(order, length(cols))) else c(1L, order)
3134
cols = c(ncol(x), cols)
@@ -39,6 +42,8 @@ frankv = function(x, cols=seq_along(x), order=1L, na.last=TRUE, ties.method=c("a
3942
idx = NULL
4043
n = nrow(x)
4144
}
45+
if ('..stats_runif..' %chin% names(x))
46+
stop("Input column '..stats_runif..' conflicts with data.table internal usage; please rename")
4247
set(x, idx, '..stats_runif..', stats::runif(n))
4348
order = if (length(order) == 1L) c(rep(order, length(cols)), 1L) else c(order, 1L)
4449
cols = c(cols, ncol(x))

‎R/fread.R‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@ yaml=FALSE, autostart=NA, tmpdir=tempdir(), tz="UTC")
2525
isTRUEorFALSE(verbose), isTRUEorFALSE(check.names), isTRUEorFALSE(logical01), isTRUEorFALSE(keepLeadingZeros), isTRUEorFALSE(yaml) )
2626
stopifnot( isTRUEorFALSE(stringsAsFactors) || (is.double(stringsAsFactors) && length(stringsAsFactors)==1L && 0.0<=stringsAsFactors && stringsAsFactors<=1.0))
2727
stopifnot( is.numeric(nrows), length(nrows)==1L )
28-
if (is.na(nrows) || nrows<0L) nrows=Inf # accept -1 to mean Inf, as read.table does
28+
nrows=as.double(nrows) #4686
29+
if (is.na(nrows) || nrows<0) nrows=Inf # accept -1 to mean Inf, as read.table does
2930
if (identical(header,"auto")) header=NA
3031
stopifnot(is.logical(header) && length(header)==1L) # TRUE, FALSE or NA
3132
stopifnot(is.numeric(nThread) && length(nThread)==1L)

0 commit comments

Comments
 (0)