diff --git a/NEWS.md b/NEWS.md index 95c0c74f1..944ded806 100644 --- a/NEWS.md +++ b/NEWS.md @@ -88,6 +88,8 @@ 19. `print.data.table()` now correctly displays data when `col.names="none"` and `row.names=FALSE`, [#7735](https://github.com/Rdatatable/data.table/issues/7735). Thanks to @jan-swissre for the report and @venom1204 for the fix. +20. `fread(yaml=TRUE)` now honors column types in the YAML schema, including empty columns written by `fwrite(yaml=TRUE)`, [#3999](https://github.com/Rdatatable/data.table/issues/3999). Unknown schema types now fall back to type inference even when no schema types are recognized. Thanks to @PavoDive and @Noskario for the report and @ben-schwen for the fix. + ### Notes 1. {data.table} now depends on R 3.5.0 (2018). diff --git a/R/fread.R b/R/fread.R index 4aa6a01ed..ec3613897 100644 --- a/R/fread.R +++ b/R/fread.R @@ -289,15 +289,22 @@ yaml=FALSE, tmpdir=tempdir(), tz="UTC") new_types = sapply(yaml_header$schema$fields, `[[`, 'type') if (any(null_idx <- vapply_1b(new_types, is.null))) new_types = do.call(c, new_types) - synonms = rbindlist(list( + synonyms = rbindlist(list( character = list(syn = c('character', 'string')), integer = list(syn = c('integer', 'int')), numeric = list(syn = c('numeric', 'number', 'double')), factor = list(syn = c('factor', 'categorical')), + ordered = list(syn = 'ordered'), + logical = list(syn = 'logical'), + complex = list(syn = 'complex'), + Date = list(syn = 'Date'), + IDate = list(syn = 'IDate'), + ITime = list(syn = 'ITime'), + POSIXct = list(syn = 'POSIXct'), integer64 = list(syn = c('integer64', 'int64')) ), idcol = 'r_type') - setkeyv(synonms, 'syn') - new_types = synonms[list(new_types)]$r_type + setkeyv(synonyms, 'syn') + new_types = synonyms[list(new_types)]$r_type new_names = sapply(yaml_header$schema$fields[!null_idx], `[[`, 'name') if ('col.names' %chin% call_args) messagef("User-supplied column names in 'col.names' will override those found in YAML metadata.") @@ -313,8 +320,8 @@ yaml=FALSE, tmpdir=tempdir(), tz="UTC") brackify(new_names[matched_name_idx[!idx_type]])) } } - # only add unmentioned columns - for (ii in which(!idx_name)) { + # only add unmentioned columns # dont override unknown types + for (ii in which(!idx_name & !is.na(new_types))) { colClasses[[ new_types[ii] ]] = c(colClasses[[ new_types[ii] ]], new_names[ii]) } } else { @@ -325,7 +332,8 @@ yaml=FALSE, tmpdir=tempdir(), tz="UTC") if (!'col.names' %chin% call_args) col.names = new_names new_names = paste0('V', seq_along(new_names)) } - colClasses = tapply(new_names, new_types, c, simplify=FALSE) + # deactivate override for all unknown types + colClasses = if (all(is.na(new_types))) NULL else tapply(new_names, new_types, c, simplify=FALSE) } } sep_syn = c('sep', 'delimiter') @@ -383,6 +391,7 @@ yaml=FALSE, tmpdir=tempdir(), tz="UTC") new_v = tryCatch({ # different to read.csv; i.e. won't error if a column won't coerce (fallback with warning instead) switch(new_class, "factor" = as_factor(v), + "ordered" = as.ordered(v), "complex" = as.complex(v), "raw" = as_raw(v), # Internal implementation "Date" = as.Date(v), diff --git a/inst/tests/other.Rraw b/inst/tests/other.Rraw index 928bc8e63..78b20dede 100644 --- a/inst/tests/other.Rraw +++ b/inst/tests/other.Rraw @@ -457,6 +457,22 @@ if (loaded[["yaml"]]) { # csvy; #1701. Was 2032-2033 in tests.Rraw, #5516 close(fcon) test(17.14, fread(f), DT) unlink(f) + + # honor YAML column types even without values to infer, #3999 + DT = data.table(date = as.Date('2006-05-01'), idate = as.IDate('2006-05-01'), + time = as.ITime('12:00:00'), datetime = as.POSIXct('2006-05-01', tz='UTC'), + ordered = ordered('a'), factor = factor('a'), logical = TRUE, complex = 1+2i) + f = tempfile() + fwrite(DT[0L], f, yaml=TRUE) + # we need the options since otherwise fread converts Date to IDate naturally + test(17.15, lapply(fread(f, yaml=TRUE), class), lapply(DT, class), options=c(datatable.old.fread.datetime.character=TRUE)) + fwrite(DT, f, yaml=TRUE) + test(17.16, lapply(fread(f, yaml=TRUE), class), lapply(DT, class), options=c(datatable.old.fread.datetime.character=TRUE)) + # unknown yaml schema types fall back to inference, even when none are recognized + DT = data.table(x = structure(1:2, class='unknown')) + fwrite(DT, f, yaml=TRUE) + test(17.17, lapply(fread(f, yaml=TRUE), class), list(x='integer')) + unlink(f) } if (loaded[["xts"]]) { # was 1465 in tests.Rraw, #5516