extend fread yaml support for other types

NEWS.md

@@ -88,6 +88,8 @@
 
 19. `print.data.table()` now correctly displays data when `col.names="none"` and `row.names=FALSE`, [#7735](https://github.com/Rdatatable/data.table/issues/7735). Thanks to @jan-swissre for the report and @venom1204 for the fix.
 
+20. `fread(yaml=TRUE)` now honors column types in the YAML schema, including empty columns written by `fwrite(yaml=TRUE)`, [#3999](https://github.com/Rdatatable/data.table/issues/3999). Unknown schema types now fall back to type inference even when no schema types are recognized. Thanks to @PavoDive and @Noskario for the report and @ben-schwen for the fix.
+
 ### Notes
 
 1. {data.table} now depends on R 3.5.0 (2018).

R/fread.R

@@ -289,15 +289,22 @@ yaml=FALSE, tmpdir=tempdir(), tz="UTC")
       new_types = sapply(yaml_header$schema$fields, `[[`, 'type')
       if (any(null_idx <- vapply_1b(new_types, is.null)))
         new_types = do.call(c, new_types)
-      synonms = rbindlist(list(
+      synonyms = rbindlist(list(
         character = list(syn = c('character', 'string')),
         integer = list(syn = c('integer', 'int')),
         numeric = list(syn = c('numeric', 'number', 'double')),
         factor = list(syn = c('factor', 'categorical')),
+        ordered = list(syn = 'ordered'),
+        logical = list(syn = 'logical'),
+        complex = list(syn = 'complex'),
+        Date = list(syn = 'Date'),
+        IDate = list(syn = 'IDate'),
+        ITime = list(syn = 'ITime'),
+        POSIXct = list(syn = 'POSIXct'),
         integer64 = list(syn = c('integer64', 'int64'))
       ), idcol = 'r_type')
-      setkeyv(synonms, 'syn')
-      new_types = synonms[list(new_types)]$r_type
+      setkeyv(synonyms, 'syn')
+      new_types = synonyms[list(new_types)]$r_type
       new_names = sapply(yaml_header$schema$fields[!null_idx], `[[`, 'name')
 
       if ('col.names' %chin% call_args) messagef("User-supplied column names in 'col.names' will override those found in YAML metadata.")
@@ -313,8 +320,8 @@ yaml=FALSE, tmpdir=tempdir(), tz="UTC")
               brackify(new_names[matched_name_idx[!idx_type]]))
           }
         }
-        # only add unmentioned columns
-        for (ii in which(!idx_name)) {
+        # only add unmentioned columns # dont override unknown types
+        for (ii in which(!idx_name & !is.na(new_types))) {
           colClasses[[ new_types[ii] ]] = c(colClasses[[ new_types[ii] ]], new_names[ii])
         }
       } else {
@@ -325,7 +332,8 @@ yaml=FALSE, tmpdir=tempdir(), tz="UTC")
           if (!'col.names' %chin% call_args) col.names = new_names
           new_names = paste0('V', seq_along(new_names))
         }
-        colClasses = tapply(new_names, new_types, c, simplify=FALSE)
+        # deactivate override for all unknown types
+        colClasses = if (all(is.na(new_types))) NULL else tapply(new_names, new_types, c, simplify=FALSE)
       }
     }
     sep_syn = c('sep', 'delimiter')
@@ -383,6 +391,7 @@ yaml=FALSE, tmpdir=tempdir(), tz="UTC")
     new_v = tryCatch({    # different to read.csv; i.e. won't error if a column won't coerce (fallback with warning instead)
       switch(new_class,
              "factor" = as_factor(v),
+             "ordered" = as.ordered(v),
              "complex" = as.complex(v),
              "raw" = as_raw(v),  # Internal implementation
              "Date" = as.Date(v),

inst/tests/other.Rraw

@@ -457,6 +457,22 @@ if (loaded[["yaml"]]) {  # csvy; #1701. Was 2032-2033 in tests.Rraw, #5516
   close(fcon)
   test(17.14, fread(f), DT)
   unlink(f)
+
+  # honor YAML column types even without values to infer, #3999
+  DT = data.table(date = as.Date('2006-05-01'), idate = as.IDate('2006-05-01'),
+                  time = as.ITime('12:00:00'), datetime = as.POSIXct('2006-05-01', tz='UTC'),
+                  ordered = ordered('a'), factor = factor('a'), logical = TRUE, complex = 1+2i)
+  f = tempfile()
+  fwrite(DT[0L], f, yaml=TRUE)
+  # we need the options since otherwise fread converts Date to IDate naturally
+  test(17.15, lapply(fread(f, yaml=TRUE), class), lapply(DT, class), options=c(datatable.old.fread.datetime.character=TRUE))
+  fwrite(DT, f, yaml=TRUE)
+  test(17.16, lapply(fread(f, yaml=TRUE), class), lapply(DT, class), options=c(datatable.old.fread.datetime.character=TRUE))
+  # unknown yaml schema types fall back to inference, even when none are recognized
+  DT = data.table(x = structure(1:2, class='unknown'))
+  fwrite(DT, f, yaml=TRUE)
+  test(17.17, lapply(fread(f, yaml=TRUE), class), list(x='integer'))
+  unlink(f)
 }
 
 if (loaded[["xts"]]) {  # was 1465 in tests.Rraw, #5516