diff --git a/Makefile.am b/Makefile.am index faf22521..1694e474 100644 --- a/Makefile.am +++ b/Makefile.am @@ -82,7 +82,12 @@ endif dist_man1_MANS = man/readstat.man man/extract_metadata.man -EXTRA_DIST = resources/datetime.sas7bdat +EXTRA_DIST = \ + resources/datetime.sas7bdat \ + resources/compression_type_0x02.sas7bdat \ + resources/compression_type_0x06.sas7bdat \ + resources/compression_type_0x09.sas7bdat \ + resources/compression_type_0x0d.sas7bdat include_HEADERS = src/readstat.h diff --git a/resources/compression_type_0x02.sas7bdat b/resources/compression_type_0x02.sas7bdat new file mode 100644 index 00000000..77e2bbca Binary files /dev/null and b/resources/compression_type_0x02.sas7bdat differ diff --git a/resources/compression_type_0x06.sas7bdat b/resources/compression_type_0x06.sas7bdat new file mode 100644 index 00000000..15ed94a2 Binary files /dev/null and b/resources/compression_type_0x06.sas7bdat differ diff --git a/resources/compression_type_0x09.sas7bdat b/resources/compression_type_0x09.sas7bdat new file mode 100644 index 00000000..18a1c28c Binary files /dev/null and b/resources/compression_type_0x09.sas7bdat differ diff --git a/resources/compression_type_0x0d.sas7bdat b/resources/compression_type_0x0d.sas7bdat new file mode 100644 index 00000000..8c047bf6 Binary files /dev/null and b/resources/compression_type_0x0d.sas7bdat differ diff --git a/src/sas/readstat_sas.h b/src/sas/readstat_sas.h index 5d589107..3b6d1ffa 100644 --- a/src/sas/readstat_sas.h +++ b/src/sas/readstat_sas.h @@ -121,10 +121,15 @@ typedef enum sas_subheader_type_e { #define SAS_PAGE_HEADER_SIZE_32BIT 24 #define SAS_PAGE_HEADER_SIZE_64BIT 40 -#define SAS_COMPRESSION_NONE 0x00 -#define SAS_COMPRESSION_TRUNC 0x01 -#define SAS_COMPRESSION_ROW 0x04 -#define SAS_COMPRESSION_DELETED_ROW 0x05 +#define SAS_COMPRESSION_NONE 0x00 +#define SAS_COMPRESSION_TRUNC 0x01 +#define SAS_COMPRESSION_NONE_MOVED 0x02 +#define SAS_COMPRESSION_REFERENCE 0x03 +#define SAS_COMPRESSION_ROW 0x04 +#define SAS_COMPRESSION_DELETED_ROW 0x05 +#define SAS_COMPRESSION_ROW_MOVED 0x06 +#define SAS_COMPRESSION_NONE_UNREFERENCED 0x09 +#define SAS_COMPRESSION_ROW_UNREFERENCED 0x0d #define SAS_COMPRESSION_SIGNATURE_RLE "SASYZCRL" #define SAS_COMPRESSION_SIGNATURE_RDC "SASYZCR2" diff --git a/src/sas/readstat_sas7bdat_read.c b/src/sas/readstat_sas7bdat_read.c index cca57e85..ec74a612 100644 --- a/src/sas/readstat_sas7bdat_read.c +++ b/src/sas/readstat_sas7bdat_read.c @@ -60,6 +60,9 @@ typedef struct sas7bdat_ctx_s { char *page; char *row; + char *moved_page; + int64_t moved_page_index; + uint64_t page_header_size; uint64_t subheader_signature_size; uint64_t subheader_pointer_size; @@ -120,6 +123,9 @@ static void sas7bdat_ctx_free(sas7bdat_ctx_t *ctx) { if (ctx->page) free(ctx->page); + if (ctx->moved_page) + free(ctx->moved_page); + if (ctx->row) free(ctx->row); @@ -935,7 +941,7 @@ static readstat_error_t sas7bdat_validate_subheader_pointer(subheader_pointer_t return READSTAT_ERROR_PARSE; if (shp_info->offset < ctx->page_header_size + subheader_count*ctx->subheader_pointer_size) return READSTAT_ERROR_PARSE; - if (shp_info->compression == SAS_COMPRESSION_NONE) { + if (shp_info->compression == SAS_COMPRESSION_NONE || shp_info->compression == SAS_COMPRESSION_NONE_MOVED) { if (shp_info->len < ctx->subheader_signature_size) return READSTAT_ERROR_PARSE; if (shp_info->offset + ctx->subheader_signature_size > page_size) @@ -945,6 +951,13 @@ static readstat_error_t sas7bdat_validate_subheader_pointer(subheader_pointer_t return READSTAT_OK; } +static int sas7bdat_is_moved_row(unsigned char compression) { + return compression == SAS_COMPRESSION_ROW_MOVED + || compression == SAS_COMPRESSION_NONE_MOVED + || compression == SAS_COMPRESSION_ROW_UNREFERENCED + || compression == SAS_COMPRESSION_NONE_UNREFERENCED; +} + /* First, extract column text */ static readstat_error_t sas7bdat_parse_page_pass1(const char *page, size_t page_size, sas7bdat_ctx_t *ctx) { readstat_error_t retval = READSTAT_OK; @@ -965,7 +978,7 @@ static readstat_error_t sas7bdat_parse_page_pass1(const char *page, size_t page_ if ((retval = sas7bdat_parse_subheader_pointer(shp, page + page_size - shp, &shp_info, ctx)) != READSTAT_OK) { goto cleanup; } - if (shp_info.len > 0 && shp_info.compression != SAS_COMPRESSION_TRUNC) { + if (shp_info.len > 0 && shp_info.compression != SAS_COMPRESSION_TRUNC && shp_info.compression != SAS_COMPRESSION_REFERENCE) { if ((retval = sas7bdat_validate_subheader_pointer(&shp_info, page_size, subheader_count, ctx)) != READSTAT_OK) { goto cleanup; } @@ -977,7 +990,8 @@ static readstat_error_t sas7bdat_parse_page_pass1(const char *page, size_t page_ goto cleanup; } } - } else if (shp_info.compression == SAS_COMPRESSION_ROW || shp_info.compression == SAS_COMPRESSION_DELETED_ROW) { + } else if (shp_info.compression == SAS_COMPRESSION_ROW || shp_info.compression == SAS_COMPRESSION_DELETED_ROW + || sas7bdat_is_moved_row(shp_info.compression)) { /* void */ } else { retval = READSTAT_ERROR_UNSUPPORTED_COMPRESSION; @@ -1012,6 +1026,117 @@ static readstat_error_t sas7bdat_parse_deleted_row_bitmap(const char *page, cons return READSTAT_OK; } +static readstat_error_t sas7bdat_parse_moved_row(uint64_t page_index, uint64_t subheader_index, sas7bdat_ctx_t *ctx) { + readstat_error_t retval = READSTAT_OK; + readstat_io_t *io = ctx->io; + + const uint64_t page_size = ctx->page_size; + int64_t saved_pos = -1; + + if (page_index >= ctx->page_count) { + retval = READSTAT_ERROR_PARSE; + goto cleanup; + } + + if (ctx->moved_page == NULL) { + if ((ctx->moved_page = readstat_malloc(page_size)) == NULL) { + retval = READSTAT_ERROR_MALLOC; + goto cleanup; + } + ctx->moved_page_index = -1; + } + char *page = ctx->moved_page; + + if (ctx->moved_page_index != (int64_t)page_index) { + saved_pos = io->seek(0, READSTAT_SEEK_CUR, io->io_ctx); + if (saved_pos == -1) { + retval = READSTAT_ERROR_SEEK; + goto cleanup; + } + if (io->seek(ctx->header_size + page_index * page_size, READSTAT_SEEK_SET, io->io_ctx) == -1) { + retval = READSTAT_ERROR_SEEK; + if (ctx->handle.error) { + snprintf(ctx->error_buf, sizeof(ctx->error_buf), "ReadStat: Failed to seek to position %" PRId64 + " (= %" PRId64 " + %" PRId64 "*%" PRId64 ")", + ctx->header_size + page_index * page_size, ctx->header_size, page_index, page_size); + ctx->handle.error(ctx->error_buf, ctx->user_ctx); + } + goto cleanup; + } + ssize_t bytes_read = io->read(page, page_size, io->io_ctx); + if (bytes_read < 0 || (uint64_t)bytes_read < page_size) { + retval = READSTAT_ERROR_READ; + goto cleanup; + } + ctx->moved_page_index = (int64_t)page_index; + } + + uint16_t page_type = sas_read2(&page[ctx->page_header_size - 8], ctx->bswap); + if ((page_type & SAS_PAGE_TYPE_MASK) == SAS_PAGE_TYPE_DATA || page_type & SAS_PAGE_TYPE_COMP) { + retval = READSTAT_ERROR_READ; + goto cleanup; + } + uint16_t subheader_count = sas_read2(&page[ctx->page_header_size - 4], ctx->bswap); + if (subheader_index >= subheader_count) { + retval = READSTAT_ERROR_READ; + goto cleanup; + } + uint64_t shp_offset = ctx->page_header_size + subheader_index * ctx->subheader_pointer_size; + if (shp_offset + ctx->subheader_pointer_size > page_size) { + retval = READSTAT_ERROR_READ; + goto cleanup; + } + + const char *shp = &page[shp_offset]; + subheader_pointer_t shp_info = { 0 }; + if ((retval = sas7bdat_parse_subheader_pointer(shp, page + page_size - shp, &shp_info, ctx)) != READSTAT_OK) { + goto cleanup; + } + if ((retval = sas7bdat_validate_subheader_pointer(&shp_info, page_size, subheader_count, ctx)) != READSTAT_OK) { + goto cleanup; + } + if ((retval = sas7bdat_submit_columns_if_needed(ctx, 1)) != READSTAT_OK) { + goto cleanup; + } + + if (shp_info.compression == SAS_COMPRESSION_NONE_MOVED) { + sas_subheader_type_t subheader_type = sas7bdat_parse_subheader_type(page + shp_info.offset, ctx); + if (!shp_info.is_compressed_data || subheader_type != SAS_SUBHEADER_TYPE_DATA) { + retval = READSTAT_ERROR_READ; + goto cleanup; + } + if (shp_info.len != ctx->row_length) { + retval = READSTAT_ERROR_ROW_WIDTH_MISMATCH; + goto cleanup; + } + if ((retval = sas7bdat_parse_single_row(page + shp_info.offset, ctx)) != READSTAT_OK) { + goto cleanup; + } + } else if (shp_info.compression == SAS_COMPRESSION_ROW_MOVED) { + if ((retval = sas7bdat_parse_subheader_compressed(page + shp_info.offset, shp_info.len, ctx)) != READSTAT_OK) { + goto cleanup; + } + } else { + retval = READSTAT_ERROR_UNSUPPORTED_COMPRESSION; + goto cleanup; + } + +cleanup: + + if (saved_pos != -1) { + if (io->seek(saved_pos, READSTAT_SEEK_SET, io->io_ctx) == -1 && retval == READSTAT_OK) { + retval = READSTAT_ERROR_SEEK; + if (ctx->handle.error) { + snprintf(ctx->error_buf, sizeof(ctx->error_buf), + "ReadStat: Failed to seek to position %" PRId64, saved_pos); + ctx->handle.error(ctx->error_buf, ctx->user_ctx); + } + } + } + + return retval; +} + static readstat_error_t sas7bdat_parse_page_pass2(const char *page, size_t page_size, sas7bdat_ctx_t *ctx) { uint16_t page_type; @@ -1041,7 +1166,13 @@ static readstat_error_t sas7bdat_parse_page_pass2(const char *page, size_t page_ if ((retval = sas7bdat_parse_subheader_pointer(shp, page + page_size - shp, &shp_info, ctx)) != READSTAT_OK) { goto cleanup; } - if (shp_info.len > 0 && shp_info.compression != SAS_COMPRESSION_TRUNC) { + if (shp_info.len > 0 && shp_info.compression == SAS_COMPRESSION_REFERENCE) { + uint64_t page_index = shp_info.offset - 1; + uint64_t subheader_index = shp_info.len - 1; + if ((retval = sas7bdat_parse_moved_row(page_index, subheader_index, ctx)) != READSTAT_OK) { + goto cleanup; + } + } else if (shp_info.len > 0 && shp_info.compression != SAS_COMPRESSION_TRUNC) { if ((retval = sas7bdat_validate_subheader_pointer(&shp_info, page_size, subheader_count, ctx)) != READSTAT_OK) { goto cleanup; } @@ -1076,6 +1207,8 @@ static readstat_error_t sas7bdat_parse_page_pass2(const char *page, size_t page_ if ((retval = sas7bdat_register_deleted_row(ctx)) != READSTAT_OK) { goto cleanup; } + } else if (sas7bdat_is_moved_row(shp_info.compression)) { + /* void */ } else { retval = READSTAT_ERROR_UNSUPPORTED_COMPRESSION; goto cleanup; @@ -1298,6 +1431,7 @@ readstat_error_t readstat_parse_sas7bdat(readstat_parser_t *parser, const char * ctx->user_ctx = user_ctx; ctx->io = parser->io; ctx->row_limit = parser->row_limit; + ctx->moved_page_index = -1; if (parser->row_offset > 0) ctx->row_offset = parser->row_offset; diff --git a/src/test/test_list.h b/src/test/test_list.h index 9b78c98d..a6e8526d 100644 --- a/src/test/test_list.h +++ b/src/test/test_list.h @@ -2383,6 +2383,160 @@ static rt_test_group_t _test_groups[] = { } } } + }, + + { + .resource_name = "compression_type_0x02.sas7bdat", + .label = "SAS file with moved uncompressed row", + .test_formats = RT_FORMAT_SAS7BDAT_64BIT_COMP_NONE, + .rows = 5, + .columns_count = 3, + .columns = { + { + .name = "id", + .type = READSTAT_TYPE_DOUBLE, + .values = { + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 1 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 2 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 3 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 4 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 5 } } + } + }, + { + .name = "category", + .type = READSTAT_TYPE_STRING, + .values = { + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_1" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_2" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_3" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_4" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_5" } } + } + }, + { + .name = "payload_to_update", + .type = READSTAT_TYPE_STRING, + .skip_value_comparison = 1 + } + } + }, + + { + .resource_name = "compression_type_0x06.sas7bdat", + .label = "SAS file with moved compressed row", + .test_formats = RT_FORMAT_SAS7BDAT_64BIT_COMP_ROWS, + .rows = 5, + .columns_count = 3, + .columns = { + { + .name = "id", + .type = READSTAT_TYPE_DOUBLE, + .values = { + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 1 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 2 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 3 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 4 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 5 } } + } + }, + { + .name = "category", + .type = READSTAT_TYPE_STRING, + .values = { + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_1" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_2" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_3" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_4" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_5" } } + } + }, + { + .name = "payload_to_update", + .type = READSTAT_TYPE_STRING, + .skip_value_comparison = 1 + } + } + }, + + { + .resource_name = "compression_type_0x09.sas7bdat", + .label = "SAS file with unreferenced uncompressed row", + .test_formats = RT_FORMAT_SAS7BDAT_64BIT_COMP_NONE, + .rows = 4, + .columns_count = 3, + .columns = { + { + .name = "id", + .type = READSTAT_TYPE_DOUBLE, + .values = { + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 1 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 2 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 4 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 5 } } + } + }, + { + .name = "category", + .type = READSTAT_TYPE_STRING, + .values = { + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_1" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_2" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_4" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_5" } } + } + }, + { + .name = "payload_to_update", + .type = READSTAT_TYPE_STRING, + .values = { + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "" } } + } + } + } + }, + + { + .resource_name = "compression_type_0x0d.sas7bdat", + .label = "SAS file with unreferenced compressed row", + .test_formats = RT_FORMAT_SAS7BDAT_64BIT_COMP_ROWS, + .rows = 4, + .columns_count = 3, + .columns = { + { + .name = "id", + .type = READSTAT_TYPE_DOUBLE, + .values = { + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 1 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 2 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 4 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 5 } } + } + }, + { + .name = "category", + .type = READSTAT_TYPE_STRING, + .values = { + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_1" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_2" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_4" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_5" } } + } + }, + { + .name = "payload_to_update", + .type = READSTAT_TYPE_STRING, + .values = { + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "" } } + } + } + } } } } diff --git a/src/test/test_read.c b/src/test/test_read.c index 1bd93895..d3ef0bb3 100644 --- a/src/test/test_read.c +++ b/src/test/test_read.c @@ -265,14 +265,16 @@ static int handle_value(int obs_index, readstat_variable_t *variable, readstat_v rt_column_t *column = &rt_ctx->file->columns[rt_ctx->var_index]; - if (column->type == READSTAT_TYPE_STRING_REF) { - push_error_if_strings_differ(rt_ctx, - rt_ctx->file->string_refs[readstat_int32_value(column->values[file_obs_index])], - readstat_string_value(value), "String ref values"); - } else { - push_error_if_values_differ(rt_ctx, - column->values[file_obs_index], - value, "Data values"); + if (!column->skip_value_comparison) { + if (column->type == READSTAT_TYPE_STRING_REF) { + push_error_if_strings_differ(rt_ctx, + rt_ctx->file->string_refs[readstat_int32_value(column->values[file_obs_index])], + readstat_string_value(value), "String ref values"); + } else { + push_error_if_values_differ(rt_ctx, + column->values[file_obs_index], + value, "Data values"); + } } return READSTAT_HANDLER_OK; diff --git a/src/test/test_types.h b/src/test/test_types.h index 57839333..d0862844 100644 --- a/src/test/test_types.h +++ b/src/test/test_types.h @@ -42,6 +42,8 @@ typedef struct rt_column_s { long missing_ranges_count; char label_set[RT_MAX_STRING]; + + int skip_value_comparison; } rt_column_t; typedef struct rt_test_file_s {