From ddf4d679bf205f8208e165d3f38901bd1ab1543c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henning=20P=C3=B6ttker?= <25299532+hpoettker@users.noreply.github.com> Date: Fri, 11 Sep 2026 19:20:23 +0200 Subject: [PATCH] Read 64-bit multi-page SAS catalog files and skip format default values --- Makefile.am | 3 +- resources/format_with_default.sas7bcat | Bin 0 -> 24576 bytes src/sas/readstat_sas7bcat_read.c | 118 +++++++++++++++++-------- src/sas/readstat_sas7bcat_write.c | 114 ++++++++++++++++-------- src/test/test_list.h | 46 ++++++++++ src/test/test_read.c | 4 +- src/test/test_readstat.h | 8 +- src/test/test_write.c | 1 + 8 files changed, 215 insertions(+), 79 deletions(-) create mode 100644 resources/format_with_default.sas7bcat diff --git a/Makefile.am b/Makefile.am index 1694e474..8b9156c2 100644 --- a/Makefile.am +++ b/Makefile.am @@ -87,7 +87,8 @@ EXTRA_DIST = \ resources/compression_type_0x02.sas7bdat \ resources/compression_type_0x06.sas7bdat \ resources/compression_type_0x09.sas7bdat \ - resources/compression_type_0x0d.sas7bdat + resources/compression_type_0x0d.sas7bdat \ + resources/format_with_default.sas7bcat include_HEADERS = src/readstat.h diff --git a/resources/format_with_default.sas7bcat b/resources/format_with_default.sas7bcat new file mode 100644 index 0000000000000000000000000000000000000000..389a89473909f40f186c223ff70689b36e4f9bcf GIT binary patch literal 24576 zcmeI3%WD%+6o+q~jbL$tT_^}%bkS}SCQSOsLU1feEhJ6JL}(UbK?EO=Qbg=dx^O2h z+*tnv!JP|TD6JKAQxNxpT3vOaU34kb`#Ljb!q7yj6#Ba%Ip?0&+fsMG6XNByj`>lwNmv)`_58eX2pDAB5&tL zN1c30==ncfN?n>5yE0Y`tn#=!n-8DTGw^6_eK>hvY8~qf2Vo8ZAOHd&00JNY0w4ea zAOHd&00JNY0w4eaAOHd&00JNY0w4eaAOHd&00RGrz+2O|4wv_v`Bs9(+5EAvA9D}@ z0T2KI5C8!X009sH0T2KI5CDO{C*ZZbM%iod*{KWD&6+zKRVi*9+X*Wp)k?ilbDNyE z%HB*pDoNNW&q}eB3|kZD|1a)EC(KnRFGhP0m**70lW$wxYaj>RZ}5Fk&*{1eh1zLt z55H0zVhvOeUmhYs00ck)1V8`;KmY^|B7rx1oc{-FPOE>o!Fj_MwI|?a@Yg&D zg$4p300JNY0w4eaAOHd&00JNY0w8dp2)v8f|KGjHadrM5^u>Oe>jApsrS=_&29nOp z4eKshO!VzW*noKGL0xgWijdvPEkAj@c#it$s-y_zCVt+?u&_k?eL(gEsbDz3%#+L( z8dW8^x~<*5{JAT}r;mD`UFpoCL#&yaBv)n9?vLwwp4mvTqaq?L)^(|UtG#fyEjC!l utr=<23vFGyEoPEJS}b{HK`0m?00JNY0w4eaAOHd&00JNY0wC}o3H%1r-kNRz literal 0 HcmV?d00001 diff --git a/src/sas/readstat_sas7bcat_read.c b/src/sas/readstat_sas7bcat_read.c index 20eddb67..a9d8e9cb 100644 --- a/src/sas/readstat_sas7bcat_read.c +++ b/src/sas/readstat_sas7bcat_read.c @@ -43,16 +43,25 @@ static void sas7bcat_ctx_free(sas7bcat_ctx_t *ctx) { } static readstat_error_t sas7bcat_parse_value_labels(const char *value_start, size_t value_labels_len, - int label_count_used, int label_count_capacity, const char *name, sas7bcat_ctx_t *ctx) { + int label_count_used, int label_count_capacity, int default_label_pos, int string_offset, + const char *name, sas7bcat_ctx_t *ctx) { readstat_error_t retval = READSTAT_OK; int i; const char *lbp1 = value_start; - uint32_t *value_offset = readstat_calloc(label_count_used, sizeof(uint32_t)); + uint32_t *value_offset = NULL; /* Doubles appear to be stored as big-endian, always */ int bswap_doubles = machine_is_little_endian(); - int is_string = (name[0] == '$'); + int is_string = 0; char *label = NULL; + char *string_val = NULL; + if (name[0] == '$') { + is_string = 1; + } else if (name[0] == '@') { + goto cleanup; // Skip informats + } + + value_offset = readstat_calloc(label_count_used, sizeof(uint32_t)); if (value_offset == NULL) { retval = READSTAT_ERROR_MALLOC; goto cleanup; @@ -70,7 +79,7 @@ static readstat_error_t sas7bcat_parse_value_labels(const char *value_start, siz goto cleanup; } uint32_t label_pos = sas_read4(&lbp1[10+ctx->pad1], ctx->bswap); - if (label_pos >= label_count_used) { + if (label_pos >= label_count_used || (default_label_pos > 0 && label_pos == default_label_pos - 1)) { retval = READSTAT_ERROR_PARSE; goto cleanup; } @@ -82,52 +91,69 @@ static readstat_error_t sas7bcat_parse_value_labels(const char *value_start, siz const char *lbp2 = lbp1; /* Pass 2 -- parse pairs of values & labels */ - for (i=0; i 0 && i == default_label_pos - 1; + if (!is_default_label) { + lbp1 = value_start + value_offset[i]; - if (&lbp1[30] - value_start > value_labels_len || - &lbp2[10] - value_start > value_labels_len) { - retval = READSTAT_ERROR_PARSE; - goto cleanup; - } - readstat_value_t value = { .type = is_string ? READSTAT_TYPE_STRING : READSTAT_TYPE_DOUBLE }; - char string_val[4*16+1]; - if (is_string) { - size_t value_entry_len = 6 + sas_read2(&lbp1[2], ctx->bswap); - retval = readstat_convert(string_val, sizeof(string_val), - &lbp1[value_entry_len-16], 16, ctx->converter); - if (retval != READSTAT_OK) + if (&lbp1[30] - value_start > value_labels_len) { + retval = READSTAT_ERROR_PARSE; goto cleanup; + } + value.type = is_string ? READSTAT_TYPE_STRING : READSTAT_TYPE_DOUBLE; + if (is_string) { + size_t value_entry_len = 6 + sas_read2(&lbp1[2], ctx->bswap); + size_t string_start = 22 + string_offset; + if (value_entry_len < string_start) { + retval = READSTAT_ERROR_PARSE; + goto cleanup; + } + if (&lbp1[value_entry_len] - value_start > value_labels_len) { + retval = READSTAT_ERROR_PARSE; + goto cleanup; + } + size_t value_len = value_entry_len - string_start; + string_val = readstat_realloc(string_val, 4 * value_len + 1); + if (string_val == NULL) { + retval = READSTAT_ERROR_MALLOC; + goto cleanup; + } + retval = readstat_convert(string_val, 4 * value_len + 1, + &lbp1[string_start], value_len, ctx->converter); + if (retval != READSTAT_OK) + goto cleanup; - value.v.string_value = string_val; - } else { - uint64_t val = sas_read8(&lbp1[22], bswap_doubles); - double dval = NAN; - if ((val | 0xFF0000000000) == 0xFFFFFFFFFFFF) { - sas_assign_tag(&value, (val >> 40)); + value.v.string_value = string_val; } else { - memcpy(&dval, &val, 8); - if (dval > 0.0) { - val = ~val; - memcpy(&dval, &val, 8); + uint64_t val = sas_read8(&lbp1[22], bswap_doubles); + double dval = NAN; + if ((val | 0xFF0000000000) == 0xFFFFFFFFFFFF) { + sas_assign_tag(&value, (val >> 40)); } else { - dval *= -1.0; + memcpy(&dval, &val, 8); + if (dval > 0.0) { + val = ~val; + memcpy(&dval, &val, 8); + } else { + dval *= -1.0; + } } - } - value.v.double_value = dval; + value.v.double_value = dval; + } } - size_t label_len = sas_read2(&lbp2[8], ctx->bswap); if (&lbp2[10] > value_start + value_labels_len) { retval = READSTAT_ERROR_PARSE; goto cleanup; } + size_t label_len = sas_read2(&lbp2[8], ctx->bswap); /* Some labels seem to overflow the reported block length, truncate it */ /* (Observed with formats.sasbcat from GSS2021, produced with 9.0401M6X64_SR12R2 */ if (label_len > value_start + value_labels_len - &lbp2[10]) { label_len = value_start + value_labels_len - &lbp2[10]; } - if (ctx->value_label_handler) { + if (ctx->value_label_handler && !is_default_label) { label = realloc(label, 4 * label_len + 1); retval = readstat_convert(label, 4 * label_len + 1, &lbp2[10], label_len, ctx->converter); @@ -144,6 +170,7 @@ static readstat_error_t sas7bcat_parse_value_labels(const char *value_start, siz } cleanup: + free(string_val); free(label); free(value_offset); return retval; @@ -155,23 +182,37 @@ static readstat_error_t sas7bcat_parse_block(const char *data, size_t data_size, size_t pad = 0; uint64_t label_count_capacity = 0; uint64_t label_count_used = 0; + uint64_t default_label_pos = 0; + uint32_t string_offset = 0; int payload_offset = 106; uint16_t flags = 0; char name[4*32+1]; + if (ctx->u64) + payload_offset += 32; if (data_size < payload_offset) goto cleanup; flags = sas_read2(&data[2], ctx->bswap); pad = (flags & 0x08) ? 4 : 0; // might be 0x10, not sure + if (pad && data_size < payload_offset + pad) + goto cleanup; + if (ctx->u64) { label_count_capacity = sas_read8(&data[42+pad], ctx->bswap); label_count_used = sas_read8(&data[50+pad], ctx->bswap); - - payload_offset += 32; + default_label_pos = sas_read8(&data[74+pad], ctx->bswap); + string_offset = sas_read4(&data[124+pad], ctx->bswap); } else { label_count_capacity = sas_read4(&data[38+pad], ctx->bswap); label_count_used = sas_read4(&data[42+pad], ctx->bswap); + default_label_pos = sas_read4(&data[66+pad], ctx->bswap); + string_offset = sas_read2(&data[104+pad], ctx->bswap); + } + + if (default_label_pos > label_count_used) { + retval = READSTAT_ERROR_PARSE; + goto cleanup; } if ((retval = readstat_convert(name, sizeof(name), &data[8], 8, ctx->converter)) != READSTAT_OK) @@ -198,7 +239,7 @@ static readstat_error_t sas7bcat_parse_block(const char *data, size_t data_size, goto cleanup; if ((retval = sas7bcat_parse_value_labels(&data[payload_offset+pad], data_size - payload_offset - pad, - label_count_used, label_count_capacity, name, ctx)) != READSTAT_OK) + label_count_used, label_count_capacity, default_label_pos, string_offset, name, ctx)) != READSTAT_OK) goto cleanup; cleanup: @@ -479,8 +520,9 @@ readstat_error_t readstat_parse_sas7bcat(readstat_parser_t *parser, const char * retval = READSTAT_ERROR_READ; goto cleanup; } - if (memcmp(&page[16], "XLSR", sizeof("XLSR")-1) == 0) { - retval = sas7bcat_augment_index(&page[16], ctx->page_size - 16, ctx); + int8_t offset = ctx->u64 ? 32 : 16; + if (memcmp(&page[offset], "XLSR", sizeof("XLSR")-1) == 0) { + retval = sas7bcat_augment_index(&page[offset], ctx->page_size - offset, ctx); if (retval != READSTAT_OK) goto cleanup; } diff --git a/src/sas/readstat_sas7bcat_write.c b/src/sas/readstat_sas7bcat_write.c index da9a84da..f8611ca6 100644 --- a/src/sas/readstat_sas7bcat_write.c +++ b/src/sas/readstat_sas7bcat_write.c @@ -14,26 +14,31 @@ typedef struct sas7bcat_block_s { char data[1]; // Flexible array; use [1] for C++-98 compatibility } sas7bcat_block_t; -static sas7bcat_block_t *sas7bcat_block_for_label_set(readstat_label_set_t *r_label_set) { - size_t len = 0; +static sas7bcat_block_t *sas7bcat_block_for_label_set( + readstat_label_set_t *r_label_set, readstat_writer_t *writer) { + size_t len = 106; size_t name_len = strlen(r_label_set->name); int j; char name[32]; - len += 106; - if (name_len > 8) { len += 32; // long name if (name_len > 32) { name_len = 32; } } + if (writer->is_64bit) { + len += 32; + } memcpy(&name[0], r_label_set->name, name_len); for (j=0; jvalue_labels_count; j++) { readstat_value_label_t *value_label = readstat_get_value_label(r_label_set, j); - len += 30; // Value: 14-byte header + 16-byte padded value + len += 38; // Value: 22-byte header + 16-byte padded value + if (r_label_set->type == READSTAT_TYPE_STRING && value_label->string_key_len > 16) { + len += value_label->string_key_len - 16; + } len += 8 + 2 + value_label->label_len + 1; } @@ -42,16 +47,24 @@ static sas7bcat_block_t *sas7bcat_block_for_label_set(readstat_label_set_t *r_la block->len = len; off_t begin = 106; - int32_t count = r_label_set->value_labels_count; - memcpy(&block->data[38], &count, sizeof(int32_t)); - memcpy(&block->data[42], &count, sizeof(int32_t)); + int64_t count = r_label_set->value_labels_count; + if (writer->is_64bit) { + memcpy(&block->data[42], &count, sizeof(int64_t)); + memcpy(&block->data[50], &count, sizeof(int64_t)); + } else { + memcpy(&block->data[38], &count, sizeof(int32_t)); + memcpy(&block->data[42], &count, sizeof(int32_t)); + } + if (writer->is_64bit) { + begin += 32; + } if (name_len > 8) { - int16_t flags = 0x80; + int16_t flags = writer->is_64bit ? 0x20 : 0x80; memcpy(&block->data[2], &flags, sizeof(int16_t)); memcpy(&block->data[8], name, 8); - memset(&block->data[106], ' ', 32); - memcpy(&block->data[106], name, name_len); + memset(&block->data[begin], ' ', 32); + memcpy(&block->data[begin], name, name_len); begin += 32; } else { @@ -60,20 +73,19 @@ static sas7bcat_block_t *sas7bcat_block_for_label_set(readstat_label_set_t *r_la } char *lbp1 = &block->data[begin]; - char *lbp2 = &block->data[begin+r_label_set->value_labels_count*30]; for (j=0; jvalue_labels_count; j++) { readstat_value_label_t *value_label = readstat_get_value_label(r_label_set, j); - int16_t value_entry_len = 24; // size - 6 - memcpy(&lbp1[2], &value_entry_len, sizeof(int16_t)); + int16_t value_entry_len = 32; // size - 6 int32_t index = j; memcpy(&lbp1[10], &index, sizeof(int32_t)); if (r_label_set->type == READSTAT_TYPE_STRING) { size_t string_len = value_label->string_key_len; - if (string_len > 16) - string_len = 16; - memset(&lbp1[14], ' ', 16); - memcpy(&lbp1[14], value_label->string_key, string_len); + if (string_len > 16) { + value_entry_len += string_len - 16; + } + memset(&lbp1[22], ' ', 16); + memcpy(&lbp1[22], value_label->string_key, string_len); } else { uint64_t big_endian_value; double double_value = value_label->double_key; @@ -90,11 +102,18 @@ static sas7bcat_block_t *sas7bcat_block_for_label_set(readstat_label_set_t *r_la memcpy(&lbp1[22], &big_endian_value, sizeof(uint64_t)); } + memcpy(&lbp1[2], &value_entry_len, sizeof(int16_t)); + lbp1 += 6 + value_entry_len; + } + + char *lbp2 = lbp1; + for (j=0; jvalue_labels_count; j++) { + readstat_value_label_t *value_label = readstat_get_value_label(r_label_set, j); + int16_t label_len = value_label->label_len; memcpy(&lbp2[8], &label_len, sizeof(int16_t)); memcpy(&lbp2[10], value_label->label, label_len); - lbp1 += 6 + value_entry_len; lbp2 += 8 + 2 + value_label->label_len + 1; } @@ -122,12 +141,12 @@ static readstat_error_t sas7bcat_begin_data(void *writer_ctx) { readstat_error_t retval = READSTAT_OK; int i; - sas_header_info_t *hinfo = sas_header_info_init(writer, 0); + sas_header_info_t *hinfo = sas_header_info_init(writer, writer->is_64bit); sas7bcat_block_t **blocks = malloc(writer->label_sets_count * sizeof(sas7bcat_block_t)); char *page = malloc(hinfo->page_size); for (i=0; ilabel_sets_count; i++) { - blocks[i] = sas7bcat_block_for_label_set(writer->label_sets[i]); + blocks[i] = sas7bcat_block_for_label_set(writer->label_sets[i], writer); } hinfo->page_count = 4; @@ -145,23 +164,37 @@ static readstat_error_t sas7bcat_begin_data(void *writer_ctx) { memset(page, '\0', hinfo->page_size); // Page 1 - char *xlsr = &page[856]; - int32_t block_idx = 4; + int64_t xlsr_size = 212; + int64_t xlsr_offset = 856; + int64_t xlsr_O_offset = 50; + if (writer->is_64bit) { + xlsr_size += 72; + xlsr_offset += 144; + xlsr_O_offset += 24; + } + + char *xlsr = &page[xlsr_offset]; + int64_t block_idx = 4; int16_t block_off = 16; for (i=0; ilabel_sets_count; i++) { - if (xlsr + 212 > page + hinfo->page_size) + if (xlsr + xlsr_size > page + hinfo->page_size) break; memcpy(&xlsr[0], "XLSR", 4); - memcpy(&xlsr[4], &block_idx, sizeof(int32_t)); - memcpy(&xlsr[8], &block_off, sizeof(int16_t)); + if (writer->is_64bit) { + memcpy(&xlsr[8], &block_idx, sizeof(int64_t)); + memcpy(&xlsr[16], &block_off, sizeof(int16_t)); + } else { + memcpy(&xlsr[4], &block_idx, sizeof(int32_t)); + memcpy(&xlsr[8], &block_off, sizeof(int16_t)); + } - xlsr[50] = 'O'; + xlsr[xlsr_O_offset] = 'O'; block_off += blocks[i]->len; - xlsr += 212; + xlsr += xlsr_size; } retval = readstat_write_bytes(writer, page, hinfo->page_size); @@ -176,23 +209,32 @@ static readstat_error_t sas7bcat_begin_data(void *writer_ctx) { // Page 3 memset(page, '\0', hinfo->page_size); - char block_header[16]; + char block_header[32]; + size_t block_header_size = writer->is_64bit ? 32 : 16; block_off = 16; for (i=0; ilabel_sets_count; i++) { - if (block_off + sizeof(block_header) + blocks[i]->len > hinfo->page_size) + if (block_off + block_header_size + blocks[i]->len > hinfo->page_size) break; - memset(block_header, '\0', sizeof(block_header)); + memset(block_header, '\0', block_header_size); + + off_t next_page_offset = 0; + off_t next_page_pos_offset = 4; + off_t block_len_offset = 6; + if (writer->is_64bit) { + next_page_pos_offset += 4; + block_len_offset += 4; + } int32_t next_page = 0; int16_t next_off = 0; int16_t block_len = blocks[i]->len; - memcpy(&block_header[0], &next_page, sizeof(int32_t)); - memcpy(&block_header[4], &next_off, sizeof(int16_t)); - memcpy(&block_header[6], &block_len, sizeof(int16_t)); + memcpy(&block_header[next_page_offset], &next_page, sizeof(int32_t)); + memcpy(&block_header[next_page_pos_offset], &next_off, sizeof(int16_t)); + memcpy(&block_header[block_len_offset], &block_len, sizeof(int16_t)); - memcpy(&page[block_off], block_header, sizeof(block_header)); - block_off += sizeof(block_header); + memcpy(&page[block_off], block_header, block_header_size); + block_off += block_header_size; memcpy(&page[block_off], blocks[i]->data, blocks[i]->len); block_off += blocks[i]->len; diff --git a/src/test/test_list.h b/src/test/test_list.h index 687bd02c..62f2ea1a 100644 --- a/src/test/test_list.h +++ b/src/test/test_list.h @@ -1489,7 +1489,30 @@ static rt_test_group_t _test_groups[] = { { .value = { .type = READSTAT_TYPE_STRING, .v = { .string_value = "2" } }, .label = "Two" + } + } + } + } + }, + + { + .label = "Long SAS string value labels", + .test_formats = RT_FORMAT_SAS7BCAT, + .label_sets_count = 1, + .label_sets = { + { + .name = "$LongValueStringLabelSet", + .type = READSTAT_TYPE_STRING, + .value_labels_count = 2, + .value_labels = { + { + .value = { .type = READSTAT_TYPE_STRING, .v = { .string_value = "One" } }, + .label = "1" }, + { + .value = { .type = READSTAT_TYPE_STRING, .v = { .string_value = "abcdeABCDEabcdeAB" } }, + .label = "2" + } } } } @@ -2565,6 +2588,29 @@ static rt_test_group_t _test_groups[] = { } } } + }, + + { + .resource_name = "format_with_default.sas7bcat", + .test_formats = RT_FORMAT_SAS7BCAT_64BIT, + .label_sets_count = 1, + .label_sets = { + { + .name = "YESNO", + .type = READSTAT_TYPE_DOUBLE, + .value_labels_count = 2, + .value_labels = { + { + .value = { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 1.0 } }, + .label = "Yes" + }, + { + .value = { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 2.0 } }, + .label = "No" + } + } + } + } } } } diff --git a/src/test/test_read.c b/src/test/test_read.c index fbdd3684..f1dd5df6 100644 --- a/src/test/test_read.c +++ b/src/test/test_read.c @@ -39,8 +39,10 @@ char *file_extension(long format) { return "zsav"; if (format == RT_FORMAT_POR) return "por"; - if (format == RT_FORMAT_SAS7BCAT) + if (format == RT_FORMAT_SAS7BCAT_32BIT) return "sas7bcat"; + if (format == RT_FORMAT_SAS7BCAT_64BIT) + return "sas7bcat64"; if (format == RT_FORMAT_SAS7BDAT_32BIT_COMP_NONE) return "sas7bdat32"; if (format == RT_FORMAT_SAS7BDAT_32BIT_COMP_ROWS) diff --git a/src/test/test_readstat.h b/src/test/test_readstat.h index 0aa30af6..e985b19a 100644 --- a/src/test/test_readstat.h +++ b/src/test/test_readstat.h @@ -50,10 +50,12 @@ #define RT_FORMAT_SAS7BDAT (RT_FORMAT_SAS7BDAT_32BIT | RT_FORMAT_SAS7BDAT_64BIT) -#define RT_FORMAT_SAS7BCAT 0x100000 +#define RT_FORMAT_SAS7BCAT_32BIT 0x100000 +#define RT_FORMAT_SAS7BCAT_64BIT 0x200000 +#define RT_FORMAT_SAS7BCAT (RT_FORMAT_SAS7BCAT_32BIT | RT_FORMAT_SAS7BCAT_64BIT) -#define RT_FORMAT_XPORT_5 0x200000 -#define RT_FORMAT_XPORT_8 0x400000 +#define RT_FORMAT_XPORT_5 0x400000 +#define RT_FORMAT_XPORT_8 0x800000 #define RT_FORMAT_XPORT (RT_FORMAT_XPORT_5 | RT_FORMAT_XPORT_8) diff --git a/src/test/test_write.c b/src/test/test_write.c index 4a6b0501..6c782396 100644 --- a/src/test/test_write.c +++ b/src/test/test_write.c @@ -168,6 +168,7 @@ readstat_error_t write_file_to_buffer(rt_test_file_t *file, rt_buffer_t *buffer, readstat_writer_set_file_format_is_64bit(writer, !!(format & RT_FORMAT_SAS7BDAT_64BIT)); error = readstat_begin_writing_sas7bdat(writer, buffer, file->rows); } else if ((format & RT_FORMAT_SAS7BCAT)) { + readstat_writer_set_file_format_is_64bit(writer, !!(format & RT_FORMAT_SAS7BCAT_64BIT)); error = readstat_begin_writing_sas7bcat(writer, buffer); } else if ((format & RT_FORMAT_XPORT)) { readstat_writer_set_file_format_version(writer, sas_file_format_version(format));