From 6fe23edcb3e0ffc8d2d87615b2f9f8e6de03201b Mon Sep 17 00:00:00 2001 From: sdf-jkl Date: Fri, 10 Jul 2026 16:57:57 -0400 Subject: [PATCH] Mark shredded_variant cases 41, 131, 132, 138 as INVALID (missing 'value' column) These cases omit the required 'value' column from a variant group (cases 41, 131, 138 at the top level; case 132 in the shredded object field groups) but were labeled as valid cases. Per the discussion in apache/parquet-format#591, the spec requires the 'value' field to always be present, so these files are not spec-compliant. Relabel them following the existing convention used by cases 43, 84 and 125: '-INVALID' filenames plus a 'notes' entry stating that implementations can choose to error or read the shredded value. The expected variant outputs are kept for implementations that choose to read them. Fixes #116 Co-Authored-By: Claude Fable 5 --- ...e-041.parquet => case-041-INVALID.parquet} | Bin ...bin => case-041-INVALID_row-0.variant.bin} | Bin ...e-131.parquet => case-131-INVALID.parquet} | Bin ...bin => case-131-INVALID_row-0.variant.bin} | Bin ...e-132.parquet => case-132-INVALID.parquet} | Bin ...bin => case-132-INVALID_row-0.variant.bin} | Bin ...e-138.parquet => case-138-INVALID.parquet} | Bin ...bin => case-138-INVALID_row-0.variant.bin} | Bin shredded_variant/cases.json | 20 +++++++++++------- 9 files changed, 12 insertions(+), 8 deletions(-) rename shredded_variant/{case-041.parquet => case-041-INVALID.parquet} (100%) rename shredded_variant/{case-041_row-0.variant.bin => case-041-INVALID_row-0.variant.bin} (100%) rename shredded_variant/{case-131.parquet => case-131-INVALID.parquet} (100%) rename shredded_variant/{case-131_row-0.variant.bin => case-131-INVALID_row-0.variant.bin} (100%) rename shredded_variant/{case-132.parquet => case-132-INVALID.parquet} (100%) rename shredded_variant/{case-132_row-0.variant.bin => case-132-INVALID_row-0.variant.bin} (100%) rename shredded_variant/{case-138.parquet => case-138-INVALID.parquet} (100%) rename shredded_variant/{case-138_row-0.variant.bin => case-138-INVALID_row-0.variant.bin} (100%) diff --git a/shredded_variant/case-041.parquet b/shredded_variant/case-041-INVALID.parquet similarity index 100% rename from shredded_variant/case-041.parquet rename to shredded_variant/case-041-INVALID.parquet diff --git a/shredded_variant/case-041_row-0.variant.bin b/shredded_variant/case-041-INVALID_row-0.variant.bin similarity index 100% rename from shredded_variant/case-041_row-0.variant.bin rename to shredded_variant/case-041-INVALID_row-0.variant.bin diff --git a/shredded_variant/case-131.parquet b/shredded_variant/case-131-INVALID.parquet similarity index 100% rename from shredded_variant/case-131.parquet rename to shredded_variant/case-131-INVALID.parquet diff --git a/shredded_variant/case-131_row-0.variant.bin b/shredded_variant/case-131-INVALID_row-0.variant.bin similarity index 100% rename from shredded_variant/case-131_row-0.variant.bin rename to shredded_variant/case-131-INVALID_row-0.variant.bin diff --git a/shredded_variant/case-132.parquet b/shredded_variant/case-132-INVALID.parquet similarity index 100% rename from shredded_variant/case-132.parquet rename to shredded_variant/case-132-INVALID.parquet diff --git a/shredded_variant/case-132_row-0.variant.bin b/shredded_variant/case-132-INVALID_row-0.variant.bin similarity index 100% rename from shredded_variant/case-132_row-0.variant.bin rename to shredded_variant/case-132-INVALID_row-0.variant.bin diff --git a/shredded_variant/case-138.parquet b/shredded_variant/case-138-INVALID.parquet similarity index 100% rename from shredded_variant/case-138.parquet rename to shredded_variant/case-138-INVALID.parquet diff --git a/shredded_variant/case-138_row-0.variant.bin b/shredded_variant/case-138-INVALID_row-0.variant.bin similarity index 100% rename from shredded_variant/case-138_row-0.variant.bin rename to shredded_variant/case-138-INVALID_row-0.variant.bin diff --git a/shredded_variant/cases.json b/shredded_variant/cases.json index be8c131..1ae06c0 100644 --- a/shredded_variant/cases.json +++ b/shredded_variant/cases.json @@ -236,8 +236,9 @@ }, { "case_number" : 41, "test" : "testArrayMissingValueColumn", - "parquet_file" : "case-041.parquet", - "variant_file" : "case-041_row-0.variant.bin", + "notes" : "This parquet file is not valid according to the spec (the variant group omits the required 'value' column) and implementations can choose to error, or read the shredded value", + "parquet_file" : "case-041-INVALID.parquet", + "variant_file" : "case-041-INVALID_row-0.variant.bin", "variant" : "Variant(metadata=VariantMetadata(dict={}), value=VariantArray([Variant(type=STRING, value=comedy), Variant(type=STRING, value=drama)]))" }, { "case_number" : 42, @@ -775,14 +776,16 @@ }, { "case_number" : 131, "test" : "testMissingValueColumn", - "parquet_file" : "case-131.parquet", - "variant_file" : "case-131_row-0.variant.bin", + "notes" : "This parquet file is not valid according to the spec (the variant group omits the required 'value' column) and implementations can choose to error, or read the shredded value", + "parquet_file" : "case-131-INVALID.parquet", + "variant_file" : "case-131-INVALID_row-0.variant.bin", "variant" : "Variant(metadata=VariantMetadata(dict={}), value=Variant(type=INT32, value=34))" }, { "case_number" : 132, "test" : "testShreddedObjectMissingFieldValueColumn", - "parquet_file" : "case-132.parquet", - "variant_file" : "case-132_row-0.variant.bin", + "notes" : "This parquet file is not valid according to the spec (the shredded object field groups omit the required 'value' column) and implementations can choose to error, or read the shredded value", + "parquet_file" : "case-132-INVALID.parquet", + "variant_file" : "case-132-INVALID_row-0.variant.bin", "variant" : "Variant(metadata=VariantMetadata(dict={0 => a, 1 => b, 2 => c, 3 => d, 4 => e}), value=VariantObject(fields={b: Variant(type=STRING, value=iceberg)}))" }, { "case_number" : 133, @@ -816,7 +819,8 @@ }, { "case_number" : 138, "test" : "testShreddedObjectMissingValueColumn", - "parquet_file" : "case-138.parquet", - "variant_file" : "case-138_row-0.variant.bin", + "notes" : "This parquet file is not valid according to the spec (the variant group omits the required 'value' column) and implementations can choose to error, or read the shredded value", + "parquet_file" : "case-138-INVALID.parquet", + "variant_file" : "case-138-INVALID_row-0.variant.bin", "variant" : "Variant(metadata=VariantMetadata(dict={0 => a, 1 => b, 2 => c, 3 => d, 4 => e}), value=VariantObject(fields={a: Variant(type=INT16, value=1234), b: Variant(type=STRING, value=iceberg)}))" } ] \ No newline at end of file