From 5bd1ca373d8b54e880de34a9c004a26a8d33ee19 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Mon, 27 Jul 2026 19:00:40 +0200 Subject: [PATCH 1/5] Iceberg deletion vector --- .../table-engines/integrations/iceberg.md | 11 +- docs/en/interfaces/formats.md | 2 + docs/en/interfaces/formats/Puffin/Puffin.md | 46 + .../formats/Puffin/PuffinMetadata.md | 38 + docs/en/sql-reference/statements/system.md | 4 + .../sql-reference/table-functions/iceberg.md | 17 +- programs/local/LocalServer.cpp | 15 + programs/server/Server.cpp | 15 + src/Access/Common/AccessType.h | 1 + .../AggregateFunctionGroupBitmapData.h | 7 + .../BuzzHouse/Generator/SessionSettings.cpp | 1 + src/Common/CurrentMetrics.cpp | 2 + src/Common/ProfileEvents.cpp | 5 + src/Core/Defines.h | 4 + src/Core/ServerSettings.cpp | 9 + src/Core/Settings.cpp | 8 + src/Core/SettingsChangesHistory.cpp | 70 + src/Formats/registerFormats.cpp | 4 + src/Interpreters/ClusterFunctionReadTask.cpp | 40 + src/Interpreters/Context.cpp | 48 + src/Interpreters/Context.h | 6 + src/Interpreters/InterpreterSystemQuery.cpp | 5 + src/Parsers/ASTSystemQuery.cpp | 1 + src/Parsers/ASTSystemQuery.h | 1 + src/Parsers/ParserSystemQuery.cpp | 1 + .../Impl/ParquetV3BlockInputFormat.cpp | 71 +- .../Formats/Impl/ParquetV3BlockInputFormat.h | 11 +- .../Formats/Impl/PuffinBlockInputFormat.cpp | 1179 +++++++++++++++++ .../Formats/Impl/PuffinBlockInputFormat.h | 83 ++ ...est_parquet_v3_need_only_count_buckets.cpp | 179 +++ .../tests/gtest_puffin_reset_parser.cpp | 130 ++ .../Common/AvroForIcebergDeserializer.cpp | 40 +- .../DataLakes/DataLakeObjectMetadata.h | 9 + .../DataLakes/DeletionVectorTransform.cpp | 10 + .../DataLakes/Iceberg/Constant.h | 2 + .../Iceberg/IcebergDataObjectInfo.cpp | 46 +- .../DataLakes/Iceberg/IcebergDataObjectInfo.h | 14 + .../Iceberg/IcebergDeletionVector.cpp | 161 +++ .../DataLakes/Iceberg/IcebergDeletionVector.h | 28 + .../DataLakes/Iceberg/IcebergIterator.cpp | 129 +- .../DataLakes/Iceberg/IcebergIterator.h | 4 +- .../DataLakes/Iceberg/IcebergMetadata.cpp | 25 +- .../DataLakes/Iceberg/ManifestFile.h | 18 +- .../DataLakes/Iceberg/Snapshot.h | 10 + .../tests/gtest_iceberg_count_shortcuts.cpp | 66 + .../gtest_iceberg_data_object_info_clone.cpp | 64 + ...test_iceberg_parquet_row_deletes_guard.cpp | 47 + .../DataLakes/PuffinDeletionVectorReader.cpp | 291 ++++ .../DataLakes/PuffinDeletionVectorReader.h | 22 + .../DataLakes/PuffinFilesCache.cpp | 110 ++ .../DataLakes/PuffinFilesCache.h | 138 ++ ...deletion_vector_before_equality_filter.cpp | 68 + .../gtest_has_non_empty_excluded_rows.cpp | 28 + .../gtest_puffin_deletion_vector_bounds.cpp | 63 + ...est_puffin_deletion_vector_cardinality.cpp | 96 ++ .../gtest_puffin_deletion_vector_envelope.cpp | 98 ++ .../tests/gtest_puffin_files_cache_clone.cpp | 87 ++ .../tests/gtest_puffin_files_cache_key.cpp | 33 + .../tests/gtest_puffin_files_cache_weight.cpp | 46 + .../ObjectStorage/IObjectIterator.cpp | 8 +- src/Storages/ObjectStorage/IObjectIterator.h | 4 + .../StorageObjectStorageSource.cpp | 54 +- .../tests/gtest_rendezvous_hashing.cpp | 239 ++++ .../test_deletion_vectors.py | 265 ++++ .../01271_show_privileges.reference | 1 + .../04077_puffin_happy_path.reference | 34 + .../0_stateless/04077_puffin_happy_path.sh | 61 + ...117_parser_system_query_variants.reference | 1 + .../04117_parser_system_query_variants.sql | 1 + .../0_stateless/04257_puffin_errors.reference | 38 + .../0_stateless/04257_puffin_errors.sh | 97 ++ .../04260_puffin_sparse_large_key.reference | 4 + .../04260_puffin_sparse_large_key.sh | 21 + .../04261_iceberg_deletion_vector.reference | 197 +++ .../04261_iceberg_deletion_vector.sh | 16 + ..._iceberg_deletion_vector_complex.reference | 172 +++ .../04262_iceberg_deletion_vector_complex.sh | 32 + ...04263_iceberg_puffin_files_cache.reference | 19 + .../04263_iceberg_puffin_files_cache.sh | 65 + .../04548_puffin_stdin_pipe.reference | 4 + .../0_stateless/04548_puffin_stdin_pipe.sh | 19 + .../04549_puffin_allow_seeks_off.reference | 6 + .../04549_puffin_allow_seeks_off.sh | 35 + ...12_puffin_metadata_footer_errors.reference | 40 + .../04612_puffin_metadata_footer_errors.sh | 85 ++ ...4613_puffin_metadata_type_errors.reference | 48 + .../04613_puffin_metadata_type_errors.sh | 74 ++ .../data_minio/dv_puffin_warehouse/README.md | 35 + ...9df-a585-047546b341d9-00001-deletes.puffin | Bin 0 -> 2141 bytes ...ced-935b-d2b8ef79eb76-00001-deletes.puffin | Bin 0 -> 1346 bytes ...d07-9633-95670a571224-00001-deletes.puffin | Bin 0 -> 539 bytes ...3b9-9112-c569f91929b9-00001-deletes.puffin | Bin 0 -> 1337 bytes ...94e-b336-f4d3301291b4-00001-deletes.puffin | Bin 0 -> 1351 bytes ...429-4dc8-9457-9b0d5a6a6e9b-0-00005.parquet | Bin 0 -> 779 bytes ...0de-4110-9928-e74531d8b125-0-00005.parquet | Bin 0 -> 1003 bytes ...-de4e-4d07-9633-95670a571224-00001.parquet | Bin 0 -> 969 bytes ...f4f-47e6-ad97-00c53284ff87-0-00001.parquet | Bin 0 -> 1173 bytes ...256-45aa-9ed1-48200445069f-0-00005.parquet | Bin 0 -> 759 bytes ...429-4dc8-9457-9b0d5a6a6e9b-0-00001.parquet | Bin 0 -> 766 bytes ...0de-4110-9928-e74531d8b125-0-00001.parquet | Bin 0 -> 1027 bytes ...ebb-4cfc-a3b7-3f2a1034a6dc-0-00001.parquet | Bin 0 -> 1192 bytes ...256-45aa-9ed1-48200445069f-0-00001.parquet | Bin 0 -> 784 bytes ...429-4dc8-9457-9b0d5a6a6e9b-0-00004.parquet | Bin 0 -> 791 bytes ...0de-4110-9928-e74531d8b125-0-00004.parquet | Bin 0 -> 1043 bytes ...8a4-482a-9a88-e08ad8bdd2e6-0-00001.parquet | Bin 0 -> 1197 bytes ...256-45aa-9ed1-48200445069f-0-00004.parquet | Bin 0 -> 806 bytes ...429-4dc8-9457-9b0d5a6a6e9b-0-00002.parquet | Bin 0 -> 800 bytes ...0de-4110-9928-e74531d8b125-0-00002.parquet | Bin 0 -> 1002 bytes ...583-48b6-983a-c2993f261fec-0-00001.parquet | Bin 0 -> 1148 bytes ...256-45aa-9ed1-48200445069f-0-00002.parquet | Bin 0 -> 772 bytes ...429-4dc8-9457-9b0d5a6a6e9b-0-00003.parquet | Bin 0 -> 800 bytes ...0de-4110-9928-e74531d8b125-0-00003.parquet | Bin 0 -> 1028 bytes ...8dd-4d8a-a31a-6db198e852d4-0-00001.parquet | Bin 0 -> 1176 bytes ...256-45aa-9ed1-48200445069f-0-00003.parquet | Bin 0 -> 766 bytes ...03abfe-66a8-43ab-a765-a9e8072b59ed-m0.avro | Bin 0 -> 8221 bytes ...ca5b2b-d3f2-44b4-8494-f0aadd4e0b34-m0.avro | Bin 0 -> 8126 bytes ...1000ac-b08e-430f-9c79-e40d5e4aa42d-m0.avro | Bin 0 -> 8145 bytes ...1000ac-b08e-430f-9c79-e40d5e4aa42d-m1.avro | Bin 0 -> 8132 bytes ...ec1ceb-2dbe-4ab7-ab4e-0d18bed2d2da-m0.avro | Bin 0 -> 8230 bytes ...caec64-2cad-46f5-a966-dd75e3635abc-m0.avro | Bin 0 -> 8135 bytes ...caec64-2cad-46f5-a966-dd75e3635abc-m1.avro | Bin 0 -> 8141 bytes ...66c606-c296-4fe4-a5f8-d8be592f6b96-m0.avro | Bin 0 -> 8142 bytes ...66c606-c296-4fe4-a5f8-d8be592f6b96-m1.avro | Bin 0 -> 8318 bytes ...66c606-c296-4fe4-a5f8-d8be592f6b96-m2.avro | Bin 0 -> 8298 bytes ...66c606-c296-4fe4-a5f8-d8be592f6b96-m3.avro | Bin 0 -> 8297 bytes ...66c606-c296-4fe4-a5f8-d8be592f6b96-m4.avro | Bin 0 -> 8436 bytes ...19b33f-2be5-4073-820c-82cb1c7d76cb-m0.avro | Bin 0 -> 8179 bytes ...8e784a-6b9c-4894-827d-c56978599ecd-m0.avro | Bin 0 -> 8301 bytes ...1c4572-6006-4649-85b1-47e82e73a396-m0.avro | Bin 0 -> 8157 bytes ...-c21c4572-6006-4649-85b1-47e82e73a396.avro | Bin 0 -> 5015 bytes ...-b68e784a-6b9c-4894-827d-c56978599ecd.avro | Bin 0 -> 5072 bytes ...-23ca5b2b-d3f2-44b4-8494-f0aadd4e0b34.avro | Bin 0 -> 4885 bytes ...-0c03abfe-66a8-43ab-a765-a9e8072b59ed.avro | Bin 0 -> 4966 bytes ...-6a19b33f-2be5-4073-820c-82cb1c7d76cb.avro | Bin 0 -> 5124 bytes ...-321000ac-b08e-430f-9c79-e40d5e4aa42d.avro | Bin 0 -> 4908 bytes ...-39ec1ceb-2dbe-4ab7-ab4e-0d18bed2d2da.avro | Bin 0 -> 4800 bytes ...-5b66c606-c296-4fe4-a5f8-d8be592f6b96.avro | Bin 0 -> 5139 bytes ...-41caec64-2cad-46f5-a966-dd75e3635abc.avro | Bin 0 -> 5192 bytes .../metadata/v1.metadata.json | 1 + .../metadata/v10.metadata.json | 1 + .../metadata/v11.metadata.json | 1 + .../metadata/v2.metadata.json | 1 + .../metadata/v3.metadata.json | 1 + .../metadata/v4.metadata.json | 1 + .../metadata/v5.metadata.json | 1 + .../metadata/v6.metadata.json | 1 + .../metadata/v7.metadata.json | 1 + .../metadata/v8.metadata.json | 1 + .../metadata/v9.metadata.json | 1 + .../metadata/version-hint.text | 1 + ...697-43f9-b2ea-9894139e9373-0-00001.parquet | Bin 0 -> 734 bytes ...65e-b4d7-6366f5ebda40-00001-deletes.puffin | Bin 0 -> 532 bytes ...8edb12-fdf0-4ce1-b6ab-c9c2a05d6d57-m0.avro | Bin 0 -> 7802 bytes ...4940c4-6b84-4560-85a3-35e17802d85d-m0.avro | Bin 0 -> 7785 bytes ...-c34940c4-6b84-4560-85a3-35e17802d85d.avro | Bin 0 -> 4789 bytes ...-0e8edb12-fdf0-4ce1-b6ab-c9c2a05d6d57.avro | Bin 0 -> 4870 bytes .../metadata/v1.metadata.json | 1 + .../metadata/v2.metadata.json | 1 + .../metadata/v3.metadata.json | 1 + .../metadata/version-hint.text | 1 + .../data_minio/generate_iceberg_dv_fixture.py | 160 +++ .../data_puffin/blob_overlaps_footer.puffin | Bin 0 -> 298 bytes .../data_puffin/bool_fields_element.puffin | Bin 0 -> 302 bytes .../data_puffin/bool_offset.puffin | Bin 0 -> 301 bytes ...ality_exceeds_materialization_limit.puffin | Bin 0 -> 290 bytes .../cardinality_mismatch_large_bitmap.puffin | Bin 0 -> 298 bytes .../data_puffin/compressed_footer.puffin | Bin 0 -> 497 bytes .../data_puffin/compression_codec_null.puffin | Bin 0 -> 241 bytes .../compression_codec_number.puffin | Bin 0 -> 238 bytes .../data_puffin/dense_range_100k.puffin | Bin 0 -> 294 bytes .../data_puffin/dv_envelope_bad_magic.puffin | Bin 0 -> 284 bytes .../dv_envelope_crc_mismatch.puffin | Bin 0 -> 284 bytes .../dv_envelope_length_mismatch.puffin | Bin 0 -> 284 bytes .../data_puffin/dv_nonzero_snapshot_id.puffin | Bin 0 -> 297 bytes .../dv_with_compression_codec.puffin | Bin 0 -> 326 bytes .../fields_element_out_of_int32_range.puffin | Bin 0 -> 311 bytes .../fields_element_out_of_int64_range.puffin | Bin 0 -> 318 bytes .../data_puffin/file_properties_ok.puffin | Bin 0 -> 333 bytes .../data_puffin/float_fields_element.puffin | Bin 0 -> 301 bytes .../data_puffin/float_offset.puffin | Bin 0 -> 300 bytes .../footer_integer_overflow.puffin | Bin 0 -> 260 bytes .../data_puffin/footer_root_array.puffin | Bin 0 -> 87 bytes .../data_puffin/generate_puffin_fixtures.py | 745 +++++++++++ .../data_puffin/generate_spark_puffin.py | 89 ++ .../data_puffin/incomplete_lz4_footer.puffin | Bin 0 -> 97 bytes .../inflated_lz4_content_size.puffin | Bin 0 -> 497 bytes .../data_puffin/invalid_bitmap_key.puffin | Bin 0 -> 268 bytes .../data_puffin/invalid_blob_entry.puffin | Bin 0 -> 106 bytes .../invalid_cardinality_negative.puffin | Bin 0 -> 285 bytes .../invalid_cardinality_non_numeric.puffin | Bin 0 -> 295 bytes .../invalid_file_properties_array.puffin | Bin 0 -> 302 bytes .../invalid_file_property_number.puffin | Bin 0 -> 330 bytes .../invalid_non_dv_properties_array.puffin | Bin 0 -> 445 bytes .../invalid_properties_array.puffin | Bin 0 -> 222 bytes ...invalid_property_cardinality_number.puffin | Bin 0 -> 296 bytes .../invalid_property_number.puffin | Bin 0 -> 308 bytes .../data_puffin/invalid_roaring_bitmap.puffin | Bin 0 -> 269 bytes .../data_puffin/length_exceeds_file.puffin | Bin 0 -> 301 bytes .../lz4_content_size_over_absolute_cap.puffin | Bin 0 -> 65872 bytes .../data_puffin/lz4_trailing_bytes.puffin | Bin 0 -> 318 bytes .../data_puffin/malformed_footer_json.puffin | Bin 0 -> 79 bytes .../data_puffin/missing_blobs.puffin | Bin 0 -> 80 bytes .../data_puffin/missing_cardinality.puffin | Bin 0 -> 278 bytes .../missing_footer_leading_magic.puffin | Bin 0 -> 29 bytes .../missing_lz4_content_size.puffin | Bin 0 -> 303 bytes .../data_puffin/missing_properties.puffin | Bin 0 -> 204 bytes .../missing_referenced_data_file.puffin | Bin 0 -> 240 bytes .../data_puffin/missing_type.puffin | Bin 0 -> 268 bytes .../data_puffin/mixed_blob_types.puffin | Bin 0 -> 445 bytes .../data_puffin/negative_offset.puffin | Bin 0 -> 299 bytes .../data_puffin/null_blob_entry.puffin | Bin 0 -> 95 bytes .../0_stateless/data_puffin/null_blobs.puffin | Bin 0 -> 93 bytes .../data_puffin/null_file_properties.puffin | Bin 0 -> 304 bytes .../data_puffin/null_non_dv_properties.puffin | Bin 0 -> 447 bytes .../data_puffin/null_properties.puffin | Bin 0 -> 224 bytes .../offset_out_of_int64_range.puffin | Bin 0 -> 317 bytes .../data_puffin/overflow_offset_length.puffin | Bin 0 -> 316 bytes .../data_puffin/puffin_errors_common.sh | 87 ++ .../data_puffin/spark_deletion_vector.puffin | Bin 0 -> 499 bytes .../data_puffin/sparse_large_key.puffin | Bin 0 -> 282 bytes .../data_puffin/string_offset.puffin | Bin 0 -> 300 bytes .../data_puffin/type_number.puffin | Bin 0 -> 281 bytes 222 files changed, 6457 insertions(+), 82 deletions(-) create mode 100644 docs/en/interfaces/formats/Puffin/Puffin.md create mode 100644 docs/en/interfaces/formats/Puffin/PuffinMetadata.md create mode 100644 src/Processors/Formats/Impl/PuffinBlockInputFormat.cpp create mode 100644 src/Processors/Formats/Impl/PuffinBlockInputFormat.h create mode 100644 src/Processors/tests/gtest_parquet_v3_need_only_count_buckets.cpp create mode 100644 src/Processors/tests/gtest_puffin_reset_parser.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.h create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_count_shortcuts.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_data_object_info_clone.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_parquet_row_deletes_guard.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.h create mode 100644 src/Storages/ObjectStorage/DataLakes/PuffinFilesCache.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/PuffinFilesCache.h create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_deletion_vector_before_equality_filter.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_has_non_empty_excluded_rows.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_bounds.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_cardinality.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_envelope.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_clone.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_key.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_weight.cpp create mode 100644 tests/integration/test_storage_iceberg_with_spark/test_deletion_vectors.py create mode 100644 tests/queries/0_stateless/04077_puffin_happy_path.reference create mode 100755 tests/queries/0_stateless/04077_puffin_happy_path.sh create mode 100644 tests/queries/0_stateless/04257_puffin_errors.reference create mode 100755 tests/queries/0_stateless/04257_puffin_errors.sh create mode 100644 tests/queries/0_stateless/04260_puffin_sparse_large_key.reference create mode 100755 tests/queries/0_stateless/04260_puffin_sparse_large_key.sh create mode 100644 tests/queries/0_stateless/04261_iceberg_deletion_vector.reference create mode 100755 tests/queries/0_stateless/04261_iceberg_deletion_vector.sh create mode 100644 tests/queries/0_stateless/04262_iceberg_deletion_vector_complex.reference create mode 100755 tests/queries/0_stateless/04262_iceberg_deletion_vector_complex.sh create mode 100644 tests/queries/0_stateless/04263_iceberg_puffin_files_cache.reference create mode 100755 tests/queries/0_stateless/04263_iceberg_puffin_files_cache.sh create mode 100644 tests/queries/0_stateless/04548_puffin_stdin_pipe.reference create mode 100755 tests/queries/0_stateless/04548_puffin_stdin_pipe.sh create mode 100644 tests/queries/0_stateless/04549_puffin_allow_seeks_off.reference create mode 100755 tests/queries/0_stateless/04549_puffin_allow_seeks_off.sh create mode 100644 tests/queries/0_stateless/04612_puffin_metadata_footer_errors.reference create mode 100755 tests/queries/0_stateless/04612_puffin_metadata_footer_errors.sh create mode 100644 tests/queries/0_stateless/04613_puffin_metadata_type_errors.reference create mode 100755 tests/queries/0_stateless/04613_puffin_metadata_type_errors.sh create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/README.md create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-14-ba106366-1379-49df-a585-047546b341d9-00001-deletes.puffin create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-18-0da8af70-7e2e-4ced-935b-d2b8ef79eb76-00001-deletes.puffin create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-20-04b90311-de4e-4d07-9633-95670a571224-00001-deletes.puffin create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-6-74340066-9c57-43b9-9112-c569f91929b9-00001-deletes.puffin create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-9-dcc19f77-3570-494e-b336-f4d3301291b4-00001-deletes.puffin create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=0/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00005.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=0/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00005.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=0/00000-20-04b90311-de4e-4d07-9633-95670a571224-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=0/00000-25-ee6fe6d5-2f4f-47e6-ad97-00c53284ff87-0-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=0/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00005.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=1/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=1/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=1/00000-24-0b9f04c6-aebb-4cfc-a3b7-3f2a1034a6dc-0-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=1/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00004.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00004.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-23-292ac3e8-88a4-482a-9a88-e08ad8bdd2e6-0-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00004.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00002.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00002.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-21-55307a4a-f583-48b6-983a-c2993f261fec-0-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00002.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00003.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00003.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-22-cbc4c1d2-88dd-4d8a-a31a-6db198e852d4-0-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00003.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/0c03abfe-66a8-43ab-a765-a9e8072b59ed-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/23ca5b2b-d3f2-44b4-8494-f0aadd4e0b34-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/321000ac-b08e-430f-9c79-e40d5e4aa42d-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/321000ac-b08e-430f-9c79-e40d5e4aa42d-m1.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/39ec1ceb-2dbe-4ab7-ab4e-0d18bed2d2da-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/41caec64-2cad-46f5-a966-dd75e3635abc-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/41caec64-2cad-46f5-a966-dd75e3635abc-m1.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/5b66c606-c296-4fe4-a5f8-d8be592f6b96-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/5b66c606-c296-4fe4-a5f8-d8be592f6b96-m1.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/5b66c606-c296-4fe4-a5f8-d8be592f6b96-m2.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/5b66c606-c296-4fe4-a5f8-d8be592f6b96-m3.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/5b66c606-c296-4fe4-a5f8-d8be592f6b96-m4.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/6a19b33f-2be5-4073-820c-82cb1c7d76cb-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/b68e784a-6b9c-4894-827d-c56978599ecd-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/c21c4572-6006-4649-85b1-47e82e73a396-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-212327459647764229-1-c21c4572-6006-4649-85b1-47e82e73a396.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-2216958009875447676-1-b68e784a-6b9c-4894-827d-c56978599ecd.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-2245381059480343696-1-23ca5b2b-d3f2-44b4-8494-f0aadd4e0b34.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-243132775900853210-1-0c03abfe-66a8-43ab-a765-a9e8072b59ed.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-399037466005519109-1-6a19b33f-2be5-4073-820c-82cb1c7d76cb.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-7131460395349610904-1-321000ac-b08e-430f-9c79-e40d5e4aa42d.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-8352556540874311077-1-39ec1ceb-2dbe-4ab7-ab4e-0d18bed2d2da.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-8541702041949031574-1-5b66c606-c296-4fe4-a5f8-d8be592f6b96.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-9221420601809522049-1-41caec64-2cad-46f5-a966-dd75e3635abc.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v1.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v10.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v11.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v2.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v3.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v4.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v5.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v6.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v7.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v8.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v9.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/version-hint.text create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/data/00000-0-e150e3cb-5697-43f9-b2ea-9894139e9373-0-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/data/00000-2-ee31c25d-1223-465e-b4d7-6366f5ebda40-00001-deletes.puffin create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/0e8edb12-fdf0-4ce1-b6ab-c9c2a05d6d57-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/c34940c4-6b84-4560-85a3-35e17802d85d-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/snap-1455293623190430422-1-c34940c4-6b84-4560-85a3-35e17802d85d.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/snap-8278841256334127309-1-0e8edb12-fdf0-4ce1-b6ab-c9c2a05d6d57.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/v1.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/v2.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/v3.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/version-hint.text create mode 100644 tests/queries/0_stateless/data_minio/generate_iceberg_dv_fixture.py create mode 100644 tests/queries/0_stateless/data_puffin/blob_overlaps_footer.puffin create mode 100644 tests/queries/0_stateless/data_puffin/bool_fields_element.puffin create mode 100644 tests/queries/0_stateless/data_puffin/bool_offset.puffin create mode 100644 tests/queries/0_stateless/data_puffin/cardinality_exceeds_materialization_limit.puffin create mode 100644 tests/queries/0_stateless/data_puffin/cardinality_mismatch_large_bitmap.puffin create mode 100644 tests/queries/0_stateless/data_puffin/compressed_footer.puffin create mode 100644 tests/queries/0_stateless/data_puffin/compression_codec_null.puffin create mode 100644 tests/queries/0_stateless/data_puffin/compression_codec_number.puffin create mode 100644 tests/queries/0_stateless/data_puffin/dense_range_100k.puffin create mode 100644 tests/queries/0_stateless/data_puffin/dv_envelope_bad_magic.puffin create mode 100644 tests/queries/0_stateless/data_puffin/dv_envelope_crc_mismatch.puffin create mode 100644 tests/queries/0_stateless/data_puffin/dv_envelope_length_mismatch.puffin create mode 100644 tests/queries/0_stateless/data_puffin/dv_nonzero_snapshot_id.puffin create mode 100644 tests/queries/0_stateless/data_puffin/dv_with_compression_codec.puffin create mode 100644 tests/queries/0_stateless/data_puffin/fields_element_out_of_int32_range.puffin create mode 100644 tests/queries/0_stateless/data_puffin/fields_element_out_of_int64_range.puffin create mode 100644 tests/queries/0_stateless/data_puffin/file_properties_ok.puffin create mode 100644 tests/queries/0_stateless/data_puffin/float_fields_element.puffin create mode 100644 tests/queries/0_stateless/data_puffin/float_offset.puffin create mode 100644 tests/queries/0_stateless/data_puffin/footer_integer_overflow.puffin create mode 100644 tests/queries/0_stateless/data_puffin/footer_root_array.puffin create mode 100644 tests/queries/0_stateless/data_puffin/generate_puffin_fixtures.py create mode 100644 tests/queries/0_stateless/data_puffin/generate_spark_puffin.py create mode 100644 tests/queries/0_stateless/data_puffin/incomplete_lz4_footer.puffin create mode 100644 tests/queries/0_stateless/data_puffin/inflated_lz4_content_size.puffin create mode 100644 tests/queries/0_stateless/data_puffin/invalid_bitmap_key.puffin create mode 100644 tests/queries/0_stateless/data_puffin/invalid_blob_entry.puffin create mode 100644 tests/queries/0_stateless/data_puffin/invalid_cardinality_negative.puffin create mode 100644 tests/queries/0_stateless/data_puffin/invalid_cardinality_non_numeric.puffin create mode 100644 tests/queries/0_stateless/data_puffin/invalid_file_properties_array.puffin create mode 100644 tests/queries/0_stateless/data_puffin/invalid_file_property_number.puffin create mode 100644 tests/queries/0_stateless/data_puffin/invalid_non_dv_properties_array.puffin create mode 100644 tests/queries/0_stateless/data_puffin/invalid_properties_array.puffin create mode 100644 tests/queries/0_stateless/data_puffin/invalid_property_cardinality_number.puffin create mode 100644 tests/queries/0_stateless/data_puffin/invalid_property_number.puffin create mode 100644 tests/queries/0_stateless/data_puffin/invalid_roaring_bitmap.puffin create mode 100644 tests/queries/0_stateless/data_puffin/length_exceeds_file.puffin create mode 100644 tests/queries/0_stateless/data_puffin/lz4_content_size_over_absolute_cap.puffin create mode 100644 tests/queries/0_stateless/data_puffin/lz4_trailing_bytes.puffin create mode 100644 tests/queries/0_stateless/data_puffin/malformed_footer_json.puffin create mode 100644 tests/queries/0_stateless/data_puffin/missing_blobs.puffin create mode 100644 tests/queries/0_stateless/data_puffin/missing_cardinality.puffin create mode 100644 tests/queries/0_stateless/data_puffin/missing_footer_leading_magic.puffin create mode 100644 tests/queries/0_stateless/data_puffin/missing_lz4_content_size.puffin create mode 100644 tests/queries/0_stateless/data_puffin/missing_properties.puffin create mode 100644 tests/queries/0_stateless/data_puffin/missing_referenced_data_file.puffin create mode 100644 tests/queries/0_stateless/data_puffin/missing_type.puffin create mode 100644 tests/queries/0_stateless/data_puffin/mixed_blob_types.puffin create mode 100644 tests/queries/0_stateless/data_puffin/negative_offset.puffin create mode 100644 tests/queries/0_stateless/data_puffin/null_blob_entry.puffin create mode 100644 tests/queries/0_stateless/data_puffin/null_blobs.puffin create mode 100644 tests/queries/0_stateless/data_puffin/null_file_properties.puffin create mode 100644 tests/queries/0_stateless/data_puffin/null_non_dv_properties.puffin create mode 100644 tests/queries/0_stateless/data_puffin/null_properties.puffin create mode 100644 tests/queries/0_stateless/data_puffin/offset_out_of_int64_range.puffin create mode 100644 tests/queries/0_stateless/data_puffin/overflow_offset_length.puffin create mode 100644 tests/queries/0_stateless/data_puffin/puffin_errors_common.sh create mode 100644 tests/queries/0_stateless/data_puffin/spark_deletion_vector.puffin create mode 100644 tests/queries/0_stateless/data_puffin/sparse_large_key.puffin create mode 100644 tests/queries/0_stateless/data_puffin/string_offset.puffin create mode 100644 tests/queries/0_stateless/data_puffin/type_number.puffin diff --git a/docs/en/engines/table-engines/integrations/iceberg.md b/docs/en/engines/table-engines/integrations/iceberg.md index 7a93d3f18c35..4cd0667f68cc 100644 --- a/docs/en/engines/table-engines/integrations/iceberg.md +++ b/docs/en/engines/table-engines/integrations/iceberg.md @@ -131,9 +131,16 @@ ClickHouse supports reading Iceberg tables that use the following deletion metho - [Position deletes](https://iceberg.apache.org/spec/#position-delete-files) - [Equality deletes](https://iceberg.apache.org/spec/#equality-delete-files) (supported from version 25.8+) +- [Deletion vectors](https://iceberg.apache.org/spec/#deletion-vectors) stored in Puffin files (Iceberg v3, read-only) -The following deletion method is **not supported**: -- [Deletion vectors](https://iceberg.apache.org/spec/#deletion-vectors) (introduced in v3) +The following limitations apply to deletion vectors: + +- Only `deletion-vector-v1` Puffin blobs are supported +- Data files must be in Parquet format +- Column-scoped deletion vectors (`fields` in the Puffin blob) are not supported +- Writing deletion vectors is not supported + +Parsed deletion vectors can be cached in memory when `use_puffin_files_cache` is enabled and the puffin file has a non-empty `etag`. Empty deletion vectors are cached as well, so repeated reads do not re-fetch the puffin file. The cache can be cleared with `SYSTEM DROP PUFFIN_FILES_CACHE`. ### Basic usage {#basic-usage} ```sql diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 38e16b525be6..c8be12560afc 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -110,6 +110,8 @@ The supported formats are: | [AvroConfluent](./formats/Avro/AvroConfluent.md) | ✔ | ✔ | | [Parquet](./formats/Parquet/Parquet.md) | ✔ | ✔ | | [ParquetMetadata](./formats/Parquet/ParquetMetadata.md) | ✔ | ✗ | +| [Puffin](./formats/Puffin/Puffin.md) | ✔ | ✗ | +| [PuffinMetadata](./formats/Puffin/PuffinMetadata.md) | ✔ | ✗ | | [Arrow](./formats/Arrow/Arrow.md) | ✔ | ✔ | | [ArrowStream](./formats/Arrow/ArrowStream.md) | ✔ | ✔ | | [ORC](./formats/ORC.md) | ✔ | ✔ | diff --git a/docs/en/interfaces/formats/Puffin/Puffin.md b/docs/en/interfaces/formats/Puffin/Puffin.md new file mode 100644 index 000000000000..8532fcff0080 --- /dev/null +++ b/docs/en/interfaces/formats/Puffin/Puffin.md @@ -0,0 +1,46 @@ +--- +description: 'Documentation for the Puffin format' +keywords: ['Puffin'] +slug: /interfaces/formats/Puffin +title: 'Puffin' +doc_type: 'reference' +--- + +## Description {#description} + +Input format for reading [Apache Iceberg Puffin](https://iceberg.apache.org/puffin-spec/) files. + +The format exposes deleted row positions from `deletion-vector-v1` blobs. Other blob types (for example `apache-datasketches-theta-v1`) are skipped. +If a puffin file contains multiple `deletion-vector-v1` blobs, the format outputs one row per such blob. + +Fixed output columns: +- `referenced_data_file` (`String`) - location of the data file the deletion vector applies to (`referenced-data-file` blob property) +- `deleted_rows` (`Array(UInt64)`) - 64-bit row positions deleted according to the deletion vector roaring bitmap + +Deletion vectors whose declared `cardinality` exceeds an absolute materialization ceiling are rejected when `deleted_rows` is requested. Footer `deletion-vector-v1` properties (including that `cardinality` parses as an unsigned integer) are always validated. Selecting only `referenced_data_file` skips on-disk payload I/O and therefore also skips envelope, CRC, roaring deserialize, and the materialization ceiling — intentionally, so a path-only projection does not read up to the blob-size cap. + +On-disk `deletion-vector-v1` blob length is bounded by an absolute ceiling (aligned with Iceberg's 2 GiB content-size check). When `deleted_rows` is requested, the reader peeks the envelope header (combined length and magic) before allocating the full payload; CRC is verified after the bounded read. + +LZ4-compressed and uncompressed puffin footers are supported. Footer payload size (and declared LZ4 content size) is bounded by a compression ratio where applicable and an absolute ceiling; oversized footers are rejected before allocation. + +Only a subset of output columns can be requested. A user-provided structure with unexpected column names or types is rejected when the format is created. + +## Example usage {#example-usage} + +Read deleted row positions with the referenced data file: + +```sql +SELECT referenced_data_file, deleted_rows +FROM file(deletes.puffin, Puffin); +``` + +Expand deleted positions into individual rows: + +```sql +SELECT referenced_data_file, row_number +FROM file(deletes.puffin, Puffin) +ARRAY JOIN deleted_rows AS row_number +ORDER BY referenced_data_file, row_number; +``` + +Use `PuffinMetadata` to inspect footer blob descriptors before reading deletion vectors. diff --git a/docs/en/interfaces/formats/Puffin/PuffinMetadata.md b/docs/en/interfaces/formats/Puffin/PuffinMetadata.md new file mode 100644 index 000000000000..dd716aeda62a --- /dev/null +++ b/docs/en/interfaces/formats/Puffin/PuffinMetadata.md @@ -0,0 +1,38 @@ +--- +description: 'Documentation for the PuffinMetadata format' +keywords: ['PuffinMetadata'] +slug: /interfaces/formats/PuffinMetadata +title: 'PuffinMetadata' +doc_type: 'reference' +--- + +## Description {#description} + +Special input format for reading [Apache Iceberg Puffin](https://iceberg.apache.org/puffin-spec/) file footer metadata. +It outputs one row per blob entry from the footer `BlobMetadata` list. + +Fixed output columns: +- `blob_type` (`String`) - blob type, for example `deletion-vector-v1` +- `snapshot_id` (`Int64`) - snapshot id of the blob +- `sequence_number` (`Int64`) - sequence number of the blob +- `fields` (`Array(Int32)`) - list of field ids the blob applies to +- `offset` (`Int64`) - offset of the blob payload in the file +- `length` (`Int64`) - length of the blob payload in bytes +- `compression_codec` (`String`) - compression codec of the blob payload, if present +- `properties` (`Map(String, String)`) - blob-specific properties + +Optional top-level `FileMetadata.properties` in the footer (for example `created-by`) are type-checked when present but are not returned as columns. If the key is present it must be a JSON object with string values (null is rejected). + +LZ4-compressed and uncompressed puffin footers are supported. Footer payload size (and declared LZ4 content size) is bounded by a compression ratio where applicable and an absolute ceiling; oversized footers are rejected before allocation. + +## Example usage {#example-usage} + +Inspect footer blobs: + +```sql +SELECT blob_type, snapshot_id, sequence_number, offset, length, compression_codec, + mapKeys(properties), mapValues(properties) +FROM file(deletes.puffin, PuffinMetadata); +``` + +Pair with the `Puffin` format to read `deletion-vector-v1` blob payloads. diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md index 69b2b59cb0ae..9707c96b0d15 100644 --- a/docs/en/sql-reference/statements/system.md +++ b/docs/en/sql-reference/statements/system.md @@ -109,6 +109,10 @@ Clears the per-URL Confluent Schema Registry caches used by the `AvroConfluent` Clears the parquet metadata cache. +## SYSTEM DROP PUFFIN_FILES_CACHE {#drop-puffin-files-cache} + +Clears the Puffin files cache used for parsed Iceberg puffin file content such as deletion vectors. + ## SYSTEM CLEAR|DROP TEXT INDEX CACHES {#drop-text-index-caches} Clears the text index's header, dictionary and postings caches. diff --git a/docs/en/sql-reference/table-functions/iceberg.md b/docs/en/sql-reference/table-functions/iceberg.md index 22f567d590cd..9ab0b5b8071c 100644 --- a/docs/en/sql-reference/table-functions/iceberg.md +++ b/docs/en/sql-reference/table-functions/iceberg.md @@ -120,11 +120,20 @@ ClickHouse supports time travel for Iceberg tables, allowing you to query histor ## Processing of tables with deleted rows {#deleted-rows} -Currently, only Iceberg tables with [position deletes](https://iceberg.apache.org/spec/#position-delete-files) are supported. +ClickHouse supports reading Iceberg tables that use the following deletion methods: -The following deletion methods are **not supported**: -- [Equality deletes](https://iceberg.apache.org/spec/#equality-delete-files) -- [Deletion vectors](https://iceberg.apache.org/spec/#deletion-vectors) (introduced in v3) +- [Position deletes](https://iceberg.apache.org/spec/#position-delete-files) +- [Equality deletes](https://iceberg.apache.org/spec/#equality-delete-files) (supported from version 25.8+) +- [Deletion vectors](https://iceberg.apache.org/spec/#deletion-vectors) stored in Puffin files (Iceberg v3, read-only) + +The following limitations apply to deletion vectors: + +- Only `deletion-vector-v1` Puffin blobs are supported +- Data files must be in Parquet format +- Column-scoped deletion vectors (`fields` in the Puffin blob) are not supported +- Writing deletion vectors is not supported + +Parsed deletion vectors can be cached in memory when `use_puffin_files_cache` is enabled and the puffin file has a non-empty `etag`. Empty deletion vectors are cached as well, so repeated reads do not re-fetch the puffin file. The cache can be cleared with `SYSTEM DROP PUFFIN_FILES_CACHE`. ### Basic usage {#basic-usage} diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index ded35800452f..e2a08bfe6c7b 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -172,6 +172,10 @@ namespace ServerSetting extern const ServerSettingsUInt64 parquet_metadata_cache_size; extern const ServerSettingsUInt64 parquet_metadata_cache_max_entries; extern const ServerSettingsDouble parquet_metadata_cache_size_ratio; + extern const ServerSettingsString puffin_files_cache_policy; + extern const ServerSettingsUInt64 puffin_files_cache_size; + extern const ServerSettingsUInt64 puffin_files_cache_max_entries; + extern const ServerSettingsDouble puffin_files_cache_size_ratio; extern const ServerSettingsUInt64 max_active_parts_loading_thread_pool_size; extern const ServerSettingsUInt64 max_io_thread_pool_free_size; extern const ServerSettingsUInt64 max_io_thread_pool_size; @@ -1540,6 +1544,17 @@ void LocalServer::processConfig() global_context->setParquetMetadataCache(parquet_metadata_cache_policy, parquet_metadata_cache_size, parquet_metadata_cache_max_entries, parquet_metadata_cache_size_ratio); #endif + String puffin_files_cache_policy = server_settings[ServerSetting::puffin_files_cache_policy]; + size_t puffin_files_cache_size = server_settings[ServerSetting::puffin_files_cache_size]; + size_t puffin_files_cache_max_entries = server_settings[ServerSetting::puffin_files_cache_max_entries]; + double puffin_files_cache_size_ratio = server_settings[ServerSetting::puffin_files_cache_size_ratio]; + if (puffin_files_cache_size > max_cache_size) + { + puffin_files_cache_size = max_cache_size; + LOG_INFO(log, "Lowered Puffin files cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(puffin_files_cache_size)); + } + global_context->setPuffinFilesCache(puffin_files_cache_policy, puffin_files_cache_size, puffin_files_cache_max_entries, puffin_files_cache_size_ratio); + Names allowed_disks_table_engines; splitInto<','>(allowed_disks_table_engines, server_settings[ServerSetting::allowed_disks_for_table_engines].value); global_context->setAllowedDisksForTableEngines(std::unordered_set(allowed_disks_table_engines.begin(), allowed_disks_table_engines.end())); diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index f449c2cc6858..8231e30d087c 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -288,6 +288,10 @@ namespace ServerSetting extern const ServerSettingsUInt64 parquet_metadata_cache_size; extern const ServerSettingsUInt64 parquet_metadata_cache_max_entries; extern const ServerSettingsDouble parquet_metadata_cache_size_ratio; + extern const ServerSettingsString puffin_files_cache_policy; + extern const ServerSettingsUInt64 puffin_files_cache_size; + extern const ServerSettingsUInt64 puffin_files_cache_max_entries; + extern const ServerSettingsDouble puffin_files_cache_size_ratio; extern const ServerSettingsUInt64 io_thread_pool_queue_size; extern const ServerSettingsBool jemalloc_enable_global_profiler; extern const ServerSettingsBool jemalloc_collect_global_profile_samples_in_trace_log; @@ -2276,6 +2280,16 @@ try } global_context->setParquetMetadataCache(parquet_metadata_cache_policy, parquet_metadata_cache_size, parquet_metadata_cache_max_entries, parquet_metadata_cache_size_ratio); #endif + String puffin_files_cache_policy = server_settings[ServerSetting::puffin_files_cache_policy]; + size_t puffin_files_cache_size = server_settings[ServerSetting::puffin_files_cache_size]; + size_t puffin_files_cache_max_entries = server_settings[ServerSetting::puffin_files_cache_max_entries]; + double puffin_files_cache_size_ratio = server_settings[ServerSetting::puffin_files_cache_size_ratio]; + if (puffin_files_cache_size > max_cache_size) + { + puffin_files_cache_size = max_cache_size; + LOG_INFO(log, "Lowered Puffin files cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(puffin_files_cache_size)); + } + global_context->setPuffinFilesCache(puffin_files_cache_policy, puffin_files_cache_size, puffin_files_cache_max_entries, puffin_files_cache_size_ratio); Names allowed_disks_table_engines; splitInto<','>(allowed_disks_table_engines, server_settings[ServerSetting::allowed_disks_for_table_engines].value); @@ -2705,6 +2719,7 @@ try #if USE_PARQUET global_context->updateParquetMetadataCacheConfiguration(config(), max_cache_size_in_bytes); #endif + global_context->updatePuffinFilesCacheConfiguration(config(), max_cache_size_in_bytes); } #if USE_SSL diff --git a/src/Access/Common/AccessType.h b/src/Access/Common/AccessType.h index f8c472987345..fd1dc6b4dbae 100644 --- a/src/Access/Common/AccessType.h +++ b/src/Access/Common/AccessType.h @@ -318,6 +318,7 @@ enum class AccessType : uint8_t M(SYSTEM_DROP_ICEBERG_METADATA_CACHE, "SYSTEM CLEAR ICEBERG_METADATA_CACHE, SYSTEM DROP ICEBERG_METADATA_CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_DROP_AVRO_SCHEMA_CACHE, "SYSTEM CLEAR AVRO SCHEMA CACHE, SYSTEM DROP AVRO SCHEMA CACHE, DROP AVRO SCHEMA CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_DROP_PARQUET_METADATA_CACHE, "SYSTEM DROP PARQUET_METADATA_CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ + M(SYSTEM_DROP_PUFFIN_FILES_CACHE, "SYSTEM DROP PUFFIN_FILES_CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_PREWARM_PRIMARY_INDEX_CACHE, "SYSTEM PREWARM PRIMARY INDEX, PREWARM PRIMARY INDEX CACHE, PREWARM PRIMARY INDEX", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_DROP_PRIMARY_INDEX_CACHE, "SYSTEM CLEAR PRIMARY INDEX CACHE, SYSTEM DROP PRIMARY INDEX, DROP PRIMARY INDEX CACHE, DROP PRIMARY INDEX", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_DROP_UNCOMPRESSED_CACHE, "SYSTEM CLEAR UNCOMPRESSED CACHE, SYSTEM DROP UNCOMPRESSED, DROP UNCOMPRESSED CACHE, DROP UNCOMPRESSED", GLOBAL, SYSTEM_DROP_CACHE) \ diff --git a/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h b/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h index fde3743850ba..a8cc053d21fc 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h +++ b/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h @@ -93,6 +93,13 @@ class RoaringBitmapWithSmallSet : private boost::noncopyable return roaring_bitmap->cardinality(); } + UInt64 getAllocatedBytes() const + { + if (isSmall()) + return sizeof(small); + return roaring_bitmap->getSizeInBytes(); + } + void merge(const RoaringBitmapWithSmallSet & r1) { if (r1.isLarge()) diff --git a/src/Client/BuzzHouse/Generator/SessionSettings.cpp b/src/Client/BuzzHouse/Generator/SessionSettings.cpp index d7ea0279dd63..0057657fc26a 100644 --- a/src/Client/BuzzHouse/Generator/SessionSettings.cpp +++ b/src/Client/BuzzHouse/Generator/SessionSettings.cpp @@ -1644,6 +1644,7 @@ static std::unordered_map serverSettings2 = { {"use_page_cache_for_local_disks", trueOrFalseSetting}, {"use_page_cache_for_object_storage", trueOrFalseSetting}, {"use_parquet_metadata_cache", trueOrFalseSetting}, + {"use_puffin_files_cache", trueOrFalseSetting}, {"use_query_cache", trueOrFalseSetting}, {"use_roaring_bitmap_iceberg_positional_deletes", trueOrFalseSetting}, {"use_skip_indexes_if_final_exact_mode", CHSetting(trueOrFalse, {"0", "1"}, true)}, diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index 9b5a18748f1c..b40437684886 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -337,6 +337,8 @@ M(IcebergMetadataFilesCacheFiles, "Number of cached files in the Iceberg metadata cache") \ M(ParquetMetadataCacheBytes, "Size of the Parquet metadata cache in bytes") \ M(ParquetMetadataCacheFiles, "Number of cached files in the Parquet metadata cache") \ + M(PuffinFilesCacheBytes, "Size of the Puffin files cache in bytes") \ + M(PuffinFilesCacheFiles, "Number of cached entries in the Puffin files cache") \ M(AvroSchemaCacheBytes, "Size of the Avro schema cache in bytes") \ M(AvroSchemaCacheCells, "Number of cached Avro schemas, including both registered and fetched schemas.") \ M(AvroSchemaRegistryCacheBytes, "Size of the Avro schema registry cache in bytes") \ diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 0b0874fe3a5e..c86b108f2d9d 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -110,6 +110,11 @@ M(IcebergMetadataReturnedObjectInfos, "Total number of returned object infos from iceberg iterator.", ValueType::Number) \ M(IcebergMinMaxNonPrunedDeleteFiles, "Total number of accepted data files-position delete file pairs by minmax analysis from pairs suitable by partitioning and sequence number.", ValueType::Number) \ M(IcebergMinMaxPrunedDeleteFiles, "Total number of accepted data files-position delete file pairs by minmax analysis from pairs suitable by partitioning and sequence number.", ValueType::Number) \ + M(PuffinFilesRead, "Number of Puffin files read (footer or deletion vector blob).", ValueType::Number) \ + M(PuffinFileReadMicroseconds, "Total time spent reading Puffin files.", ValueType::Microseconds) \ + M(PuffinFilesCacheHits, "Number of times parsed Puffin file content has been found in the cache.", ValueType::Number) \ + M(PuffinFilesCacheMisses, "Number of times parsed Puffin file content has not been found in the cache and had to be read from disk.", ValueType::Number) \ + M(PuffinFilesCacheWeightLost, "Approximate number of bytes evicted from the Puffin files cache.", ValueType::Number) \ M(VectorSimilarityIndexCacheHits, "Number of times an index granule has been found in the vector index cache.", ValueType::Number) \ M(VectorSimilarityIndexCacheMisses, "Number of times an index granule has not been found in the vector index cache and had to be read from disk.", ValueType::Number) \ M(VectorSimilarityIndexCacheWeightLost, "Approximate number of bytes evicted from the vector index cache.", ValueType::Number) \ diff --git a/src/Core/Defines.h b/src/Core/Defines.h index c99199f76446..b351507566e8 100644 --- a/src/Core/Defines.h +++ b/src/Core/Defines.h @@ -124,6 +124,10 @@ static constexpr auto DEFAULT_PARQUET_METADATA_CACHE_POLICY = "SLRU"; static constexpr auto DEFAULT_PARQUET_METADATA_CACHE_MAX_SIZE = 512_MiB; static constexpr auto DEFAULT_PARQUET_METADATA_CACHE_SIZE_RATIO = 0.5; static constexpr auto DEFAULT_PARQUET_METADATA_CACHE_MAX_ENTRIES = 5000; +static constexpr auto DEFAULT_PUFFIN_FILES_CACHE_POLICY = "SLRU"; +static constexpr auto DEFAULT_PUFFIN_FILES_CACHE_MAX_SIZE = 512_MiB; +static constexpr auto DEFAULT_PUFFIN_FILES_CACHE_SIZE_RATIO = 0.5; +static constexpr auto DEFAULT_PUFFIN_FILES_CACHE_MAX_ENTRIES = 5000; static constexpr auto DEFAULT_QUERY_CONDITION_CACHE_POLICY = "SLRU"; static constexpr auto DEFAULT_QUERY_CONDITION_CACHE_MAX_SIZE = 100_MiB; static constexpr auto DEFAULT_QUERY_CONDITION_CACHE_SIZE_RATIO = 0.5l; diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index 1f6eb7e54653..41105dd5cc98 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -24,6 +24,7 @@ #if USE_PARQUET # include #endif +#include #include #include #include @@ -560,6 +561,10 @@ namespace DECLARE(UInt64, parquet_metadata_cache_size, DEFAULT_PARQUET_METADATA_CACHE_MAX_SIZE, "Maximum size of parquet metadata cache in bytes. Zero means disabled.", 0) \ DECLARE(UInt64, parquet_metadata_cache_max_entries, DEFAULT_PARQUET_METADATA_CACHE_MAX_ENTRIES, "Maximum size of parquet metadata files cache in entries. Zero means disabled.", 0) \ DECLARE(Double, parquet_metadata_cache_size_ratio, DEFAULT_PARQUET_METADATA_CACHE_SIZE_RATIO, "The size of the protected queue (in case of SLRU policy) in the parquet metadata cache relative to the cache's total size.", 0) \ + DECLARE(String, puffin_files_cache_policy, DEFAULT_PUFFIN_FILES_CACHE_POLICY, "Puffin files cache policy name.", 0) \ + DECLARE(UInt64, puffin_files_cache_size, DEFAULT_PUFFIN_FILES_CACHE_MAX_SIZE, "Maximum size of Puffin files cache in bytes. Zero means disabled.", 0) \ + DECLARE(UInt64, puffin_files_cache_max_entries, DEFAULT_PUFFIN_FILES_CACHE_MAX_ENTRIES, "Maximum size of Puffin files cache in entries. Zero means disabled.", 0) \ + DECLARE(Double, puffin_files_cache_size_ratio, DEFAULT_PUFFIN_FILES_CACHE_SIZE_RATIO, "The size of the protected queue (in case of SLRU policy) in the Puffin files cache relative to the cache's total size.", 0) \ DECLARE(String, allowed_disks_for_table_engines, "", "List of disks allowed for use with Iceberg", 0) \ DECLARE(String, vector_similarity_index_cache_policy, DEFAULT_VECTOR_SIMILARITY_INDEX_CACHE_POLICY, "Vector similarity index cache policy name.", 0) \ DECLARE(UInt64, vector_similarity_index_cache_size, DEFAULT_VECTOR_SIMILARITY_INDEX_CACHE_MAX_SIZE, R"(Size of cache for vector similarity indexes. Zero means disabled. @@ -2010,6 +2015,10 @@ ChangeableSettingsMap collectChangeableServerSettings(ContextPtr context) {"parquet_metadata_cache_size", {std::to_string(context->getParquetMetadataCache()->maxSizeInBytes()), ChangeableWithoutRestart::Yes}}); #endif + if (context->getPuffinFilesCache()) + changeable_settings.insert( + {"puffin_files_cache_size", + {std::to_string(context->getPuffinFilesCache()->maxSizeInBytes()), ChangeableWithoutRestart::Yes}}); /// `keeper_hosts` is not a regular config setting; it is derived from the `` config and follows /// it on config reload, so the live value diverges from the empty default stored in `ServerSettings`. diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 199590245db4..3fd53c612d65 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -5558,6 +5558,14 @@ If turned on, parquet format may utilize the parquet metadata cache. Possible values: +- 0 - Disabled +- 1 - Enabled +)", 0) \ + DECLARE(Bool, use_puffin_files_cache, true, R"( +If turned on, Iceberg reads may utilize the Puffin files cache for parsed puffin file content such as deletion vectors. + +Possible values: + - 0 - Disabled - 1 - Enabled )", 0) \ diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index ef7d5710e59f..70ad87748008 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -39,6 +39,76 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() /// controls new feature and it's 'true' by default, use 'false' as previous_value). /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) /// Note: please check if the key already exists to prevent duplicate entries. + addSettingsChanges(settings_changes_history, "26.8", + { + {"unique_key_probe_implementation", "auto", "auto", "New setting: selects the UNIQUE KEY probe implementation (currently only the simple baseline exists)"}, + {"allow_lossy_numeric_supertype", false, false, "New setting that lets if/multiIf/coalesce/ifNull/array/map resolve all-numeric branches with no lossless common type (e.g. Decimal + Float64) to a numeric supertype (Float64, with possible precision loss), so the result can be aggregated. Independent of use_variant_as_common_type: with it off such branches previously raised NO_COMMON_TYPE, with it on they became a Variant; either way they now resolve to Float64."}, + {"analyzer_compatibility_apply_final_to_all_joined_tables", false, false, "New setting on master (default false = the fixed behavior). The behavior flip itself is recorded under 26.6, and the introduction for backports to older release branches (with default true) under 26.4."}, + {"join_runtime_filter_min_probe_rows", 0, 1000, "New setting to control minimum probe side size for installing JOIN runtime filters. It wasn't limited before, so previous value is 0 meaning always install."}, + {"query_plan_short_circuit_constant_false_join", false, true, "New setting to short-circuit a JOIN with a constant-false ON condition so the non-contributing side is not read. previous_value=false so `compatibility` with versions before 26.8 restores the pre-existing behavior (no short-circuit)."}, + }); + addSettingsChanges(settings_changes_history, "26.7", + { + {"analyzer_compatibility_allow_non_aggregate_in_having", false, false, "New compatibility setting. When enabled, the analyzer mimics the legacy `HAVING`-to-`WHERE` rewrite for non-aggregate AND-conjuncts instead of raising `NOT_AN_AGGREGATE`."}, + {"dictionary_lazy_load", "auto", "auto", "New setting overriding the server setting `dictionaries_lazy_load` for an individual dictionary."}, + {"discard_query_data", false, false, "New setting to skip sending query result rows to the client over the native TCP protocol."}, + {"optimize_trivial_count_with_sparsity_filter", false, false, "New (experimental) setting to serve `SELECT count() FROM t WHERE ` from per-column `num_defaults` / `num_rows` recorded in `serialization.json` when `` partitions rows into defaults vs non-defaults."}, + {"merge_tree_generic_exclusion_search_max_steps", 0, 0, "New setting to limit the number of steps of the generic exclusion search over the primary key index."}, + {"use_streaming_marks_compression", false, false, "New setting to compress marks into in-memory representation one block at a time (streaming) instead of materializing the full plain marks array, reducing peak memory during marks loading for compact parts with many substreams."}, + {"s3_validate_etag_on_read", false, true, "New setting to detect concurrent in-place overwrites of S3/GCS objects during a read by validating the GET response ETag against the listed one. previous_value=false so `compatibility` with versions before 26.7 restores the pre-existing behavior (no validation)."}, + {"ai_function_text_default_credentials", "", "", "New setting"}, + {"ai_function_embedding_default_credentials", "", "", "New setting"}, + {"dead_blobs_to_delay_insert", 0, 0, "New setting to override the `MergeTree` setting with the same name per query."}, + {"dead_blobs_to_throw_insert", 0, 0, "New setting to override the `MergeTree` setting with the same name per query."}, + {"input_format_csv_missing_nullable_as_empty_string", false, false, "New setting to read a missing value of `Nullable(String)` from CSV as an empty string instead of NULL."}, + {"use_legacy_to_time", true, false, "Use the new `toTime` function (converting values to the `Time` data type) by default instead of the legacy `toTime` (which is still available as `toTimeWithFixedDate`)."}, + {"reserve_memory", 0, 0, "New setting to reserve memory for specific workload before starting a query."}, + {"parallel_replicas_plan_based", false, false, "New setting"}, + {"use_paimon_metadata_files_cache", false, false, "New setting to enable in-memory caching of parsed Paimon metadata files (manifest lists and manifests). For persistent Paimon table engines it must be enabled before metadata initialization; table functions evaluate it per query. Avoids repeated downloads and deserialization of metadata files from object storage on subsequent queries."}, + {"optimize_or_like_chain", false, true, "Enable by default: optimize OR chains of LIKE/ILIKE/match into multiSearchAny (pure-substring patterns) or multiMatchAny (other patterns, when Hyperscan/Vectorscan is permitted); when neither fast path applies the original OR chain is kept unchanged."}, + {"optimize_or_like_chain_min_patterns", 0, 10, "New setting controlling the minimum number of non-pure-substring LIKE/ILIKE/match branches (sharing the same LHS expression) required for optimize_or_like_chain to rewrite a chain into multiMatchAny. Shorter chains are kept as-is because the multiMatchAny (Hyperscan) rewrite only becomes faster than short-circuit OR evaluation from about nine branches."}, + {"optimize_or_like_chain_min_substrings", 0, 4, "New setting controlling the minimum number of pure-substring (%needle%) LIKE/ILIKE branches (sharing the same LHS expression) required for optimize_or_like_chain to rewrite a chain into multiSearchAny."}, + {"input_format_arrow_use_native_reader", false, true, "New setting to use the native ClickHouse reader for the Arrow and ArrowStream formats instead of the Apache Arrow library."}, + {"input_format_orc_use_fast_decoder", true, true, "Obsolete setting, the native ClickHouse ORC decoder is now always used (the Apache Arrow-based ORC reader has been removed)."}, + {"output_format_arrow_use_native_writer", false, true, "New setting to use the native ClickHouse writer for the Arrow and ArrowStream formats instead of the Apache Arrow library."}, + {"allow_minmax_index_for_json", true, false, "Forbid creating minmax skip index on JSON columns by default because the index serialization cannot handle heterogeneous Field values"}, + {"s3_allow_server_credentials_in_user_queries", true, false, "New setting to block S3 access from user SQL from resolving the server's own ambient credentials (environment/IMDS/IRSA/instance-profile/AWS-config-file/role_arn-STS/GCP-OAuth-metadata). The previous behavior (allowed) is restored with compatibility settings."}, + {"query_plan_merge_expression_into_join", false, true, "New setting. Allow to merge Expression step into JOIN step during join reordering optimization."}, + {"skip_unavailable_shards_mode", "unavailable_or_table_missing", "unavailable_or_table_missing", "New setting to control which exceptions from a remote shard are ignored when `skip_unavailable_shards` is enabled. The default matches the historical behavior: a shard whose table is missing is treated as unavailable."}, + {"use_text_index_tokens_cache", false, true, "Enabled the text index tokens cache globally."}, + {"use_text_index_header_cache", false, true, "Enabled the text index header cache globally."}, + {"optimize_aggregation_in_order_limit", false, true, "New setting to push the `LIMIT` into aggregation-in-order for early termination when the `ORDER BY` is a prefix of the `GROUP BY` sort description."}, + {"explain_query_plan_default", "legacy", "pretty", "From 26.7, `EXPLAIN PLAN` defaults to `actions=1, compact=1, pretty=1`. Set this to `legacy` to restore the pre-26.7 output."}, + {"format_geojson_validate_geometry", true, true, "New setting that controls whether the GeoJSON format enforces RFC 7946 geometry validity (minimum points per line and ring, ring closure, non-empty multi-geometries) when reading and writing"}, + {"use_partition_minmax_for_primary_key_pruning", false, true, "New setting to use the part's partition minmax to prune more granules during primary key analysis for `MergeTree` tables, when a primary key column is also an input column of the partition key."}, + {"allow_delta_lake_writes", false, false, "Added an alias for setting `allow_experimental_delta_lake_writes`, which was moved to Beta."}, + {"allow_experimental_delta_lake_writes", false, false, "Delta Lake writes were moved to Beta."}, + {"optimize_redundant_comparisons", false, true, "New setting to detect conflicting and redundant comparison conditions on the same expression within AND chains."}, + {"mysql_datatypes_support_level", "decimal,datetime64,date2Date32", "decimal,datetime64,date2Date32,geometry", "Map MySQL's concrete spatial types (LINESTRING, POLYGON, MULTILINESTRING, MULTIPOLYGON, MULTIPOINT) and the generic GEOMETRY type to the corresponding ClickHouse geometric types by default. The generic GEOMETRY column maps to the umbrella Geometry type; reading a value whose subtype has no ClickHouse counterpart (GEOMETRYCOLLECTION) throws at read time."}, + {"snappy_mode", "basic", "basic", "New setting to control the wire format used for snappy compression in generic file/URL I/O. The default `basic` preserves backward-compatible Hadoop snappy block format reads; HTTP `Content-Encoding: snappy` always uses the framing format independently of this setting."}, + {"compile_regular_expressions", false, true, "New setting to enable JIT compilation of simple regular expressions in functions like `match` and `extract`."}, + {"min_count_to_compile_regular_expression", 3, 3, "New setting controlling how many times a regular expression must be used before it is JIT-compiled."}, + {"allow_aggregate_partitions_independently", false, true, "Enable independent per-partition aggregation by default when the partition key suits the GROUP BY key. The existing runtime heuristics in `ReadFromMergeTree::requestOutputEachPartitionThroughSeparatePortForAggregation` already skip the optimization when the partition layout is unfavorable (too few partitions, too many partitions, or significantly skewed partition sizes), so enabling the setting is safe in the cases where it would otherwise be a no-op."}, + {"text_index_lazy_intersection_density_threshold", 0.2, 0.2, "Renamed from `text_index_density_threshold` (kept as an alias); selects the posting list intersection algorithm in lazy posting list apply mode."}, + {"allow_experimental_text_index_lazy_apply", false, true, "Lazy posting list apply mode for the text index is no longer experimental; the setting is now obsolete and has no effect (lazy mode is selected via `text_index_posting_list_apply_mode = 'lazy'`)."}, + {"allow_experimental_url_wildcard_from_index_pages", false, false, "New setting to enable expanding wildcards in the `url` table function by listing HTTP index pages."}, + {"url_wildcard_max_directories_to_read", 100000, 100000, "New setting to limit the number of directories read when expanding wildcards in the `url` table function."}, + {"allow_experimental_eval_table_function", false, false, "New setting to enable the experimental table function `eval`."}, + {"output_format_csv_header_serialize_tuple_into_separate_columns", false, true, "New setting. When output_format_csv_serialize_tuple_into_separate_columns is enabled, the CSVWithNames/CSVWithNamesAndTypes header now flattens Tuple columns into their leaf fields so the header width matches the data. Set to false to restore the previous single-name header."}, + {"enable_join_runtime_filters_index_analysis", false, false, "New setting to enable join filtering using dynamic index analysis"}, + {"vector_search_use_quantized_codes", false, false, "New setting to opt into the two-stage approximate vector-search optimization over a Quantize(...) column codec; queries stay exact by default."}, + {"reader_executor_use_long_connections", false, false, "New experimental ReaderExecutor setting (off by default): reuse a held source connection across sequential windows."}, + {"reader_executor_min_bytes_for_seek", 2097152, 2097152, "New experimental ReaderExecutor setting: forward-gap bound for bridging on a held source connection."}, + {"reader_executor_max_tail_for_drain", 1048576, 1048576, "New experimental ReaderExecutor setting: drain bound for completing a dropped long connection."}, + {"precise_float_parsing", false, true, "Use the precise (closest-representable) float parsing algorithm by default, now that it is faster than the previous fast algorithm. Set to false to restore the pre-26.7 fast-but-less-accurate parsing in conversion functions."}, + {"optimize_and_compare_chain_max_hash_work", 0, 5'000'000, "New setting that bounds the work of the `optimize_and_compare_chain` optimization (measured in query-tree nodes hashed) so it cannot dominate analysis of queries with very many or very large `AND`-chains of comparisons. The previous value `0` (unlimited) reproduces the pre-26.7 behavior where the optimization was uncapped, so `compatibility` set to an earlier version keeps deriving transitive predicates without a budget. Set to `0` to disable the budget."}, + {"iceberg_manifest_min_count_to_compact", 30, 30, "New setting to control manifest compaction for Iceberg tables."}, + {"show_remote_databases_in_system_tables", true, true, "New setting to control whether `MySQL` and `PostgreSQL` databases are shown in `system.tables`, `system.columns` and `system.completions`."}, + {"use_constant_folding_in_index_analysis", false, false, "New setting to fold partition-level constants into the filter predicate per part during MergeTree index analysis, improving pruning for filters whose branches depend on partition values."}, + {"join_runtime_filter_size_from_hash_table_stats", false, true, "Use hash table size statistics collected from previous executions to size the JOIN runtime filter. When disabled, fall back to the fixed `join_runtime_bloom_filter_bytes`."}, + {"use_puffin_files_cache", false, true, "Enables cache of parsed Puffin file content such as deletion vectors."}, + }); + addSettingsChanges(settings_changes_history, "26.6", { {"output_format_image_width", 1024, 1024, "New setting controlling the width of the output image for image output formats such as PNG."}, diff --git a/src/Formats/registerFormats.cpp b/src/Formats/registerFormats.cpp index 5c23eac98d80..b8538d0bdff2 100644 --- a/src/Formats/registerFormats.cpp +++ b/src/Formats/registerFormats.cpp @@ -107,6 +107,7 @@ void registerInputFormatJSONAsObject(FormatFactory & factory); void registerInputFormatLineAsString(FormatFactory & factory); void registerInputFormatMySQLDump(FormatFactory & factory); void registerInputFormatParquetMetadata(FormatFactory & factory); +void registerInputFormatPuffin(FormatFactory & factory); void registerInputFormatDWARF(FormatFactory & factory); void registerInputFormatOne(FormatFactory & factory); @@ -150,6 +151,7 @@ void registerTemplateSchemaReader(FormatFactory & factory); void registerMySQLSchemaReader(FormatFactory & factory); void registerBSONEachRowSchemaReader(FormatFactory & factory); void registerParquetMetadataSchemaReader(FormatFactory & factory); +void registerPuffinSchemaReaders(FormatFactory & factory); void registerDWARFSchemaReader(FormatFactory & factory); void registerOneSchemaReader(FormatFactory & factory); void registerNpySchemaReader(FormatFactory & factory); @@ -266,6 +268,7 @@ void registerFormats() registerInputFormatMySQLDump(factory); registerInputFormatParquetMetadata(factory); + registerInputFormatPuffin(factory); registerInputFormatDWARF(factory); registerInputFormatOne(factory); @@ -305,6 +308,7 @@ void registerFormats() registerMySQLSchemaReader(factory); registerBSONEachRowSchemaReader(factory); registerParquetMetadataSchemaReader(factory); + registerPuffinSchemaReaders(factory); registerDWARFSchemaReader(factory); registerOneSchemaReader(factory); registerNpySchemaReader(factory); diff --git a/src/Interpreters/ClusterFunctionReadTask.cpp b/src/Interpreters/ClusterFunctionReadTask.cpp index f3cfad1fba31..57f15dd0baa4 100644 --- a/src/Interpreters/ClusterFunctionReadTask.cpp +++ b/src/Interpreters/ClusterFunctionReadTask.cpp @@ -82,6 +82,46 @@ void ClusterFunctionReadTaskResponse::serialize(WriteBuffer & out, size_t worker { auto protocol_version = std::min(static_cast(worker_protocol_version), static_cast(DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION)); + + /// Fail closed: downgrading would omit deletion / selection vectors and return deleted rows. + if (protocol_version < DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_EXCLUDED_ROWS + && hasNonEmptyExcludedRows(data_lake_metadata)) + { + throw Exception( + ErrorCodes::UNKNOWN_PROTOCOL, + "Worker protocol version {} cannot carry `excluded_rows`, which is required for distributed " + "reads with deletion vectors / selection vectors (minimum protocol version: {})", + protocol_version, + DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_EXCLUDED_ROWS); + } + + /// Fail closed: protocol < 3 omits `iceberg_info`, so workers rebuild a plain `ObjectInfo` and + /// skip equality / parquet position delete transforms — returning deleted rows. + if (protocol_version < DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_ICEBERG_METADATA + && iceberg_info.has_value() + && (!iceberg_info->position_deletes_objects.empty() || !iceberg_info->equality_deletes_objects.empty())) + { + throw Exception( + ErrorCodes::UNKNOWN_PROTOCOL, + "Worker protocol version {} cannot carry `iceberg_info` with equality / position deletes " + "(minimum protocol version: {})", + protocol_version, + DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_ICEBERG_METADATA); + } + + /// Fail closed: protocol < 4 omits `file_bucket_info`, so each bucket task becomes a full-file + /// read and bucket-split cluster queries return duplicated rows. + if (protocol_version < DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_FILE_BUCKETS_INFO + && file_bucket_info) + { + throw Exception( + ErrorCodes::UNKNOWN_PROTOCOL, + "Worker protocol version {} cannot carry `file_bucket_info`, which is required for " + "distributed bucket-split reads (minimum protocol version: {})", + protocol_version, + DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_FILE_BUCKETS_INFO); + } + writeVarUInt(protocol_version, out); writeStringBinary(path, out); diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index fc2e58c60368..a12c044aedf8 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -55,6 +55,7 @@ #include #include #include +#include #include #include #include @@ -586,6 +587,7 @@ struct ContextSharedPart : boost::noncopyable #if USE_PARQUET mutable ParquetMetadataCachePtr parquet_metadata_cache TSA_GUARDED_BY(mutex); /// Cache of deserialized parquet metadata files. #endif + mutable PuffinFilesCachePtr puffin_files_cache TSA_GUARDED_BY(mutex); /// Cache of parsed puffin file content. AsynchronousMetrics * asynchronous_metrics TSA_GUARDED_BY(mutex) = nullptr; /// Points to asynchronous metrics mutable PageCachePtr page_cache TSA_GUARDED_BY(mutex); /// Userspace page cache. ProcessList process_list; /// Executing queries at the moment. @@ -4755,6 +4757,52 @@ void Context::clearParquetMetadataCache() const } #endif +void Context::setPuffinFilesCache(const String & cache_policy, size_t max_size_in_bytes, size_t max_entries, double size_ratio) +{ + std::lock_guard lock(shared->mutex); + + if (shared->puffin_files_cache) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Puffin files cache has been already created."); + + shared->puffin_files_cache = std::make_shared(cache_policy, max_size_in_bytes, max_entries, size_ratio); +} + +void Context::updatePuffinFilesCacheConfiguration(const Poco::Util::AbstractConfiguration & config, size_t max_cache_size) +{ + std::lock_guard lock(shared->mutex); + + if (!shared->puffin_files_cache) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Puffin files cache was not created yet."); + + size_t size = config.getUInt64("puffin_files_cache_size", DEFAULT_PUFFIN_FILES_CACHE_MAX_SIZE); + size_t max_entries = config.getUInt64("puffin_files_cache_max_entries", DEFAULT_PUFFIN_FILES_CACHE_MAX_ENTRIES); + if (size > max_cache_size) + { + size = max_cache_size; + LOG_DEBUG(shared->log, "Lowered Puffin files cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(size)); + } + shared->puffin_files_cache->setMaxSizeInBytes(size); + shared->puffin_files_cache->setMaxCount(max_entries); +} + +std::shared_ptr Context::getPuffinFilesCache() const +{ + SharedLockGuard lock(shared->mutex); + + if (!shared->puffin_files_cache) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Puffin files cache was not created yet."); + return shared->puffin_files_cache; +} + +void Context::clearPuffinFilesCache() const +{ + auto cache = getPuffinFilesCache(); + + /// Clear the cache without holding context mutex to avoid blocking context for a long time + if (cache) + cache->clear(); +} + void Context::setQueryConditionCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio) { std::lock_guard lock(shared->mutex); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 4d8d171b9daf..d8d45c63e09c 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -109,6 +109,7 @@ class MMappedFileCache; class UncompressedCache; class IcebergMetadataFilesCache; class ParquetMetadataCache; +class PuffinFilesCache; class VectorSimilarityIndexCache; class TextIndexTokensCache; class TextIndexHeaderCache; @@ -1508,6 +1509,11 @@ class Context: public ContextData, public std::enable_shared_from_this void clearParquetMetadataCache() const; #endif + void setPuffinFilesCache(const String & cache_policy, size_t max_size_in_bytes, size_t max_entries, double size_ratio); + void updatePuffinFilesCacheConfiguration(const Poco::Util::AbstractConfiguration & config, size_t max_cache_size); + std::shared_ptr getPuffinFilesCache() const; + void clearPuffinFilesCache() const; + void setAllowedDisksForTableEngines(std::unordered_set && allowed_disks_) { allowed_disks = std::move(allowed_disks_); } const std::unordered_set & getAllowedDisksForTableEngines() const { return allowed_disks; } diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 9b515844c721..422b8e4fa048 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -481,6 +481,10 @@ BlockIO InterpreterSystemQuery::execute() #else throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "The server was compiled without the support for Parquet"); #endif + case Type::CLEAR_PUFFIN_FILES_CACHE: + getContext()->checkAccess(AccessType::SYSTEM_DROP_PUFFIN_FILES_CACHE); + system_context->clearPuffinFilesCache(); + break; case Type::CLEAR_PRIMARY_INDEX_CACHE: getContext()->checkAccess(AccessType::SYSTEM_DROP_PRIMARY_INDEX_CACHE); system_context->clearPrimaryIndexCache(); @@ -2474,6 +2478,7 @@ AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster() case Type::CLEAR_ICEBERG_METADATA_CACHE: case Type::CLEAR_AVRO_SCHEMA_CACHE: case Type::CLEAR_PARQUET_METADATA_CACHE: + case Type::CLEAR_PUFFIN_FILES_CACHE: case Type::CLEAR_PRIMARY_INDEX_CACHE: case Type::CLEAR_MMAP_CACHE: case Type::CLEAR_QUERY_CONDITION_CACHE: diff --git a/src/Parsers/ASTSystemQuery.cpp b/src/Parsers/ASTSystemQuery.cpp index c4ee97348899..b6dbd9b298a8 100644 --- a/src/Parsers/ASTSystemQuery.cpp +++ b/src/Parsers/ASTSystemQuery.cpp @@ -603,6 +603,7 @@ void ASTSystemQuery::formatImpl(WriteBuffer & ostr, const FormatSettings & setti case Type::CLEAR_S3_CLIENT_CACHE: case Type::CLEAR_ICEBERG_METADATA_CACHE: case Type::CLEAR_PARQUET_METADATA_CACHE: + case Type::CLEAR_PUFFIN_FILES_CACHE: case Type::CLEAR_AVRO_SCHEMA_CACHE: case Type::RESET_COVERAGE: case Type::RESTART_REPLICAS: diff --git a/src/Parsers/ASTSystemQuery.h b/src/Parsers/ASTSystemQuery.h index adf76d898d42..814e76989893 100644 --- a/src/Parsers/ASTSystemQuery.h +++ b/src/Parsers/ASTSystemQuery.h @@ -44,6 +44,7 @@ class ASTSystemQuery : public IAST, public ASTQueryWithOnCluster CLEAR_COMPILED_EXPRESSION_CACHE, CLEAR_ICEBERG_METADATA_CACHE, CLEAR_PARQUET_METADATA_CACHE, + CLEAR_PUFFIN_FILES_CACHE, CLEAR_FILESYSTEM_CACHE, CLEAR_DISTRIBUTED_CACHE, CLEAR_DISK_METADATA_CACHE, diff --git a/src/Parsers/ParserSystemQuery.cpp b/src/Parsers/ParserSystemQuery.cpp index 07b79465ea04..d44527847f22 100644 --- a/src/Parsers/ParserSystemQuery.cpp +++ b/src/Parsers/ParserSystemQuery.cpp @@ -285,6 +285,7 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & {"DROP COMPILED EXPRESSION CACHE", Type::CLEAR_COMPILED_EXPRESSION_CACHE}, {"DROP ICEBERG METADATA CACHE", Type::CLEAR_ICEBERG_METADATA_CACHE}, {"DROP PARQUET METADATA CACHE", Type::CLEAR_PARQUET_METADATA_CACHE}, + {"DROP PUFFIN_FILES_CACHE", Type::CLEAR_PUFFIN_FILES_CACHE}, {"DROP FILESYSTEM CACHE", Type::CLEAR_FILESYSTEM_CACHE}, {"DROP DISTRIBUTED CACHE", Type::CLEAR_DISTRIBUTED_CACHE}, {"DROP DISK METADATA CACHE", Type::CLEAR_DISK_METADATA_CACHE}, diff --git a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp index edbf421ccbeb..567a018a00b6 100644 --- a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp @@ -25,6 +25,7 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; + extern const int INCORRECT_DATA; } static Parquet::ReadOptions convertReadOptions(const FormatSettings & format_settings) @@ -120,23 +121,68 @@ parquet::format::FileMetaData ParquetV3BlockInputFormat::getFileMetadata(Parquet } } -Chunk ParquetV3BlockInputFormat::read() +void ParquetV3BlockInputFormat::prepareNeedOnlyCountRowGroups(const parquet::format::FileMetaData & file_metadata) { - if (need_only_count) + const size_t num_row_groups = file_metadata.row_groups.size(); + std::vector global_offsets(num_row_groups + 1, 0); + for (size_t i = 0; i < num_row_groups; ++i) { - if (reported_count) - return {}; + if (file_metadata.row_groups[i].num_rows < 0) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Parquet row group {} has negative row count: {}", + i, + file_metadata.row_groups[i].num_rows); + global_offsets[i + 1] = global_offsets[i] + static_cast(file_metadata.row_groups[i].num_rows); + } - /// Don't init Reader and ReadManager if we only need file metadata. - Parquet::Prefetcher temp_prefetcher; - temp_prefetcher.init(in, read_options, parser_shared_resources); - parquet::format::FileMetaData file_metadata = getFileMetadata(temp_prefetcher); + need_only_count_row_groups.clear(); + need_only_count_next = 0; + if (!buckets_to_read) + { + if (global_offsets.back() > 0) + need_only_count_row_groups.push_back({.row_num_offset = 0, .num_rows = global_offsets.back()}); + return; + } - auto chunk = getChunkForCount(size_t(file_metadata.num_rows)); - chunk.getChunkInfos().add(std::make_shared(0)); + for (size_t row_group_id : buckets_to_read->row_group_ids) + { + if (row_group_id >= num_row_groups) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Parquet bucket row group {} is out of range (file has {} row groups)", + row_group_id, + num_row_groups); + + const size_t num_rows = static_cast(file_metadata.row_groups[row_group_id].num_rows); + if (num_rows == 0) + continue; + + need_only_count_row_groups.push_back( + {.row_num_offset = global_offsets[row_group_id], .num_rows = num_rows}); + } +} + +Chunk ParquetV3BlockInputFormat::read() +{ + if (need_only_count) + { + if (!need_only_count_prepared) + { + /// Don't init Reader and ReadManager if we only need file metadata. + Parquet::Prefetcher temp_prefetcher; + temp_prefetcher.init(in, read_options, parser_shared_resources); + prepareNeedOnlyCountRowGroups(getFileMetadata(temp_prefetcher)); + need_only_count_prepared = true; + } + + if (need_only_count_next >= need_only_count_row_groups.size()) + return {}; - reported_count = true; + const auto & row_group = need_only_count_row_groups[need_only_count_next++]; + auto chunk = getChunkForCount(row_group.num_rows); + chunk.getChunkInfos().add(std::make_shared(row_group.row_num_offset)); return chunk; } @@ -199,6 +245,9 @@ void ParquetV3BlockInputFormat::resetParser() reader.reset(); } previous_block_missing_values.clear(); + need_only_count_prepared = false; + need_only_count_next = 0; + need_only_count_row_groups.clear(); IInputFormat::resetParser(); } diff --git a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.h b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.h index 87a5192ae641..2c5baf00ea92 100644 --- a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.h +++ b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.h @@ -81,12 +81,21 @@ class ParquetV3BlockInputFormat final : public IInputFormat std::mutex reader_mutex; std::optional reader; - bool reported_count = false; // if need_only_count + bool need_only_count_prepared = false; + size_t need_only_count_next = 0; + + struct NeedOnlyCountRowGroup + { + size_t row_num_offset = 0; + size_t num_rows = 0; + }; + std::vector need_only_count_row_groups; BlockMissingValues previous_block_missing_values; size_t previous_approx_bytes_read_for_chunk = 0; void initializeIfNeeded(); + void prepareNeedOnlyCountRowGroups(const parquet::format::FileMetaData & file_metadata); std::shared_ptr buckets_to_read; parquet::format::FileMetaData getFileMetadata(Parquet::Prefetcher & prefetcher) const; diff --git a/src/Processors/Formats/Impl/PuffinBlockInputFormat.cpp b/src/Processors/Formats/Impl/PuffinBlockInputFormat.cpp new file mode 100644 index 000000000000..caea204cacab --- /dev/null +++ b/src/Processors/Formats/Impl/PuffinBlockInputFormat.cpp @@ -0,0 +1,1179 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ProfileEvents +{ +extern const Event PuffinFilesRead; +extern const Event PuffinFileReadMicroseconds; +} + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int LZ4_DECODER_FAILED; +} + +namespace +{ + +struct ScopedPuffinFileReadProfileEvent +{ + ProfileEventTimeIncrement watch; + + ScopedPuffinFileReadProfileEvent() + : watch(ProfileEvents::PuffinFileReadMicroseconds) + { + ProfileEvents::increment(ProfileEvents::PuffinFilesRead); + } +}; + +constexpr UInt8 PUFFIN_MAGIC[4] = {0x50, 0x46, 0x41, 0x31}; +constexpr UInt8 PUFFIN_FOOTER_COMPRESSED_FLAG = 0x01; +constexpr size_t PUFFIN_FOOTER_TRAILER_SIZE = 12; +constexpr size_t PUFFIN_FOOTER_LZ4_MAX_RATIO = 255; +/// Absolute cap on footer payload size (uncompressed JSON bytes, or declared LZ4 contentSize). +/// Prevents crafted FooterPayloadSize / contentSize values from forcing huge allocations. +/// Intentionally a compile-time safety ceiling (same class as other format hard caps), not a +/// FormatSettings / input_format_puffin_* knob: oversized footers are never legitimate input. +constexpr size_t PUFFIN_FOOTER_MAX_PAYLOAD_SIZE = 16 * 1024 * 1024; +/// Absolute cap on materialized deleted positions (~800 MiB of UInt64s at this limit). +/// Applied only when `deleted_rows` is requested; subset reads that skip materialization do not +/// enforce it. Intentionally not a FormatSettings knob: this is a fail-closed amplification +/// guard for Array(UInt64) expansion of roaring bitmaps, not a tunable acceptance policy. +/// Global `max_memory_usage` still applies later; this rejects before allocating the array. +constexpr UInt64 PUFFIN_DV_MAX_MATERIALIZED_POSITIONS = 100'000'000; +/// Absolute cap on on-disk deletion-vector-v1 blob length before reading bytes into memory. +/// Aligns with Iceberg DeleteLoader's 2 GiB content-size check. Fail closed before allocate: +/// footer `length` alone can point at the whole blob region while `cardinality` stays small. +/// Combined with an 8-byte envelope peek (magic + combined_length vs footer length) so a mismatched +/// header cannot force a full read up to this cap. A consistent huge envelope still reads up to the +/// cap (CRC needs the payload); streaming roaring without holding the blob is intentionally out of +/// scope. Intentionally a compile-time safety ceiling, not a FormatSettings knob. +constexpr size_t PUFFIN_DV_MAX_BLOB_SIZE = 2ULL * 1024 * 1024 * 1024; +constexpr UInt8 DELETION_VECTOR_MAGIC[4] = {0xD1, 0xD3, 0x39, 0x64}; +/// Matches Iceberg `RoaringPositionBitmap.MAX_POSITION` / iceberg-cpp `kMaxPosition`. +/// Key INT32_MAX is unsupported: the reference bitmap array length is a signed Int32. +constexpr Int64 DELETION_VECTOR_MAX_POSITION = 0x7FFFFFFE80000000LL; +constexpr Int32 DELETION_VECTOR_MAX_KEY = std::numeric_limits::max() - 1; + +UInt32 readBigEndianUInt32(const UInt8 * data) +{ + return (static_cast(data[0]) << 24) + | (static_cast(data[1]) << 16) + | (static_cast(data[2]) << 8) + | static_cast(data[3]); +} + +UInt64 positionFromKeyAndSubPosition(UInt32 key, UInt32 sub_position) +{ + return (static_cast(key) << 32) | static_cast(sub_position); +} + +void checkMagic(const UInt8 * p, const char * context) +{ + if (std::memcmp(p, PUFFIN_MAGIC, 4) != 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid Puffin magic ({})", context); +} + +/// `blob_region_end` is the end of the `Blob₁ ... Blobₙ` region, i.e. the offset of the footer's +/// leading magic. Blob offset/length must stay within [PUFFIN_MAGIC, blob_region_end), so a malformed +/// footer cannot point a blob into the footer payload or trailer. +void validatePuffinBlobBounds(Int64 offset, Int64 length, size_t blob_region_end, size_t blob_index) +{ + if (offset < static_cast(sizeof(PUFFIN_MAGIC)) || length < 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: offset/length out of bounds", blob_index); + + Int64 end = 0; + if (common::addOverflow(offset, length, end)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: offset/length out of bounds", blob_index); + + if (static_cast(end) > blob_region_end) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: offset/length out of bounds", blob_index); +} + +void validatePuffinFooterFlags(const UInt8 flags[4]) +{ + if (flags[1] != 0 || flags[2] != 0 || flags[3] != 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown Puffin footer flags"); + if ((flags[0] & ~PUFFIN_FOOTER_COMPRESSED_FLAG) != 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown Puffin footer flags"); +} + +String decompressPuffinFooterPayload(const char * data, size_t size) +{ + LZ4F_dctx * dctx = nullptr; + size_t ret = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION); + if (LZ4F_isError(ret)) + throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Failed to create LZ4 decompression context: {}", LZ4F_getErrorName(ret)); + + struct DecompressionContextGuard + { + LZ4F_dctx * dctx; + explicit DecompressionContextGuard(LZ4F_dctx * dctx_) : dctx(dctx_) { } + ~DecompressionContextGuard() { LZ4F_freeDecompressionContext(dctx); } + }; + DecompressionContextGuard guard(dctx); + + LZ4F_frameInfo_t frame_info{}; + size_t header_size = size; + ret = LZ4F_getFrameInfo(dctx, &frame_info, data, &header_size); + if (LZ4F_isError(ret)) + throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Failed to read LZ4 frame info for Puffin footer: {}", LZ4F_getErrorName(ret)); + + const char * src = data + header_size; + size_t src_remaining = size - header_size; + + size_t max_by_ratio = 0; + if (common::mulOverflow(size, PUFFIN_FOOTER_LZ4_MAX_RATIO, max_by_ratio)) + throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Puffin footer compressed payload is too large"); + + if (frame_info.contentSize == 0) + throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Puffin footer LZ4 frame must declare content size"); + + const size_t max_decompressed_size = std::min(max_by_ratio, PUFFIN_FOOTER_MAX_PAYLOAD_SIZE); + if (frame_info.contentSize > max_decompressed_size) + { + if (max_decompressed_size == PUFFIN_FOOTER_MAX_PAYLOAD_SIZE) + throw Exception( + ErrorCodes::LZ4_DECODER_FAILED, + "Puffin footer LZ4 content size {} exceeds absolute decompression limit {}", + frame_info.contentSize, + PUFFIN_FOOTER_MAX_PAYLOAD_SIZE); + + throw Exception( + ErrorCodes::LZ4_DECODER_FAILED, + "Puffin footer LZ4 content size {} exceeds decompression limit {}", + frame_info.contentSize, + max_decompressed_size); + } + + String result; + result.resize(static_cast(frame_info.contentSize)); + + size_t dst_offset = 0; + while (true) + { + size_t src_read = src_remaining; + size_t dst_write = result.size() - dst_offset; + ret = LZ4F_decompress(dctx, result.data() + dst_offset, &dst_write, src, &src_read, nullptr); + if (LZ4F_isError(ret)) + throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Failed to decompress Puffin footer: {}", LZ4F_getErrorName(ret)); + + src += src_read; + src_remaining -= src_read; + dst_offset += dst_write; + + if (ret == 0) + break; + + /// No forward progress while the frame is still open means the input is truncated: + /// LZ4F_decompress wants more input but we have given it everything. Stop to avoid spinning. + if (dst_write == 0 && src_read == 0) + throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Puffin footer LZ4 frame is incomplete"); + + if (dst_offset == result.size()) + throw Exception( + ErrorCodes::LZ4_DECODER_FAILED, + "Puffin footer decompressed size exceeds declared content size {}", + frame_info.contentSize); + } + + if (dst_offset != frame_info.contentSize) + throw Exception( + ErrorCodes::LZ4_DECODER_FAILED, + "Puffin footer LZ4 decompressed size {} does not match content size {}", + dst_offset, + frame_info.contentSize); + + if (src_remaining != 0) + throw Exception( + ErrorCodes::LZ4_DECODER_FAILED, + "Puffin footer LZ4 frame has {} trailing bytes", + src_remaining); + + result.resize(dst_offset); + return result; +} + +void requireBlobMetadataField(const Poco::JSON::Object::Ptr & blob_obj, const char * field_name, size_t blob_index) +{ + if (!blob_obj->has(field_name) || blob_obj->isNull(field_name)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: missing required field '{}'", blob_index, field_name); +} + +/// Poco reports JSON booleans as integers (`std::numeric_limits::is_integer`). +bool isJSONInteger(const Poco::Dynamic::Var & value) +{ + return value.isInteger() && !value.isBoolean(); +} + +/// Poco stores JSON integers that do not fit signed Int64 as unsigned; convert() would throw RangeException. +std::optional tryJSONIntegerAsInt64(const Poco::Dynamic::Var & value) +{ + if (value.isInteger() && !value.isSigned()) + { + const auto as_uint64 = value.convert(); + if (as_uint64 > static_cast(std::numeric_limits::max())) + return std::nullopt; + return static_cast(as_uint64); + } + + return value.convert(); +} + +Int64 requireBlobMetadataInt64(const Poco::JSON::Object::Ptr & blob_obj, const char * field_name, size_t blob_index) +{ + requireBlobMetadataField(blob_obj, field_name, blob_index); + const Poco::Dynamic::Var & value = blob_obj->get(field_name); + if (!isJSONInteger(value)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field '{}' must be an integer", blob_index, field_name); + + auto as_int64 = tryJSONIntegerAsInt64(value); + if (!as_int64) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field '{}' is out of Int64 range", blob_index, field_name); + return *as_int64; +} + +Int32 requireBlobMetadataFieldsElementInt32(const Poco::Dynamic::Var & value, size_t blob_index, size_t field_index) +{ + if (!isJSONInteger(value)) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: fields[{}] must be an integer", blob_index, field_index); + + auto as_int64 = tryJSONIntegerAsInt64(value); + if (!as_int64) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: fields[{}] is out of Int64 range", blob_index, field_index); + + if (*as_int64 < std::numeric_limits::min() || *as_int64 > std::numeric_limits::max()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: fields[{}] is out of Int32 range", blob_index, field_index); + + return static_cast(*as_int64); +} + +String requireJSONStringValue(const Poco::Dynamic::Var & value, size_t blob_index, const char * field_name) +{ + if (!value.isString()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field '{}' must be a string", blob_index, field_name); + return value.extract(); +} + +String requireBlobMetadataString(const Poco::JSON::Object::Ptr & blob_obj, const char * field_name, size_t blob_index) +{ + requireBlobMetadataField(blob_obj, field_name, blob_index); + return requireJSONStringValue(blob_obj->get(field_name), blob_index, field_name); +} + +String optBlobMetadataString(const Poco::JSON::Object::Ptr & blob_obj, const char * field_name, size_t blob_index) +{ + if (!blob_obj->has(field_name)) + return {}; + if (blob_obj->isNull(field_name)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field '{}' must be a string", blob_index, field_name); + return requireJSONStringValue(blob_obj->get(field_name), blob_index, field_name); +} + +void requireDeletionVectorV1Properties(const PuffinBlob & blob, size_t blob_index) +{ + static constexpr const char * required_properties[] = {"referenced-data-file", "cardinality"}; + for (const char * key : required_properties) + { + auto it = blob.properties.find(key); + if (it == blob.properties.end() || it->second.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {}: deletion-vector-v1 missing required property '{}'", + blob_index, + key); + } + + UInt64 cardinality = 0; + if (!tryParse(cardinality, blob.properties.at("cardinality"))) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {}: deletion-vector-v1 property 'cardinality' must be an unsigned integer", + blob_index); +} + +void parseStringValuedProperties( + const Poco::JSON::Object::Ptr & props_obj, + std::map * out, + bool for_blob, + size_t blob_index) +{ + for (const auto & [key, val] : *props_obj) + { + if (!val.isString()) + { + if (for_blob) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: property '{}' must be a string", blob_index, key); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer property '{}' must be a string", key); + } + if (out) + out->emplace(key, val.extract()); + } +} + +void parseBlobProperties(const Poco::JSON::Object::Ptr & blob_obj, PuffinBlob & blob, size_t blob_index, bool required) +{ + if (!blob_obj->has("properties")) + { + if (required) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: missing required field 'properties'", blob_index); + return; + } + + if (blob_obj->isNull("properties")) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field 'properties' must be an object", blob_index); + + auto props_obj = blob_obj->getObject("properties"); + if (!props_obj) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field 'properties' must be an object", blob_index); + + parseStringValuedProperties(props_obj, &blob.properties, /*for_blob=*/true, blob_index); +} + +/// Optional FileMetadata.properties: JSON object with string values (Iceberg Puffin spec). +/// Absent is allowed; present null / non-object is rejected. +void validateFileMetadataProperties(const Poco::JSON::Object::Ptr & footer_obj) +{ + if (!footer_obj->has("properties")) + return; + + if (footer_obj->isNull("properties")) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer field 'properties' must be an object"); + + auto props_obj = footer_obj->getObject("properties"); + if (!props_obj) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer field 'properties' must be an object"); + + parseStringValuedProperties(props_obj, /*out=*/nullptr, /*for_blob=*/false, /*blob_index=*/0); +} + +std::vector parseFooterJSON(const String & footer_json, size_t blob_region_end) +{ + Poco::JSON::Parser parser; + Poco::Dynamic::Var root; + try + { + root = parser.parse(footer_json); + } + catch (const Exception &) + { + throw; + } + catch (const Poco::Exception & e) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse Puffin footer JSON: {}", e.displayText()); + } + + if (root.type() != typeid(Poco::JSON::Object::Ptr)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer JSON must be an object"); + + auto obj = root.extract(); + + validateFileMetadataProperties(obj); + + if (!obj->has("blobs") || obj->isNull("blobs")) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer is missing required field 'blobs'"); + + auto blobs_arr = obj->getArray("blobs"); + if (!blobs_arr) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer field 'blobs' must be an array"); + + std::vector blobs; + for (size_t i = 0; i < blobs_arr->size(); ++i) + { + auto blob_obj = blobs_arr->getObject(static_cast(i)); + if (!blob_obj) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: must be an object", i); + + PuffinBlob blob; + + blob.type = requireBlobMetadataString(blob_obj, "type", i); + blob.snapshot_id = requireBlobMetadataInt64(blob_obj, "snapshot-id", i); + blob.sequence_number = requireBlobMetadataInt64(blob_obj, "sequence-number", i); + blob.offset = requireBlobMetadataInt64(blob_obj, "offset", i); + blob.length = requireBlobMetadataInt64(blob_obj, "length", i); + + if (blob.type == "deletion-vector-v1") + { + if (blob.snapshot_id != -1 || blob.sequence_number != -1) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {}: deletion-vector-v1 snapshot-id and sequence-number must be -1", + i); + + /// Spec requires the key to be omitted; present null is not omission. + if (blob_obj->has("compression-codec")) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {}: deletion-vector-v1 must omit 'compression-codec'", + i); + + parseBlobProperties(blob_obj, blob, i, /*required=*/true); + requireDeletionVectorV1Properties(blob, i); + } + else + { + blob.compression_codec = optBlobMetadataString(blob_obj, "compression-codec", i); + parseBlobProperties(blob_obj, blob, i, /*required=*/false); + } + + requireBlobMetadataField(blob_obj, "fields", i); + auto fields_arr = blob_obj->getArray("fields"); + if (!fields_arr) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field 'fields' must be an array", i); + + for (size_t j = 0; j < fields_arr->size(); ++j) + blob.fields.push_back(requireBlobMetadataFieldsElementInt32(fields_arr->get(static_cast(j)), i, j)); + + validatePuffinBlobBounds(blob.offset, blob.length, blob_region_end, i); + + blobs.push_back(std::move(blob)); + } + return blobs; +} + +std::vector readPuffinFooterFromSeekable(SeekableReadBuffer & seekable, size_t file_size) +{ + if (file_size < 16) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin file too small"); + + seekable.seek(0, SEEK_SET); + char magic_buf[4]; + seekable.readStrict(magic_buf, 4); + checkMagic(reinterpret_cast(magic_buf), "header"); + + seekable.seek(static_cast(file_size - PUFFIN_FOOTER_TRAILER_SIZE), SEEK_SET); + Int32 footer_length_signed = 0; + readBinaryLittleEndian(footer_length_signed, seekable); + + UInt8 flags[4] = {}; + seekable.readStrict(reinterpret_cast(flags), sizeof(flags)); + + char trailing_buf[4]; + seekable.readStrict(trailing_buf, 4); + checkMagic(reinterpret_cast(trailing_buf), "trailing"); + validatePuffinFooterFlags(flags); + + if (footer_length_signed <= 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid Puffin footer length: {}", footer_length_signed); + + UInt64 footer_trailer_size = 0; + if (common::addOverflow( + static_cast(footer_length_signed), + static_cast(PUFFIN_FOOTER_TRAILER_SIZE), + footer_trailer_size) + || footer_trailer_size > file_size) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid Puffin footer length: {}", footer_length_signed); + + const size_t footer_length = static_cast(footer_length_signed); + const size_t payload_start = file_size - PUFFIN_FOOTER_TRAILER_SIZE - footer_length; + /// Footer layout is Magic | FooterPayload | …; require a distinct footer-open Magic after the + /// file header Magic (payload_start == 4 would reuse the header Magic when blobs is empty). + if (payload_start < 2 * sizeof(PUFFIN_MAGIC)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid Puffin footer length: {}", footer_length_signed); + + if (footer_length > PUFFIN_FOOTER_MAX_PAYLOAD_SIZE) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin footer payload size {} exceeds absolute limit {}", + footer_length, + PUFFIN_FOOTER_MAX_PAYLOAD_SIZE); + + seekable.seek(static_cast(payload_start - sizeof(PUFFIN_MAGIC)), SEEK_SET); + char leading_magic_buf[4]; + seekable.readStrict(leading_magic_buf, 4); + checkMagic(reinterpret_cast(leading_magic_buf), "footer"); + + String footer_payload(footer_length, '\0'); + seekable.seek(static_cast(payload_start), SEEK_SET); + seekable.readStrict(footer_payload.data(), footer_length); + + String footer_json; + if ((flags[0] & PUFFIN_FOOTER_COMPRESSED_FLAG) != 0) + footer_json = decompressPuffinFooterPayload(footer_payload.data(), footer_payload.size()); + else + footer_json = std::move(footer_payload); + + const size_t blob_region_end = payload_start - sizeof(PUFFIN_MAGIC); + return parseFooterJSON(footer_json, blob_region_end); +} + +PuffinFooter readPuffinFooter(ReadBuffer & buf, bool seekable_read) +{ + ScopedPuffinFileReadProfileEvent profile_event; + + PuffinFooter result; + + auto * seekable = dynamic_cast(&buf); + auto file_size_opt = tryGetFileSizeFromReadBuffer(buf); + + /// Pipes/FIFOs are SeekableReadBuffer subclasses but fstat reports size 0; require a real + /// regular file before trusting seek+size (same pattern as ORC/Arrow). Also honor + /// `input_format_allow_seeks` via FormatSettings.seekable_read. + if (seekable_read && seekable && seekable->checkIfActuallySeekable() && file_size_opt) + { + result.blobs = readPuffinFooterFromSeekable(*seekable, *file_size_opt); + } + else + { + /// Fail fast on format mismatch before buffering the rest of a (possibly huge) stream — + /// same pattern as `asArrowFileLoadIntoMemory`. + result.data.resize(sizeof(PUFFIN_MAGIC)); + const size_t magic_read = buf.read(reinterpret_cast(result.data.data()), sizeof(PUFFIN_MAGIC)); + if (magic_read < sizeof(PUFFIN_MAGIC)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin file too small"); + checkMagic(result.data.data(), "header"); + + std::vector tmp(DEFAULT_BLOCK_SIZE); + while (!buf.eof()) + { + size_t n = buf.read(reinterpret_cast(tmp.data()), tmp.size()); + result.data.insert(result.data.end(), tmp.data(), tmp.data() + n); + } + + ReadBufferFromMemory mem_buf(result.data.data(), result.data.size()); + result.blobs = readPuffinFooterFromSeekable(mem_buf, result.data.size()); + } + + return result; +} + +String readPuffinBlobBytes( + const PuffinBlob & blob, ReadBuffer & buf, const std::vector & data, bool seekable_read) +{ + const size_t length = static_cast(blob.length); + + /// When the footer path buffered the whole file, copy from that buffer — the original input + /// may still look Seekable (e.g. a consumed pipe) but must not be seeked. + if (!data.empty()) + { + if (static_cast(blob.offset) + static_cast(blob.length) > data.size()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob offset/length out of bounds of buffered data"); + + return String(reinterpret_cast(data.data() + blob.offset), length); + } + + auto * seekable = dynamic_cast(&buf); + if (!seekable_read || !seekable || !seekable->checkIfActuallySeekable()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot read Puffin blob: input is not seekable and was not buffered"); + + seekable->seek(blob.offset, SEEK_SET); + String result(length, '\0'); + seekable->readStrict(result.data(), length); + return result; +} + +void readDeletionVectorEnvelopePrefix( + const PuffinBlob & blob, ReadBuffer & buf, const std::vector & data, bool seekable_read, UInt8 header[8]) +{ + if (!data.empty()) + { + if (static_cast(blob.offset) + 8 > data.size()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob offset/length out of bounds of buffered data"); + std::memcpy(header, data.data() + static_cast(blob.offset), 8); + return; + } + + auto * seekable = dynamic_cast(&buf); + if (!seekable_read || !seekable || !seekable->checkIfActuallySeekable()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot read Puffin blob: input is not seekable and was not buffered"); + + seekable->seek(blob.offset, SEEK_SET); + seekable->readStrict(reinterpret_cast(header), 8); +} + +/// Validate absolute size and the first 8 envelope bytes (combined_length + magic) before allocating +/// `blob.length`. CRC still requires the full payload and is checked later in +/// `extractDeletionVectorPayload` after a bounded read — we intentionally do not stream roaring +/// deserialize here (larger redesign; CRC coverage is over magic+vector). +String readDeletionVectorBlobBytes( + const PuffinBlob & blob, ReadBuffer & buf, const std::vector & data, bool seekable_read) +{ + if (blob.length < 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector blob length is negative"); + + if (static_cast(blob.length) > PUFFIN_DV_MAX_BLOB_SIZE) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector blob length {} exceeds absolute limit {}", + blob.length, + PUFFIN_DV_MAX_BLOB_SIZE); + + if (static_cast(blob.length) < 12) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector blob is too small"); + + UInt8 header[8]; + readDeletionVectorEnvelopePrefix(blob, buf, data, seekable_read, header); + + const UInt32 combined_length = readBigEndianUInt32(header); + if (std::memcmp(header + sizeof(UInt32), DELETION_VECTOR_MAGIC, sizeof(DELETION_VECTOR_MAGIC)) != 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector magic"); + + if (combined_length < sizeof(DELETION_VECTOR_MAGIC)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector combined length: {}", combined_length); + + UInt64 expected_blob_size = 0; + if (common::addOverflow(static_cast(combined_length), UInt64{8}, expected_blob_size)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector combined length: {}", combined_length); + + if (static_cast(blob.length) != expected_blob_size) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector blob size {} does not match combined length {}", + blob.length, + combined_length); + + return readPuffinBlobBytes(blob, buf, data, seekable_read); +} + +roaring::Roaring readRoaringPortableSafe(const char * data, size_t size, Int32 key) +{ + try + { + return roaring::Roaring::readSafe(data, size); + } + catch (const std::exception & e) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Failed to deserialize deletion vector roaring bitmap at key {}: {}", key, e.what()); + } +} + +void deserializeRoaringPositionBitmap(std::string_view bytes, UInt64 expected_cardinality, ColumnUInt64 & positions) +{ + if (bytes.size() < sizeof(Int64)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector bitmap is too small"); + + const char * ptr = bytes.data(); + size_t remaining = bytes.size(); + + /// Iceberg deletion-vector roaring layout stores count and keys as little-endian. + const Int64 bitmap_count = unalignedLoadLittleEndian(ptr); + ptr += sizeof(Int64); + remaining -= sizeof(Int64); + + if (bitmap_count < 0 || bitmap_count > std::numeric_limits::max()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector bitmap count: {}", bitmap_count); + + Int32 last_key = -1; + Int32 remaining_count = static_cast(bitmap_count); + UInt64 running_cardinality = 0; + + while (remaining_count > 0) + { + if (remaining < sizeof(Int32)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector bitmap is truncated while reading key"); + + const Int32 key = unalignedLoadLittleEndian(ptr); + ptr += sizeof(Int32); + remaining -= sizeof(Int32); + + if (key < 0 || key > DELETION_VECTOR_MAX_KEY) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector bitmap key: {}", key); + if (key <= last_key) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector bitmap keys must be sorted in ascending order"); + + auto bitmap = readRoaringPortableSafe(ptr, remaining, key); + + const size_t bitmap_size = bitmap.getSizeInBytes(/*portable=*/true); + if (bitmap_size > remaining) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector roaring bitmap at key {} exceeds blob size", key); + + const UInt64 bitmap_cardinality = bitmap.cardinality(); + UInt64 new_running_cardinality = 0; + if (common::addOverflow(running_cardinality, bitmap_cardinality, new_running_cardinality)) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector cardinality exceeds declared cardinality {}", + expected_cardinality); + + if (new_running_cardinality > expected_cardinality) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector cardinality {} exceeds declared cardinality {}", + new_running_cardinality, + expected_cardinality); + + running_cardinality = new_running_cardinality; + + for (UInt32 sub_position : bitmap) + { + const UInt64 position = positionFromKeyAndSubPosition(static_cast(key), sub_position); + if (position > static_cast(DELETION_VECTOR_MAX_POSITION)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector position {} is out of supported range", position); + positions.insertValue(position); + } + + ptr += bitmap_size; + remaining -= bitmap_size; + last_key = key; + --remaining_count; + } + + if (remaining != 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector bitmap has {} trailing bytes", remaining); + + if (running_cardinality != expected_cardinality) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector cardinality {} does not match deserialized row count {}", + expected_cardinality, + running_cardinality); +} + +std::string_view extractDeletionVectorPayload(std::string_view blob) +{ + // Envelope size/magic are also checked in readDeletionVectorBlobBytes before allocate; CRC must + // run on the full payload and stays here after the bounded read. + if (blob.size() < 12) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector blob is too small"); + + const auto * blob_bytes = reinterpret_cast(blob.data()); + const UInt32 combined_length = readBigEndianUInt32(blob_bytes); + if (combined_length < sizeof(DELETION_VECTOR_MAGIC)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector combined length: {}", combined_length); + + const size_t vector_size = combined_length - sizeof(DELETION_VECTOR_MAGIC); + const size_t expected_blob_size = sizeof(UInt32) + combined_length + sizeof(UInt32); + if (blob.size() != expected_blob_size) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector blob size {} does not match combined length {}", blob.size(), combined_length); + + if (std::memcmp(blob_bytes + sizeof(UInt32), DELETION_VECTOR_MAGIC, sizeof(DELETION_VECTOR_MAGIC)) != 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector magic"); + + const UInt8 * crc_input = blob_bytes + sizeof(UInt32); + const size_t crc_input_size = combined_length; + const UInt32 expected_crc = readBigEndianUInt32(blob_bytes + sizeof(UInt32) + combined_length); + const UInt32 actual_crc = static_cast(crc32_z(0L, reinterpret_cast(crc_input), crc_input_size)); + if (expected_crc != actual_crc) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector CRC mismatch"); + + return std::string_view(blob.data() + 2 * sizeof(UInt32), vector_size); +} + +void deserializeDeletionVectorV1(std::string_view blob, UInt64 expected_cardinality, ColumnUInt64 & positions) +{ + // See PUFFIN_DV_MAX_MATERIALIZED_POSITIONS: hard ceiling before payload parse / array reserve. + if (expected_cardinality > PUFFIN_DV_MAX_MATERIALIZED_POSITIONS) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector cardinality {} exceeds materialization limit {}", + expected_cardinality, + PUFFIN_DV_MAX_MATERIALIZED_POSITIONS); + + deserializeRoaringPositionBitmap(extractDeletionVectorPayload(blob), expected_cardinality, positions); +} + +NamesAndTypesList getPuffinMetadataSchema() +{ + return { + {"blob_type", std::make_shared()}, + {"snapshot_id", std::make_shared()}, + {"sequence_number", std::make_shared()}, + {"fields", std::make_shared(std::make_shared())}, + {"offset", std::make_shared()}, + {"length", std::make_shared()}, + {"compression_codec", std::make_shared()}, + {"properties", std::make_shared(std::make_shared(), std::make_shared())}, + }; +} + +NamesAndTypesList getPuffinSchema() +{ + return { + {"referenced_data_file", std::make_shared()}, + {"deleted_rows", std::make_shared(std::make_shared())}, + }; +} + +void checkPuffinFormatHeader(const Block & header, const NamesAndTypesList & expected_schema, const char * format_name) +{ + std::unordered_map name_to_type; + for (const auto & [name, type] : expected_schema) + name_to_type[name] = type; + + String allowed_columns; + for (const auto & [name, type] : expected_schema) + { + if (!allowed_columns.empty()) + allowed_columns += ", "; + allowed_columns += name; + } + + for (const auto & [name, type] : header.getNamesAndTypes()) + { + auto it = name_to_type.find(name); + if (it == name_to_type.end()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Unexpected column: {}. {} format allows only the next columns: {}", + name, + format_name, + allowed_columns); + + if (!it->second->equals(*type)) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Unexpected type {} for column {}. Expected type: {}", + type->getName(), + name, + it->second->getName()); + } +} + +void checkPuffinMetadataHeader(const Block & header) +{ + checkPuffinFormatHeader(header, getPuffinMetadataSchema(), "PuffinMetadata"); +} + +void checkPuffinHeader(const Block & header) +{ + checkPuffinFormatHeader(header, getPuffinSchema(), "Puffin"); +} + +} + +PuffinMetadataInputFormat::PuffinMetadataInputFormat(ReadBuffer & buf, SharedHeader header_, const FormatSettings & format_settings_) + : IInputFormat(std::move(header_), &buf) + , seekable_read(format_settings_.seekable_read) +{ + checkPuffinMetadataHeader(getPort().getHeader()); +} + +Chunk PuffinMetadataInputFormat::read() +{ + if (!initialized) + { + blob_index = 0; + footer = readPuffinFooter(*in, seekable_read); + /// Metadata never reads blob payloads; drop the full-file buffer from the non-seekable path. + footer.data.clear(); + footer.data.shrink_to_fit(); + initialized = true; + } + if (footer.blobs.size() <= blob_index) + return {}; + + const PuffinBlob & blob = footer.blobs[blob_index++]; + + auto col_type = ColumnString::create(); + auto col_snap = ColumnInt64::create(); + auto col_seq = ColumnInt64::create(); + auto col_fields_data = ColumnInt32::create(); + auto col_fields_offsets = ColumnArray::ColumnOffsets::create(); + auto col_offset = ColumnInt64::create(); + auto col_length = ColumnInt64::create(); + auto col_codec = ColumnString::create(); + auto col_props_keys = ColumnString::create(); + auto col_props_vals = ColumnString::create(); + auto col_props_offsets = ColumnArray::ColumnOffsets::create(); + + col_type->insertData(blob.type.data(), blob.type.size()); + col_snap->insertValue(blob.snapshot_id); + col_seq->insertValue(blob.sequence_number); + for (Int32 f : blob.fields) + col_fields_data->insertValue(f); + col_fields_offsets->insertValue(blob.fields.size()); + col_offset->insertValue(blob.offset); + col_length->insertValue(blob.length); + col_codec->insertData(blob.compression_codec.data(), blob.compression_codec.size()); + for (const auto & [k, v] : blob.properties) + { + col_props_keys->insertData(k.data(), k.size()); + col_props_vals->insertData(v.data(), v.size()); + } + col_props_offsets->insertValue(blob.properties.size()); + + auto col_fields = ColumnArray::create(std::move(col_fields_data), std::move(col_fields_offsets)); + MutableColumns prop_cols; + prop_cols.push_back(std::move(col_props_keys)); + prop_cols.push_back(std::move(col_props_vals)); + MutableColumnPtr col_props_tuple = ColumnTuple::create(std::move(prop_cols)); + MutableColumnPtr col_props_arr = ColumnArray::create(std::move(col_props_tuple), std::move(col_props_offsets)); + MutableColumnPtr col_props = ColumnMap::create(std::move(col_props_arr)); + + std::unordered_map built; + built.emplace("blob_type", std::move(col_type)); + built.emplace("snapshot_id", std::move(col_snap)); + built.emplace("sequence_number", std::move(col_seq)); + built.emplace("fields", std::move(col_fields)); + built.emplace("offset", std::move(col_offset)); + built.emplace("length", std::move(col_length)); + built.emplace("compression_codec", std::move(col_codec)); + built.emplace("properties", std::move(col_props)); + + const Block & out_header = getPort().getHeader(); + MutableColumns result; + result.reserve(out_header.columns()); + for (const auto & col_with_name : out_header) + result.push_back(std::move(built.at(col_with_name.name))); + return Chunk(std::move(result), 1); +} + +PuffinInputFormat::PuffinInputFormat(ReadBuffer & buf, SharedHeader header_, const FormatSettings & format_settings_) + : IInputFormat(std::move(header_), &buf) + , seekable_read(format_settings_.seekable_read) +{ + checkPuffinHeader(getPort().getHeader()); + need_deleted_rows = getPort().getHeader().has("deleted_rows"); +} + +Chunk PuffinInputFormat::read() +{ + if (!initialized) + { + blob_index = 0; + footer = readPuffinFooter(*in, seekable_read); + /// No deletion-vector payload will be read; drop the full-file buffer from the non-seekable path. + if (!need_deleted_rows) + { + footer.data.clear(); + footer.data.shrink_to_fit(); + } + initialized = true; + } + + while (blob_index < footer.blobs.size()) + { + const size_t current_blob_index = blob_index; + const auto & blob = footer.blobs[blob_index++]; + + if (blob.type != "deletion-vector-v1") + continue; + + const auto & referenced_data_file = blob.properties.at("referenced-data-file"); + + UInt64 expected_cardinality = 0; + if (!tryParse(expected_cardinality, blob.properties.at("cardinality"))) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {}: deletion-vector-v1 property 'cardinality' must be an unsigned integer", + current_blob_index); + + auto col_file = ColumnString::create(); + col_file->insertData(referenced_data_file.data(), referenced_data_file.size()); + + MutableColumnPtr col_rows; + // Payload I/O and validation (envelope / CRC / roaring) run only when `deleted_rows` is + // projected. Footer property checks above stay projection-independent. + // Intentional: subset reads of `referenced_data_file` must not force reading up to + // PUFFIN_DV_MAX_BLOB_SIZE for blobs that will never be materialized. + // Materialization ceiling uses footer `cardinality` and must run before blob allocate. + if (need_deleted_rows) + { + if (expected_cardinality > PUFFIN_DV_MAX_MATERIALIZED_POSITIONS) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector cardinality {} exceeds materialization limit {}", + expected_cardinality, + PUFFIN_DV_MAX_MATERIALIZED_POSITIONS); + + const String blob_data = readDeletionVectorBlobBytes(blob, *in, footer.data, seekable_read); + auto col_rows_data = ColumnUInt64::create(); + deserializeDeletionVectorV1(blob_data, expected_cardinality, *col_rows_data); + + auto col_rows_offsets = ColumnArray::ColumnOffsets::create(); + col_rows_offsets->insertValue(col_rows_data->size()); + col_rows = ColumnArray::create(std::move(col_rows_data), std::move(col_rows_offsets)); + } + + const Block & out_header = getPort().getHeader(); + std::unordered_map built; + built.emplace("referenced_data_file", std::move(col_file)); + if (need_deleted_rows) + built.emplace("deleted_rows", std::move(col_rows)); + + MutableColumns result; + result.reserve(out_header.columns()); + for (const auto & col_with_name : out_header) + result.push_back(std::move(built.at(col_with_name.name))); + return Chunk(std::move(result), 1); + } + + return {}; +} + +void PuffinMetadataInputFormat::resetParser() +{ + IInputFormat::resetParser(); + initialized = false; + blob_index = 0; + footer = {}; +} + +void PuffinInputFormat::resetParser() +{ + IInputFormat::resetParser(); + initialized = false; + blob_index = 0; + footer = {}; +} + +PuffinMetadataSchemaReader::PuffinMetadataSchemaReader(ReadBuffer & in_) + : ISchemaReader(in_) +{ +} + +NamesAndTypesList PuffinMetadataSchemaReader::readSchema() +{ + return getPuffinMetadataSchema(); +} + +PuffinSchemaReader::PuffinSchemaReader(ReadBuffer & in_) + : ISchemaReader(in_) +{ +} + +NamesAndTypesList PuffinSchemaReader::readSchema() +{ + return getPuffinSchema(); +} + +void registerInputFormatPuffin(FormatFactory & factory) +{ + factory.registerInputFormat( + "PuffinMetadata", + [](ReadBuffer & buf, const Block & sample, const RowInputFormatParams &, const FormatSettings & settings) + { return std::make_shared(buf, std::make_shared(sample), settings); }); + factory.markFormatSupportsSubsetOfColumns("PuffinMetadata"); + + factory.setDocumentation("PuffinMetadata", Documentation{ + .description = R"DOCS_MD( +## Description {#description} + +Special input format for reading [Apache Iceberg Puffin](https://iceberg.apache.org/puffin-spec/) file footer metadata. +It outputs one row per blob entry from the footer `BlobMetadata` list. + +Fixed output columns: +- `blob_type` (`String`) - blob type, for example `deletion-vector-v1` +- `snapshot_id` (`Int64`) - snapshot id of the blob +- `sequence_number` (`Int64`) - sequence number of the blob +- `fields` (`Array(Int32)`) - list of field ids the blob applies to +- `offset` (`Int64`) - offset of the blob payload in the file +- `length` (`Int64`) - length of the blob payload in bytes +- `compression_codec` (`String`) - compression codec of the blob payload, if present +- `properties` (`Map(String, String)`) - blob-specific properties + +Optional top-level `FileMetadata.properties` in the footer (for example `created-by`) are type-checked when present but are not returned as columns. If the key is present it must be a JSON object with string values (null is rejected). + +LZ4-compressed and uncompressed puffin footers are supported. Footer payload size (and declared LZ4 content size) is bounded by a compression ratio where applicable and an absolute ceiling; oversized footers are rejected before allocation. + +## Example usage {#example-usage} + +Inspect footer blobs: + +```sql +SELECT blob_type, snapshot_id, sequence_number, offset, length, compression_codec, + mapKeys(properties), mapValues(properties) +FROM file(deletes.puffin, PuffinMetadata); +``` + +Pair with the `Puffin` format to read `deletion-vector-v1` blob payloads. +)DOCS_MD", + .related = {"Puffin"}, + }); + + factory.registerInputFormat( + "Puffin", + [](ReadBuffer & buf, const Block & sample, const RowInputFormatParams &, const FormatSettings & settings) + { return std::make_shared(buf, std::make_shared(sample), settings); }); + factory.markFormatSupportsSubsetOfColumns("Puffin"); + + factory.setDocumentation("Puffin", Documentation{ + .description = R"DOCS_MD( +## Description {#description} + +Input format for reading [Apache Iceberg Puffin](https://iceberg.apache.org/puffin-spec/) files. + +The format exposes deleted row positions from `deletion-vector-v1` blobs. Other blob types (for example `apache-datasketches-theta-v1`) are skipped. +If a puffin file contains multiple `deletion-vector-v1` blobs, the format outputs one row per such blob. + +Fixed output columns: +- `referenced_data_file` (`String`) - location of the data file the deletion vector applies to (`referenced-data-file` blob property) +- `deleted_rows` (`Array(UInt64)`) - 64-bit row positions deleted according to the deletion vector roaring bitmap + +Deletion vectors whose declared `cardinality` exceeds an absolute materialization ceiling are rejected when `deleted_rows` is requested, before the blob is allocated. Footer `deletion-vector-v1` properties (including that `cardinality` parses as an unsigned integer) are always validated. Selecting only `referenced_data_file` skips on-disk payload I/O and therefore also skips envelope, CRC, roaring deserialize, and the materialization ceiling — intentionally, so a path-only projection does not read up to the blob-size cap. + +On-disk `deletion-vector-v1` blob length is bounded by an absolute ceiling (aligned with Iceberg's 2 GiB content-size check). When `deleted_rows` is requested, the reader peeks the envelope header (combined length and magic) before allocating the full payload; CRC is verified after the bounded read. + +LZ4-compressed and uncompressed puffin footers are supported. Footer payload size (and declared LZ4 content size) is bounded by a compression ratio where applicable and an absolute ceiling; oversized footers are rejected before allocation. + +Only a subset of output columns can be requested. A user-provided structure with unexpected column names or types is rejected when the format is created. + +## Example usage {#example-usage} + +Read deleted row positions with the referenced data file: + +```sql +SELECT referenced_data_file, deleted_rows +FROM file(deletes.puffin, Puffin); +``` + +Expand deleted positions into individual rows: + +```sql +SELECT referenced_data_file, row_number +FROM file(deletes.puffin, Puffin) +ARRAY JOIN deleted_rows AS row_number +ORDER BY referenced_data_file, row_number; +``` + +Use `PuffinMetadata` to inspect footer blob descriptors before reading deletion vectors. +)DOCS_MD", + .related = {"PuffinMetadata"}, + }); +} + +void registerPuffinSchemaReaders(FormatFactory & factory) +{ + factory.registerSchemaReader( + "PuffinMetadata", + [](ReadBuffer & buf, const FormatSettings &) + { return std::make_shared(buf); }); + + factory.registerSchemaReader( + "Puffin", + [](ReadBuffer & buf, const FormatSettings &) + { return std::make_shared(buf); }); +} + +} diff --git a/src/Processors/Formats/Impl/PuffinBlockInputFormat.h b/src/Processors/Formats/Impl/PuffinBlockInputFormat.h new file mode 100644 index 000000000000..fc87f950b2f8 --- /dev/null +++ b/src/Processors/Formats/Impl/PuffinBlockInputFormat.h @@ -0,0 +1,83 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ + +struct PuffinBlob +{ + String type; + Int64 snapshot_id = 0; + Int64 sequence_number = 0; + std::vector fields; + Int64 offset = 0; + Int64 length = 0; + String compression_codec; + std::map properties; +}; + +struct PuffinFooter +{ + std::vector blobs; + std::vector data; +}; + +class PuffinMetadataInputFormat : public IInputFormat +{ +public: + PuffinMetadataInputFormat(ReadBuffer & buf, SharedHeader header_, const FormatSettings & format_settings_); + + String getName() const override { return "PuffinMetadata"; } + + void resetParser() override; + +private: + Chunk read() override; + + bool seekable_read = true; + PuffinFooter footer; + bool initialized = false; + size_t blob_index = 0; +}; + +class PuffinInputFormat : public IInputFormat +{ +public: + PuffinInputFormat(ReadBuffer & buf, SharedHeader header_, const FormatSettings & format_settings_); + + String getName() const override { return "Puffin"; } + + void resetParser() override; + +private: + Chunk read() override; + + bool seekable_read = true; + bool need_deleted_rows = true; + PuffinFooter footer; + bool initialized = false; + size_t blob_index = 0; +}; + +class PuffinMetadataSchemaReader : public ISchemaReader +{ +public: + explicit PuffinMetadataSchemaReader(ReadBuffer & in_); + NamesAndTypesList readSchema() override; +}; + +class PuffinSchemaReader : public ISchemaReader +{ +public: + explicit PuffinSchemaReader(ReadBuffer & in_); + NamesAndTypesList readSchema() override; +}; + +class FormatFactory; +void registerInputFormatPuffin(FormatFactory & factory); +void registerPuffinSchemaReaders(FormatFactory & factory); + +} diff --git a/src/Processors/tests/gtest_parquet_v3_need_only_count_buckets.cpp b/src/Processors/tests/gtest_parquet_v3_need_only_count_buckets.cpp new file mode 100644 index 000000000000..7bdbea541e29 --- /dev/null +++ b/src/Processors/tests/gtest_parquet_v3_need_only_count_buckets.cpp @@ -0,0 +1,179 @@ +#include + +#include + +#if USE_PARQUET + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace DB; + +namespace DB::ErrorCodes +{ +extern const int LOGICAL_ERROR; +} + +namespace +{ + +void writeMultiRowGroupParquet(const String & path, size_t rows_per_group, size_t num_groups) +{ + FormatSettings format_settings; + format_settings.parquet.row_group_rows = rows_per_group; + format_settings.parquet.parallel_encoding = false; + + Block header; + header.insert(ColumnWithTypeAndName(std::make_shared(), "x")); + + Chunks chunks; + for (size_t group = 0; group < num_groups; ++group) + { + auto column = ColumnUInt64::create(); + for (size_t i = 0; i < rows_per_group; ++i) + column->insert(group * rows_per_group + i); + chunks.emplace_back(Columns{std::move(column)}, rows_per_group); + } + + auto source = std::make_shared(std::make_shared(header), std::move(chunks)); + QueryPipelineBuilder pipeline_builder; + pipeline_builder.init(Pipe(source)); + auto pipeline = QueryPipelineBuilder::getPipeline(std::move(pipeline_builder)); + + WriteBufferFromFile write_buffer(path); + auto output = std::make_shared(write_buffer, pipeline.getSharedHeader(), format_settings, nullptr); + pipeline.complete(output); + CompletedPipelineExecutor executor(pipeline); + executor.execute(); + output->finalize(); + write_buffer.finalize(); +} + +size_t readNeedOnlyCountTotal( + ReadBuffer & in, + const SharedHeader & header, + const FormatSettings & format_settings, + FormatParserSharedResourcesPtr parser_shared_resources, + const FileBucketInfoPtr & buckets) +{ + auto input = std::make_shared( + in, + header, + format_settings, + parser_shared_resources, + std::make_shared(), + /*min_bytes_for_seek=*/ 1024); + if (buckets) + input->setBucketsToRead(buckets); + input->needOnlyCount(); + + size_t total = 0; + while (true) + { + Chunk chunk = input->generate(); + if (!chunk) + break; + total += chunk.getNumRows(); + } + return total; +} + +std::vector> readNeedOnlyCountOffsetsAndRows( + ReadBuffer & in, + const SharedHeader & header, + const FormatSettings & format_settings, + FormatParserSharedResourcesPtr parser_shared_resources, + const FileBucketInfoPtr & buckets) +{ + auto input = std::make_shared( + in, + header, + format_settings, + parser_shared_resources, + std::make_shared(), + /*min_bytes_for_seek=*/ 1024); + if (buckets) + input->setBucketsToRead(buckets); + input->needOnlyCount(); + + std::vector> result; + while (true) + { + Chunk chunk = input->generate(); + if (!chunk) + break; + auto info = chunk.getChunkInfos().get(); + if (!info) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected ChunkInfoRowNumbers on need_only_count chunk"); + result.emplace_back(info->row_num_offset, chunk.getNumRows()); + } + return result; +} + +} + +TEST(ParquetV3NeedOnlyCountBuckets, CountsOnlyAssignedRowGroups) +{ + tryRegisterFormats(); + const auto context = getContext().context; + + const String path = "/home/iantonspb/ClickHouse/tmp/gtest_parquet_v3_need_only_count_buckets.parquet"; + std::filesystem::create_directories("/home/iantonspb/ClickHouse/tmp"); + constexpr size_t rows_per_group = 10; + constexpr size_t num_groups = 3; + writeMultiRowGroupParquet(path, rows_per_group, num_groups); + + Block header; + header.insert(ColumnWithTypeAndName(std::make_shared(), "x")); + auto shared_header = std::make_shared(header); + FormatSettings format_settings; + auto parser_shared_resources = FormatParserSharedResources::singleThreaded(context->getSettingsRef()); + + { + ReadBufferFromFile in(path); + EXPECT_EQ( + readNeedOnlyCountTotal(in, shared_header, format_settings, parser_shared_resources, /*buckets=*/ nullptr), + rows_per_group * num_groups); + } + + { + ReadBufferFromFile in(path); + auto buckets = std::make_shared(std::vector{1}); + EXPECT_EQ(readNeedOnlyCountTotal(in, shared_header, format_settings, parser_shared_resources, buckets), rows_per_group); + } + + { + ReadBufferFromFile in(path); + auto buckets = std::make_shared(std::vector{0, 2}); + const auto offsets_and_rows + = readNeedOnlyCountOffsetsAndRows(in, shared_header, format_settings, parser_shared_resources, buckets); + ASSERT_EQ(offsets_and_rows.size(), 2u); + EXPECT_EQ(offsets_and_rows[0].first, 0u); + EXPECT_EQ(offsets_and_rows[0].second, rows_per_group); + EXPECT_EQ(offsets_and_rows[1].first, 2 * rows_per_group); + EXPECT_EQ(offsets_and_rows[1].second, rows_per_group); + } + + { + ReadBufferFromFile in(path); + auto buckets = std::make_shared(std::vector{}); + EXPECT_EQ(readNeedOnlyCountTotal(in, shared_header, format_settings, parser_shared_resources, buckets), 0u); + } +} + +#endif diff --git a/src/Processors/tests/gtest_puffin_reset_parser.cpp b/src/Processors/tests/gtest_puffin_reset_parser.cpp new file mode 100644 index 000000000000..c5e9075df2be --- /dev/null +++ b/src/Processors/tests/gtest_puffin_reset_parser.cpp @@ -0,0 +1,130 @@ +#include + +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace +{ + +/// Embedded copies of fixtures from tests/queries/0_stateless/data_puffin/. +/// Unit-test CI images do not ship the queries/ tree, so filesystem paths under +/// CLICKHOUSE_TESTS_CONFIG_DIR are unavailable at runtime. +constexpr UInt8 dense_range_100k_puffin[] = { + 0x50, 0x46, 0x41, 0x31, 0x00, 0x00, 0x00, 0x29, 0xD1, 0xD3, 0x39, 0x64, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3B, 0x30, 0x01, 0x00, 0x03, 0x00, 0x00, 0xFF, + 0xFF, 0x01, 0x00, 0x9F, 0x86, 0x01, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x9F, + 0x86, 0x66, 0x65, 0xF8, 0x5F, 0x50, 0x46, 0x41, 0x31, 0x7B, 0x22, 0x62, 0x6C, 0x6F, 0x62, 0x73, + 0x22, 0x3A, 0x20, 0x5B, 0x7B, 0x22, 0x74, 0x79, 0x70, 0x65, 0x22, 0x3A, 0x20, 0x22, 0x64, 0x65, + 0x6C, 0x65, 0x74, 0x69, 0x6F, 0x6E, 0x2D, 0x76, 0x65, 0x63, 0x74, 0x6F, 0x72, 0x2D, 0x76, 0x31, + 0x22, 0x2C, 0x20, 0x22, 0x66, 0x69, 0x65, 0x6C, 0x64, 0x73, 0x22, 0x3A, 0x20, 0x5B, 0x5D, 0x2C, + 0x20, 0x22, 0x73, 0x6E, 0x61, 0x70, 0x73, 0x68, 0x6F, 0x74, 0x2D, 0x69, 0x64, 0x22, 0x3A, 0x20, + 0x2D, 0x31, 0x2C, 0x20, 0x22, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6E, 0x63, 0x65, 0x2D, 0x6E, 0x75, + 0x6D, 0x62, 0x65, 0x72, 0x22, 0x3A, 0x20, 0x2D, 0x31, 0x2C, 0x20, 0x22, 0x6F, 0x66, 0x66, 0x73, + 0x65, 0x74, 0x22, 0x3A, 0x20, 0x34, 0x2C, 0x20, 0x22, 0x6C, 0x65, 0x6E, 0x67, 0x74, 0x68, 0x22, + 0x3A, 0x20, 0x34, 0x39, 0x2C, 0x20, 0x22, 0x70, 0x72, 0x6F, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, + 0x73, 0x22, 0x3A, 0x20, 0x7B, 0x22, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6E, 0x63, 0x65, 0x64, + 0x2D, 0x64, 0x61, 0x74, 0x61, 0x2D, 0x66, 0x69, 0x6C, 0x65, 0x22, 0x3A, 0x20, 0x22, 0x2F, 0x64, + 0x61, 0x74, 0x61, 0x2F, 0x74, 0x61, 0x62, 0x6C, 0x65, 0x2F, 0x70, 0x61, 0x72, 0x74, 0x2D, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x2E, 0x70, 0x61, 0x72, 0x71, 0x75, 0x65, 0x74, 0x22, 0x2C, 0x20, 0x22, + 0x63, 0x61, 0x72, 0x64, 0x69, 0x6E, 0x61, 0x6C, 0x69, 0x74, 0x79, 0x22, 0x3A, 0x20, 0x22, 0x31, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x22, 0x7D, 0x7D, 0x5D, 0x7D, 0xE1, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x50, 0x46, 0x41, 0x31, +}; + +constexpr UInt8 mixed_blob_types_puffin[] = { + 0x50, 0x46, 0x41, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0xD1, 0xD3, 0x39, 0x64, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3A, 0x30, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x05, 0x00, 0x2C, 0xDB, 0x9F, 0xC1, + 0x50, 0x46, 0x41, 0x31, 0x7B, 0x22, 0x62, 0x6C, 0x6F, 0x62, 0x73, 0x22, 0x3A, 0x20, 0x5B, 0x7B, + 0x22, 0x74, 0x79, 0x70, 0x65, 0x22, 0x3A, 0x20, 0x22, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2D, + 0x64, 0x61, 0x74, 0x61, 0x73, 0x6B, 0x65, 0x74, 0x63, 0x68, 0x65, 0x73, 0x2D, 0x74, 0x68, 0x65, + 0x74, 0x61, 0x2D, 0x76, 0x31, 0x22, 0x2C, 0x20, 0x22, 0x66, 0x69, 0x65, 0x6C, 0x64, 0x73, 0x22, + 0x3A, 0x20, 0x5B, 0x31, 0x5D, 0x2C, 0x20, 0x22, 0x73, 0x6E, 0x61, 0x70, 0x73, 0x68, 0x6F, 0x74, + 0x2D, 0x69, 0x64, 0x22, 0x3A, 0x20, 0x2D, 0x31, 0x2C, 0x20, 0x22, 0x73, 0x65, 0x71, 0x75, 0x65, + 0x6E, 0x63, 0x65, 0x2D, 0x6E, 0x75, 0x6D, 0x62, 0x65, 0x72, 0x22, 0x3A, 0x20, 0x2D, 0x31, 0x2C, + 0x20, 0x22, 0x6F, 0x66, 0x66, 0x73, 0x65, 0x74, 0x22, 0x3A, 0x20, 0x34, 0x2C, 0x20, 0x22, 0x6C, + 0x65, 0x6E, 0x67, 0x74, 0x68, 0x22, 0x3A, 0x20, 0x31, 0x36, 0x2C, 0x20, 0x22, 0x70, 0x72, 0x6F, + 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x22, 0x3A, 0x20, 0x7B, 0x7D, 0x7D, 0x2C, 0x20, 0x7B, + 0x22, 0x74, 0x79, 0x70, 0x65, 0x22, 0x3A, 0x20, 0x22, 0x64, 0x65, 0x6C, 0x65, 0x74, 0x69, 0x6F, + 0x6E, 0x2D, 0x76, 0x65, 0x63, 0x74, 0x6F, 0x72, 0x2D, 0x76, 0x31, 0x22, 0x2C, 0x20, 0x22, 0x66, + 0x69, 0x65, 0x6C, 0x64, 0x73, 0x22, 0x3A, 0x20, 0x5B, 0x5D, 0x2C, 0x20, 0x22, 0x73, 0x6E, 0x61, + 0x70, 0x73, 0x68, 0x6F, 0x74, 0x2D, 0x69, 0x64, 0x22, 0x3A, 0x20, 0x2D, 0x31, 0x2C, 0x20, 0x22, + 0x73, 0x65, 0x71, 0x75, 0x65, 0x6E, 0x63, 0x65, 0x2D, 0x6E, 0x75, 0x6D, 0x62, 0x65, 0x72, 0x22, + 0x3A, 0x20, 0x2D, 0x31, 0x2C, 0x20, 0x22, 0x6F, 0x66, 0x66, 0x73, 0x65, 0x74, 0x22, 0x3A, 0x20, + 0x32, 0x30, 0x2C, 0x20, 0x22, 0x6C, 0x65, 0x6E, 0x67, 0x74, 0x68, 0x22, 0x3A, 0x20, 0x34, 0x34, + 0x2C, 0x20, 0x22, 0x70, 0x72, 0x6F, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x22, 0x3A, 0x20, + 0x7B, 0x22, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6E, 0x63, 0x65, 0x64, 0x2D, 0x64, 0x61, 0x74, + 0x61, 0x2D, 0x66, 0x69, 0x6C, 0x65, 0x22, 0x3A, 0x20, 0x22, 0x2F, 0x64, 0x61, 0x74, 0x61, 0x2F, + 0x74, 0x61, 0x62, 0x6C, 0x65, 0x2F, 0x70, 0x61, 0x72, 0x74, 0x2D, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x2E, 0x70, 0x61, 0x72, 0x71, 0x75, 0x65, 0x74, 0x22, 0x2C, 0x20, 0x22, 0x63, 0x61, 0x72, 0x64, + 0x69, 0x6E, 0x61, 0x6C, 0x69, 0x74, 0x79, 0x22, 0x3A, 0x20, 0x22, 0x32, 0x22, 0x7D, 0x7D, 0x5D, + 0x7D, 0x6D, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x46, 0x41, 0x31, +}; + +Block makeSampleBlock(const String & format_name) +{ + EmptyReadBuffer schema_buf; + NamesAndTypesList schema; + if (format_name == "Puffin") + { + PuffinSchemaReader schema_reader(schema_buf); + schema = schema_reader.readSchema(); + } + else + { + PuffinMetadataSchemaReader schema_reader(schema_buf); + schema = schema_reader.readSchema(); + } + + ColumnsWithTypeAndName columns; + for (const auto & [name, type] : schema) + columns.emplace_back(type->createColumn(), type, name); + return Block(std::move(columns)); +} + +size_t executePuffinFormat(const String & format_name, const UInt8 * data, size_t size) +{ + const auto context = getContext().context; + const Block sample = makeSampleBlock(format_name); + + EmptyReadBuffer empty_buffer; + auto format = context->getInputFormat(format_name, empty_buffer, sample, 8192); + StreamingFormatExecutor executor(sample, format); + + ReadBufferFromMemory first_buffer(reinterpret_cast(data), size); + const size_t first_rows = executor.execute(first_buffer); + EXPECT_GT(first_rows, 0u); + + ReadBufferFromMemory second_buffer(reinterpret_cast(data), size); + const size_t second_rows = executor.execute(second_buffer); + EXPECT_EQ(second_rows, first_rows); + + return first_rows; +} + +} + +TEST(PuffinResetParser, PuffinReusedAcrossBuffers) +{ + tryRegisterFormats(); + EXPECT_EQ( + executePuffinFormat("Puffin", dense_range_100k_puffin, sizeof(dense_range_100k_puffin)), + 1u); +} + +TEST(PuffinResetParser, PuffinMetadataReusedAcrossBuffers) +{ + tryRegisterFormats(); + EXPECT_EQ( + executePuffinFormat("PuffinMetadata", mixed_blob_types_puffin, sizeof(mixed_blob_types_puffin)), + 2u); +} + +} diff --git a/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp b/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp index 4517834d0dec..1d5dcfff1f1d 100644 --- a/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp @@ -16,6 +16,7 @@ #include #include #include +#include namespace DB::ErrorCodes { @@ -242,6 +243,22 @@ ParsedManifestFileEntryPtr AvroForIcebergDeserializer::createParsedManifestFileE const auto record_count = getValueFromRowByName(row_index, c_data_file_record_count, TypeIndex::Int64).safeGet(); const auto file_size_in_bytes = getValueFromRowByName(row_index, c_data_file_file_size_in_bytes, TypeIndex::Int64).safeGet(); + std::optional content_offset; + if (hasPath(c_data_file_content_offset)) + { + const auto content_offset_value = getValueFromRowByName(row_index, c_data_file_content_offset); + if (!content_offset_value.isNull()) + content_offset = content_offset_value.safeGet(); + } + + std::optional content_size_in_bytes; + if (hasPath(c_data_file_content_size_in_bytes)) + { + const auto content_size_value = getValueFromRowByName(row_index, c_data_file_content_size_in_bytes); + if (!content_size_value.isNull()) + content_size_in_bytes = content_size_value.safeGet(); + } + switch (content_type) { case FileContentType::DATA: { @@ -292,6 +309,25 @@ ParsedManifestFileEntryPtr AvroForIcebergDeserializer::createParsedManifestFileE upper_reference_data_file_path.emplace(Iceberg::IcebergPathFromMetadata::deserialize(upper.safeGet())); } } + + if (Poco::toLower(file_format) == "puffin") + { + if (!content_offset.has_value() || !content_size_in_bytes.has_value()) + { + throw Exception( + DB::ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Puffin deletion vector entry in manifest file '{}' is missing content_offset or content_size_in_bytes", + manifest_file_path); + } + if (!lower_reference_data_file_path.has_value()) + { + throw Exception( + DB::ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Puffin deletion vector entry in manifest file '{}' is missing referenced_data_file", + manifest_file_path); + } + } + return std::make_shared( FileContentType::POSITION_DELETE, file_path_key, @@ -308,7 +344,9 @@ ParsedManifestFileEntryPtr AvroForIcebergDeserializer::createParsedManifestFileE /*equality_ids*/ std::nullopt, /*sort_order_id = */ std::nullopt, record_count, - file_size_in_bytes); + file_size_in_bytes, + content_offset, + content_size_in_bytes); } case FileContentType::EQUALITY_DELETE: { std::vector equality_ids; diff --git a/src/Storages/ObjectStorage/DataLakes/DataLakeObjectMetadata.h b/src/Storages/ObjectStorage/DataLakes/DataLakeObjectMetadata.h index fcc2c0495e18..566b1965e0df 100644 --- a/src/Storages/ObjectStorage/DataLakes/DataLakeObjectMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/DataLakeObjectMetadata.h @@ -1,6 +1,8 @@ #pragma once #include +#include + namespace DB { template @@ -17,4 +19,11 @@ struct DataLakeObjectMetadata ExcludedRowsPtr excluded_rows; }; +/// True when a deletion vector (or similar) will filter rows via DeletionVectorTransform. +/// Count-from-files cache must skip these objects: the cache key is data-file identity only, +/// while excluded_rows can change independently (Iceberg puffin DVs, DeltaLake selection vectors). +/// Cluster task serialization must also fail closed when these cannot be carried on the wire. +bool hasNonEmptyExcludedRows(const DataLakeObjectMetadata & metadata); +bool hasNonEmptyExcludedRows(const std::optional & metadata); + } diff --git a/src/Storages/ObjectStorage/DataLakes/DeletionVectorTransform.cpp b/src/Storages/ObjectStorage/DataLakes/DeletionVectorTransform.cpp index 3fd353ef99a1..9b21a7bb8b0b 100644 --- a/src/Storages/ObjectStorage/DataLakes/DeletionVectorTransform.cpp +++ b/src/Storages/ObjectStorage/DataLakes/DeletionVectorTransform.cpp @@ -13,6 +13,16 @@ namespace DB::ErrorCodes namespace DB { +bool hasNonEmptyExcludedRows(const DataLakeObjectMetadata & metadata) +{ + return metadata.excluded_rows && metadata.excluded_rows->size() > 0; +} + +bool hasNonEmptyExcludedRows(const std::optional & metadata) +{ + return metadata.has_value() && hasNonEmptyExcludedRows(*metadata); +} + DeletionVectorTransform::DeletionVectorTransform( const DB::SharedHeader & header_, ExcludedRowsPtr excluded_rows_) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Constant.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/Constant.h index 6495b22f7cf0..0bb9b586326b 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Constant.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Constant.h @@ -175,6 +175,8 @@ DEFINE_ICEBERG_FIELD_COMPOUND(data_file, null_value_counts); DEFINE_ICEBERG_FIELD_COMPOUND(data_file, lower_bounds); DEFINE_ICEBERG_FIELD_COMPOUND(data_file, upper_bounds); DEFINE_ICEBERG_FIELD_COMPOUND(data_file, referenced_data_file); +DEFINE_ICEBERG_FIELD_COMPOUND(data_file, content_offset); +DEFINE_ICEBERG_FIELD_COMPOUND(data_file, content_size_in_bytes); DEFINE_ICEBERG_FIELD_COMPOUND(data_file, sort_order_id); DEFINE_ICEBERG_FIELD_COMPOUND(data_file, record_count); DEFINE_ICEBERG_FIELD_COMPOUND(data_file, file_size_in_bytes); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.cpp index 4e161ca8c863..6e99248bcba4 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.cpp @@ -15,6 +15,7 @@ #include #include +#include namespace DB::ErrorCodes { @@ -25,6 +26,23 @@ extern const int UNKNOWN_PROTOCOL; using namespace DB::Iceberg; +namespace DB::Iceberg +{ + +void requireParquetDataFileForRowDeletes(const String & file_format, std::string_view feature_name) +{ + if (Poco::toUpper(file_format) != "PARQUET") + { + throw Exception( + DB::ErrorCodes::NOT_IMPLEMENTED, + "{} are only supported for data files of Parquet format in Iceberg, but got {}", + feature_name, + file_format); + } +} + +} + namespace DB { @@ -97,13 +115,7 @@ std::shared_ptr IcebergDataObjectInfo::getPositionDeleteTransf void IcebergDataObjectInfo::addPositionDeleteObject(Iceberg::ProcessedManifestFileEntryPtr position_delete_object, const String & resolved_storage_path) { - if (Poco::toUpper(info.file_format) != "PARQUET") - { - throw Exception( - ErrorCodes::NOT_IMPLEMENTED, - "Position deletes are only supported for data files of Parquet format in Iceberg, but got {}", - info.file_format); - } + Iceberg::requireParquetDataFileForRowDeletes(info.file_format, "Position deletes"); info.position_deletes_objects.emplace_back( resolved_storage_path, position_delete_object->parsed_entry->file_format, std::nullopt, position_delete_object->sequence_number); @@ -118,6 +130,26 @@ void IcebergDataObjectInfo::addEqualityDeleteObject(const Iceberg::ProcessedMani equality_delete_object->resolved_schema_id); } +ObjectInfoPtr IcebergDataObjectInfo::clone() const +{ + auto result = std::make_shared(relative_path_with_metadata, info); + result->data_lake_metadata = data_lake_metadata; + result->file_bucket_info = file_bucket_info; + return result; +} + +bool hasIcebergEqualityDeletes(const ObjectInfoPtr & object_info) +{ + const auto * iceberg = dynamic_cast(object_info.get()); + return iceberg && !iceberg->info.equality_deletes_objects.empty(); +} + +bool hasIcebergPositionDeletes(const ObjectInfoPtr & object_info) +{ + const auto * iceberg = dynamic_cast(object_info.get()); + return iceberg && !iceberg->info.position_deletes_objects.empty(); +} + #endif void IcebergObjectSerializableInfo::serializeForClusterFunctionProtocol(WriteBuffer & out, size_t protocol_version) const diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.h index 6763d1013201..9e646eedd52c 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.h @@ -11,12 +11,18 @@ #include #include +#include + namespace DB::Iceberg { String computePartitionId(const Row & partition_key_value); +/// Position deletes and deletion vectors need file-relative row numbers (`ChunkInfoRowNumbers`), +/// which Parquet readers emit. Reject other data-file formats early with NOT_IMPLEMENTED. +void requireParquetDataFileForRowDeletes(const String & file_format, std::string_view feature_name); + struct IcebergObjectSerializableInfo { @@ -84,10 +90,18 @@ struct IcebergDataObjectInfo : public ObjectInfo, std::enable_shared_from_this; + +/// Equality deletes need real column values — disable need_only_count / count-from-files cache. +bool hasIcebergEqualityDeletes(const ObjectInfoPtr & object_info); +/// Position-delete attachment changes independently of data-file identity — skip count-from-files cache. +bool hasIcebergPositionDeletes(const ObjectInfoPtr & object_info); } #endif diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.cpp new file mode 100644 index 000000000000..100e399a682e --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.cpp @@ -0,0 +1,161 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ICEBERG_SPECIFICATION_VIOLATION; +} + +namespace Setting +{ +extern const SettingsBool use_puffin_files_cache; +} + +} + +namespace DB::Iceberg +{ + +namespace +{ + +DataLakeObjectMetadata::ExcludedRowsPtr loadDeletionVectorUncached( + ObjectStoragePtr object_storage, + const String & puffin_path, + Int64 content_offset, + Int64 content_size_in_bytes, + const IcebergPathFromMetadata & expected_data_file, + UInt64 expected_cardinality, + ContextPtr context, + LoggerPtr log, + bool disable_filesystem_cache) +{ + RelativePathWithMetadata puffin_object{puffin_path}; + auto read_settings = context->getReadSettings(); + if (disable_filesystem_cache) + read_settings.enable_filesystem_cache = false; + + auto read_buffer = createReadBuffer(puffin_object, object_storage, context, log, read_settings); + auto deleted_positions = readDeletionVectorFromPuffin( + *read_buffer, content_offset, content_size_in_bytes, expected_cardinality); + + if (deleted_positions.empty()) + return nullptr; + + auto bitmap = std::make_shared(); + for (UInt64 position : deleted_positions) + bitmap->add(static_cast(position)); + + LOG_DEBUG( + log, + "Loaded deletion vector from puffin file '{}' for data file '{}': {} deleted rows", + puffin_path, + expected_data_file.serialize(), + deleted_positions.size()); + + return bitmap; +} + +} + +DataLakeObjectMetadata::ExcludedRowsPtr loadDeletionVector( + ObjectStoragePtr object_storage, + const String & puffin_path, + Int64 content_offset, + Int64 content_size_in_bytes, + const IcebergPathFromMetadata & expected_data_file, + const std::optional & referenced_data_file, + Int64 expected_cardinality, + ContextPtr context, + LoggerPtr log) +{ + if (referenced_data_file.has_value() && referenced_data_file.value() != expected_data_file) + { + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Deletion vector referenced_data_file '{}' does not match data file '{}'", + referenced_data_file->serialize(), + expected_data_file.serialize()); + } + + if (expected_cardinality < 0) + { + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Deletion vector record_count {} must be non-negative", + expected_cardinality); + } + + const UInt64 expected_cardinality_u64 = static_cast(expected_cardinality); + + const bool use_cache = context->getSettingsRef()[Setting::use_puffin_files_cache]; + if (!use_cache) + { + LOG_TRACE(log, "Not using Puffin files cache for '{}', because the setting use_puffin_files_cache is false", puffin_path); + return loadDeletionVectorUncached( + object_storage, + puffin_path, + content_offset, + content_size_in_bytes, + expected_data_file, + expected_cardinality_u64, + context, + log, + false); + } + + RelativePathWithMetadata puffin_object{puffin_path}; + if (!puffin_object.metadata) + puffin_object.metadata = object_storage->getObjectMetadata(puffin_object.getPath(), /*with_tags=*/ false); + + auto cache_key = PuffinFilesCache::tryCreateKey( + puffin_path, + puffin_object.metadata->etag, + content_offset, + content_size_in_bytes, + expected_data_file.serialize()); + + if (!cache_key) + { + LOG_TRACE(log, "Not using Puffin files cache for '{}', because etag is empty", puffin_path); + return loadDeletionVectorUncached( + object_storage, + puffin_path, + content_offset, + content_size_in_bytes, + expected_data_file, + expected_cardinality_u64, + context, + log, + false); + } + + auto cache = context->getPuffinFilesCache(); + return cache->getOrSetDeletionVector(*cache_key, [&]() + { + return loadDeletionVectorUncached( + object_storage, + puffin_path, + content_offset, + content_size_in_bytes, + expected_data_file, + expected_cardinality_u64, + context, + log, + true); + }); +} + +} diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.h new file mode 100644 index 000000000000..59ec8005aed2 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.h @@ -0,0 +1,28 @@ +#pragma once +#include "config.h" + +#if USE_AVRO + +#include +#include +#include +#include +#include + +namespace DB::Iceberg +{ + +DataLakeObjectMetadata::ExcludedRowsPtr loadDeletionVector( + ObjectStoragePtr object_storage, + const String & puffin_path, + Int64 content_offset, + Int64 content_size_in_bytes, + const IcebergPathFromMetadata & expected_data_file, + const std::optional & referenced_data_file, + Int64 expected_cardinality, + ContextPtr context, + LoggerPtr log); + +} + +#endif diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp index 39cc26a5fd6a..b624c2216aae 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp @@ -43,6 +43,7 @@ #include #include #include +#include #include #include @@ -71,6 +72,7 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; +extern const int ICEBERG_SPECIFICATION_VIOLATION; } namespace Setting { @@ -270,6 +272,7 @@ IcebergIterator::IcebergIterator( persistent_components_) , blocking_queue(100) , callback(std::move(callback_)) + , local_context(local_context_) { auto delete_file = deletes_iterator.next(); while (delete_file.has_value()) @@ -278,15 +281,25 @@ IcebergIterator::IcebergIterator( { equality_deletes_files.emplace_back(std::move(delete_file.value())); } + else if (delete_file.value()->parsed_entry->isDeletionVector()) + { + deletion_vector_files.emplace_back(std::move(delete_file.value())); + } else { - position_deletes_files.emplace_back(std::move(delete_file.value())); + parquet_position_deletes_files.emplace_back(std::move(delete_file.value())); } delete_file = deletes_iterator.next(); } - LOG_DEBUG(logger, "Taken {} position deletes file and {} equality deletes files in iceberg iterator", position_deletes_files.size(), equality_deletes_files.size()); + LOG_DEBUG( + logger, + "Taken {} deletion vector files, {} parquet position delete files and {} equality delete files in iceberg iterator", + deletion_vector_files.size(), + parquet_position_deletes_files.size(), + equality_deletes_files.size()); std::sort(equality_deletes_files.begin(), equality_deletes_files.end()); - std::sort(position_deletes_files.begin(), position_deletes_files.end()); + std::sort(deletion_vector_files.begin(), deletion_vector_files.end()); + std::sort(parquet_position_deletes_files.begin(), parquet_position_deletes_files.end()); producer_task = std::make_unique( [this, thread_group = CurrentThread::getGroup()]() { @@ -333,42 +346,88 @@ ObjectInfoPtr IcebergIterator::next(size_t) manifest_file_entry, persistent_components.path_resolver.resolve(manifest_file_entry->parsed_entry->file_path_key), table_state_snapshot->schema_id); - for (const auto & position_delete : - defineDeletesSpan(manifest_file_entry, position_deletes_files, /* is_equality_delete */ false, logger)) + + const auto & data_file_path = object_info->info.data_object_file_path_key; + bool has_deletion_vector = false; + + for (const auto & deletion_vector : + defineDeletesSpan(manifest_file_entry, deletion_vector_files, /* is_equality_delete */ false, logger)) { - const auto & data_file_path = object_info->info.data_object_file_path_key; - const auto & lower = position_delete->parsed_entry->lower_reference_data_file_path; - const auto & upper = position_delete->parsed_entry->upper_reference_data_file_path; - bool can_contain_data_file_deletes - = (!lower.has_value() || *lower <= data_file_path) - && (!upper.has_value() || *upper >= data_file_path); - /// Skip position deletes that do not match the data file path. - if (!can_contain_data_file_deletes) + const auto & referenced_data_file = deletion_vector->parsed_entry->lower_reference_data_file_path; + if (!referenced_data_file.has_value() || referenced_data_file.value() != data_file_path) + continue; + + if (has_deletion_vector) { - ProfileEvents::increment(ProfileEvents::IcebergMinMaxPrunedDeleteFiles); - LOG_TEST( - logger, - "Skipping position delete file `{}` for data file `{}` because position delete has out of bounds reference data file " - "bounds: " - "(lower bound: `{}`, upper bound: `{}`)", - position_delete->parsed_entry->file_path_key, - data_file_path, - lower.has_value() ? lower->serialize() : "[no lower bound]", - upper.has_value() ? upper->serialize() : "[no upper bound]"); + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Multiple deletion vectors match data file '{}'", + data_file_path); } - else + + Iceberg::requireParquetDataFileForRowDeletes(object_info->info.file_format, "Deletion vectors"); + + const auto & parsed_entry = deletion_vector->parsed_entry; + auto excluded_rows = Iceberg::loadDeletionVector( + object_storage, + persistent_components.path_resolver.resolve(parsed_entry->file_path_key), + parsed_entry->content_offset.value(), + parsed_entry->content_size_in_bytes.value(), + data_file_path, + referenced_data_file, + parsed_entry->record_count, + local_context, + logger); + + object_info->data_lake_metadata.emplace(); + if (excluded_rows) + object_info->data_lake_metadata->excluded_rows = std::move(excluded_rows); + has_deletion_vector = true; + LOG_DEBUG( + logger, + "Attached deletion vector from puffin file `{}` to data file `{}`", + parsed_entry->file_path_key, + data_file_path); + } + + if (!has_deletion_vector) + { + for (const auto & position_delete : + defineDeletesSpan(manifest_file_entry, parquet_position_deletes_files, /* is_equality_delete */ false, logger)) { - ProfileEvents::increment(ProfileEvents::IcebergMinMaxNonPrunedDeleteFiles); - LOG_TEST( - logger, - "Processing position delete file `{}` for data file `{}` with reference data file bounds: " - "(lower bound: `{}`, upper bound: `{}`)", - position_delete->parsed_entry->file_path_key, - data_file_path, - lower.has_value() ? lower->serialize() : "[no lower bound]", - upper.has_value() ? upper->serialize() : "[no upper bound]"); - object_info->addPositionDeleteObject( - position_delete, persistent_components.path_resolver.resolve(position_delete->parsed_entry->file_path_key)); + const auto & lower = position_delete->parsed_entry->lower_reference_data_file_path; + const auto & upper = position_delete->parsed_entry->upper_reference_data_file_path; + bool can_contain_data_file_deletes + = (!lower.has_value() || *lower <= data_file_path) + && (!upper.has_value() || *upper >= data_file_path); + /// Skip position deletes that do not match the data file path. + if (!can_contain_data_file_deletes) + { + ProfileEvents::increment(ProfileEvents::IcebergMinMaxPrunedDeleteFiles); + LOG_TEST( + logger, + "Skipping position delete file `{}` for data file `{}` because position delete has out of bounds reference data file " + "bounds: " + "(lower bound: `{}`, upper bound: `{}`)", + position_delete->parsed_entry->file_path_key, + data_file_path, + lower.has_value() ? lower->serialize() : "[no lower bound]", + upper.has_value() ? upper->serialize() : "[no upper bound]"); + } + else + { + ProfileEvents::increment(ProfileEvents::IcebergMinMaxNonPrunedDeleteFiles); + LOG_TEST( + logger, + "Processing position delete file `{}` for data file `{}` with reference data file bounds: " + "(lower bound: `{}`, upper bound: `{}`)", + position_delete->parsed_entry->file_path_key, + data_file_path, + lower.has_value() ? lower->serialize() : "[no lower bound]", + upper.has_value() ? upper->serialize() : "[no upper bound]"); + object_info->addPositionDeleteObject( + position_delete, persistent_components.path_resolver.resolve(position_delete->parsed_entry->file_path_key)); + } } } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h index 09c21b8ec292..973c40440470 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h @@ -93,8 +93,10 @@ class IcebergIterator : public IObjectIterator ConcurrentBoundedQueue blocking_queue; std::unique_ptr producer_task; IDataLakeMetadata::FileProgressCallback callback; - std::vector position_deletes_files; + std::vector deletion_vector_files; + std::vector parquet_position_deletes_files; std::vector equality_deletes_files; + ContextPtr local_context; std::exception_ptr exception; std::mutex exception_mutex; }; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index fd46034ee62a..3817e3f1e7d8 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -392,6 +392,7 @@ IcebergDataSnapshotPtr IcebergMetadata::createIcebergDataSnapshotFromSnapshotJSO std::optional total_rows; std::optional total_bytes; std::optional total_position_deletes; + std::optional total_equality_deletes; if (snapshot_object->has(f_summary)) { @@ -406,6 +407,9 @@ IcebergDataSnapshotPtr IcebergMetadata::createIcebergDataSnapshotFromSnapshotJSO { total_position_deletes = summary_object->getValue(f_total_position_deletes); } + + if (summary_object->has(f_total_equality_deletes)) + total_equality_deletes = summary_object->getValue(f_total_equality_deletes); } if (!snapshot_object->has(f_schema_id)) @@ -419,7 +423,8 @@ IcebergDataSnapshotPtr IcebergMetadata::createIcebergDataSnapshotFromSnapshotJSO schema_id, total_rows, total_bytes, - total_position_deletes); + total_position_deletes, + total_equality_deletes); } IcebergDataSnapshotPtr @@ -1047,13 +1052,22 @@ std::optional IcebergMetadata::totalRows(ContextPtr local_context) const return 0; } + /// Equality deletes remove data rows by value match; summary `total-equality-deletes` counts + /// rows in delete files, not deleted data rows. Fail closed when the field is present and > 0. + /// If the field is absent, skip the summary shortcut and scan manifests for EQUALITY_DELETE files. + if (actual_data_snapshot->total_equality_delete_rows.has_value() + && *actual_data_snapshot->total_equality_delete_rows > 0) + return {}; /// All these "hints" with total rows or bytes are optional both in /// metadata files and in manifest files, so we try all of them one by one - if (auto total_rows = actual_data_snapshot->getTotalRows(); total_rows.has_value()) + if (actual_data_snapshot->allowsSnapshotTotalRowsShortcut()) { - ProfileEvents::increment(ProfileEvents::IcebergTrivialCountOptimizationApplied); - return total_rows; + if (auto total_rows = actual_data_snapshot->getTotalRows(); total_rows.has_value()) + { + ProfileEvents::increment(ProfileEvents::IcebergTrivialCountOptimizationApplied); + return total_rows; + } } Int64 result = 0; @@ -1061,6 +1075,9 @@ std::optional IcebergMetadata::totalRows(ContextPtr local_context) const { auto manifest_file_ptr = getManifestFileEntriesHandle( object_storage, persistent_components, local_context, log, manifest_list_entry, actual_table_state_snapshot.schema_id); + if (!manifest_file_ptr.getFilesWithoutDeleted(FileContentType::EQUALITY_DELETE).empty()) + return {}; + auto data_count = manifest_file_ptr.getRowsCountInAllFilesExcludingDeleted(FileContentType::DATA); auto position_deletes_count = manifest_file_ptr.getRowsCountInAllFilesExcludingDeleted(FileContentType::POSITION_DELETE); if (!data_count.has_value() || !position_deletes_count.has_value()) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h index f7c2ced00bf3..9e7a71f3ea64 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h @@ -15,6 +15,7 @@ #include #include +#include namespace DB::Iceberg { @@ -94,6 +95,17 @@ struct ParsedManifestFileEntry : boost::noncopyable Int64 record_count; Int64 file_size_in_bytes; + /// Iceberg v3 deletion vector metadata (position delete entries with puffin format) + std::optional content_offset; + std::optional content_size_in_bytes; + + bool isDeletionVector() const + { + return Poco::toLower(file_format) == "puffin" + && content_offset.has_value() + && content_size_in_bytes.has_value(); + } + ParsedManifestFileEntry( FileContentType content_type_, IcebergPathFromMetadata file_path_key_, @@ -110,7 +122,9 @@ struct ParsedManifestFileEntry : boost::noncopyable std::optional> equality_ids_, std::optional sort_order_id_, Int64 record_count_, - Int64 file_size_in_bytes_) + Int64 file_size_in_bytes_, + std::optional content_offset_ = std::nullopt, + std::optional content_size_in_bytes_ = std::nullopt) : content_type(content_type_) , file_path_key(std::move(file_path_key_)) , row_number(row_number_) @@ -127,6 +141,8 @@ struct ParsedManifestFileEntry : boost::noncopyable , sort_order_id(sort_order_id_) , record_count(record_count_) , file_size_in_bytes(file_size_in_bytes_) + , content_offset(content_offset_) + , content_size_in_bytes(content_size_in_bytes_) { } }; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h index 123a67ddc40b..a08e5dec87a7 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h @@ -19,6 +19,9 @@ struct IcebergDataSnapshot std::optional total_rows; std::optional total_bytes; std::optional total_position_delete_rows; + /// Rows in equality-delete files (snapshot summary). Not a count of deleted data rows; + /// used only to fail closed trivial COUNT when equality deletes are present. + std::optional total_equality_delete_rows; std::optional getTotalRows() const { @@ -26,6 +29,13 @@ struct IcebergDataSnapshot return *total_rows - *total_position_delete_rows; return std::nullopt; } + + /// Summary `total-equality-deletes` is optional. Only trust the cheap getTotalRows() shortcut + /// when the field is present and explicitly zero; absent or >0 must fall through / fail closed. + bool allowsSnapshotTotalRowsShortcut() const + { + return total_equality_delete_rows.has_value() && *total_equality_delete_rows == 0; + } }; using IcebergDataSnapshotPtr = std::shared_ptr; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_count_shortcuts.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_count_shortcuts.cpp new file mode 100644 index 000000000000..5fe1ee441d9a --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_count_shortcuts.cpp @@ -0,0 +1,66 @@ +#include + +#include "config.h" + +#if USE_AVRO + +#include +#include +#include + +using namespace DB; + +TEST(IcebergCountShortcuts, HasEqualityAndPositionDeleteHelpers) +{ + Iceberg::IcebergObjectSerializableInfo info; + info.data_object_file_path_key = Iceberg::IcebergPathFromMetadata::deserialize("s3://bucket/data/file.parquet"); + info.file_format = "PARQUET"; + + auto plain = std::make_shared(RelativePathWithMetadata{"data/file.parquet"}); + EXPECT_FALSE(hasIcebergEqualityDeletes(plain)); + EXPECT_FALSE(hasIcebergPositionDeletes(plain)); + + auto iceberg = std::make_shared(RelativePathWithMetadata{"data/file.parquet"}, info); + EXPECT_FALSE(hasIcebergEqualityDeletes(iceberg)); + EXPECT_FALSE(hasIcebergPositionDeletes(iceberg)); + + iceberg->info.equality_deletes_objects.push_back( + Iceberg::EqualityDeleteObject{ + .file_path = "s3://bucket/deletes/eq.parquet", + .file_format = "PARQUET", + .equality_ids = std::vector{1}, + .schema_id = 0, + }); + EXPECT_TRUE(hasIcebergEqualityDeletes(iceberg)); + EXPECT_FALSE(hasIcebergPositionDeletes(iceberg)); + + iceberg->info.position_deletes_objects.push_back( + Iceberg::PositionDeleteObject{ + .file_path = "s3://bucket/deletes/pos.parquet", + .file_format = "PARQUET", + .reference_data_file_path = std::nullopt, + .sequence_number = 1, + }); + EXPECT_TRUE(hasIcebergEqualityDeletes(iceberg)); + EXPECT_TRUE(hasIcebergPositionDeletes(iceberg)); +} + +TEST(IcebergCountShortcuts, SnapshotShortcutRequiresExplicitZeroEqualityDeletes) +{ + Iceberg::IcebergDataSnapshot snapshot; + snapshot.total_rows = 100; + snapshot.total_position_delete_rows = 10; + + /// Field absent: getTotalRows() would succeed, but shortcut must not be used. + ASSERT_TRUE(snapshot.getTotalRows().has_value()); + EXPECT_FALSE(snapshot.allowsSnapshotTotalRowsShortcut()); + + snapshot.total_equality_delete_rows = 5; + EXPECT_FALSE(snapshot.allowsSnapshotTotalRowsShortcut()); + + snapshot.total_equality_delete_rows = 0; + EXPECT_TRUE(snapshot.allowsSnapshotTotalRowsShortcut()); + EXPECT_EQ(*snapshot.getTotalRows(), 90u); +} + +#endif diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_data_object_info_clone.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_data_object_info_clone.cpp new file mode 100644 index 000000000000..b863ee605446 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_data_object_info_clone.cpp @@ -0,0 +1,64 @@ +#include + +#include "config.h" + +#if USE_AVRO + +#include +#include +#include +#include + +using namespace DB; + +TEST(IcebergDataObjectInfoClone, PreservesEqualityAndPositionDeletes) +{ + Iceberg::IcebergObjectSerializableInfo info; + info.data_object_file_path_key = Iceberg::IcebergPathFromMetadata::deserialize("s3://bucket/data/file.parquet"); + info.file_format = "PARQUET"; + info.equality_deletes_objects.push_back( + Iceberg::EqualityDeleteObject{ + .file_path = "s3://bucket/deletes/eq.parquet", + .file_format = "PARQUET", + .equality_ids = std::vector{1, 2}, + .schema_id = 7, + }); + info.position_deletes_objects.push_back( + Iceberg::PositionDeleteObject{ + .file_path = "s3://bucket/deletes/pos.parquet", + .file_format = "PARQUET", + .reference_data_file_path = std::nullopt, + .sequence_number = 42, + }); + + auto original = std::make_shared( + RelativePathWithMetadata{"data/file.parquet"}, info); + original->data_lake_metadata.emplace(); + original->data_lake_metadata->excluded_rows = std::make_shared(); + original->data_lake_metadata->excluded_rows->add(11); + + /// Object-slicing (the old ObjectIteratorSplitByBuckets path) drops Iceberg metadata. + ObjectInfo sliced_value = *original; + auto sliced = std::make_shared(sliced_value); + EXPECT_FALSE(std::dynamic_pointer_cast(sliced)); + ASSERT_TRUE(sliced->data_lake_metadata.has_value()); + ASSERT_TRUE(sliced->data_lake_metadata->excluded_rows); + EXPECT_EQ(sliced->data_lake_metadata->excluded_rows->size(), 1u); + + auto cloned = original->clone(); + auto iceberg_cloned = std::dynamic_pointer_cast(cloned); + ASSERT_TRUE(iceberg_cloned); + ASSERT_EQ(iceberg_cloned->info.equality_deletes_objects.size(), 1u); + EXPECT_EQ(iceberg_cloned->info.equality_deletes_objects[0].file_path, "s3://bucket/deletes/eq.parquet"); + ASSERT_TRUE(iceberg_cloned->info.equality_deletes_objects[0].equality_ids.has_value()); + EXPECT_EQ(iceberg_cloned->info.equality_deletes_objects[0].equality_ids->size(), 2u); + ASSERT_EQ(iceberg_cloned->info.position_deletes_objects.size(), 1u); + EXPECT_EQ(iceberg_cloned->info.position_deletes_objects[0].file_path, "s3://bucket/deletes/pos.parquet"); + EXPECT_EQ(iceberg_cloned->info.position_deletes_objects[0].sequence_number, 42); + ASSERT_TRUE(iceberg_cloned->data_lake_metadata.has_value()); + ASSERT_TRUE(iceberg_cloned->data_lake_metadata->excluded_rows); + EXPECT_EQ(iceberg_cloned->data_lake_metadata->excluded_rows->size(), 1u); + EXPECT_TRUE(iceberg_cloned->data_lake_metadata->excluded_rows->rb_contains(11)); +} + +#endif diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_parquet_row_deletes_guard.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_parquet_row_deletes_guard.cpp new file mode 100644 index 000000000000..ce5262728d1b --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_parquet_row_deletes_guard.cpp @@ -0,0 +1,47 @@ +#include + +#include +#include + +using namespace DB; + +namespace DB +{ +namespace ErrorCodes +{ +extern const int NOT_IMPLEMENTED; +} +} + +TEST(IcebergParquetRowDeletesGuard, AcceptsParquet) +{ + EXPECT_NO_THROW(Iceberg::requireParquetDataFileForRowDeletes("parquet", "Deletion vectors")); + EXPECT_NO_THROW(Iceberg::requireParquetDataFileForRowDeletes("PARQUET", "Position deletes")); + EXPECT_NO_THROW(Iceberg::requireParquetDataFileForRowDeletes("Parquet", "Deletion vectors")); +} + +TEST(IcebergParquetRowDeletesGuard, RejectsNonParquet) +{ + try + { + Iceberg::requireParquetDataFileForRowDeletes("ORC", "Deletion vectors"); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::NOT_IMPLEMENTED); + EXPECT_NE(e.message().find("Deletion vectors are only supported"), std::string::npos); + EXPECT_NE(e.message().find("ORC"), std::string::npos); + } + + try + { + Iceberg::requireParquetDataFileForRowDeletes("AVRO", "Position deletes"); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::NOT_IMPLEMENTED); + EXPECT_NE(e.message().find("Position deletes are only supported"), std::string::npos); + } +} diff --git a/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.cpp b/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.cpp new file mode 100644 index 000000000000..4cf59f725ffe --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.cpp @@ -0,0 +1,291 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +namespace ProfileEvents +{ +extern const Event PuffinFilesRead; +extern const Event PuffinFileReadMicroseconds; +} + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + +namespace +{ + +struct ScopedPuffinFileReadProfileEvent +{ + ProfileEventTimeIncrement watch; + + ScopedPuffinFileReadProfileEvent() + : watch(ProfileEvents::PuffinFileReadMicroseconds) + { + ProfileEvents::increment(ProfileEvents::PuffinFilesRead); + } +}; + +constexpr UInt8 DELETION_VECTOR_MAGIC[4] = {0xD1, 0xD3, 0x39, 0x64}; +/// Matches Iceberg DeleteLoader / Puffin format absolute blob-size ceiling. +constexpr size_t PUFFIN_DV_MAX_BLOB_SIZE = 2ULL * 1024 * 1024 * 1024; +constexpr UInt64 PUFFIN_DV_MAX_MATERIALIZED_POSITIONS = 100'000'000; +constexpr Int64 DELETION_VECTOR_MAX_POSITION = 0x7FFFFFFE80000000LL; +constexpr Int32 DELETION_VECTOR_MAX_KEY = std::numeric_limits::max() - 1; + +UInt32 readBigEndianUInt32(const UInt8 * data) +{ + return (static_cast(data[0]) << 24) + | (static_cast(data[1]) << 16) + | (static_cast(data[2]) << 8) + | static_cast(data[3]); +} + +UInt64 positionFromKeyAndSubPosition(UInt32 key, UInt32 sub_position) +{ + return (static_cast(key) << 32) | static_cast(sub_position); +} + +roaring::Roaring readRoaringPortableSafe(const char * data, size_t size, Int32 key) +{ + try + { + return roaring::Roaring::readSafe(data, size); + } + catch (const std::exception & e) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Failed to deserialize deletion vector roaring bitmap at key {}: {}", key, e.what()); + } +} + +std::vector deserializeRoaringPositionBitmap(std::string_view bytes, std::optional expected_cardinality) +{ + if (bytes.size() < sizeof(Int64)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector bitmap is too small"); + + const char * ptr = bytes.data(); + size_t remaining = bytes.size(); + + /// Iceberg deletion-vector roaring layout stores count and keys as little-endian. + const Int64 bitmap_count = unalignedLoadLittleEndian(ptr); + ptr += sizeof(Int64); + remaining -= sizeof(Int64); + + if (bitmap_count < 0 || bitmap_count > std::numeric_limits::max()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector bitmap count: {}", bitmap_count); + + std::vector positions; + Int32 last_key = -1; + Int32 remaining_count = static_cast(bitmap_count); + UInt64 running_cardinality = 0; + + while (remaining_count > 0) + { + if (remaining < sizeof(Int32)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector bitmap is truncated while reading key"); + + const Int32 key = unalignedLoadLittleEndian(ptr); + ptr += sizeof(Int32); + remaining -= sizeof(Int32); + + if (key < 0 || key > DELETION_VECTOR_MAX_KEY) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector bitmap key: {}", key); + if (key <= last_key) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector bitmap keys must be sorted in ascending order"); + + auto bitmap = readRoaringPortableSafe(ptr, remaining, key); + + const size_t bitmap_size = bitmap.getSizeInBytes(/*portable=*/true); + if (bitmap_size > remaining) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector roaring bitmap at key {} exceeds blob size", key); + + if (expected_cardinality.has_value()) + { + const UInt64 bitmap_cardinality = bitmap.cardinality(); + UInt64 new_running_cardinality = 0; + if (common::addOverflow(running_cardinality, bitmap_cardinality, new_running_cardinality)) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector cardinality exceeds declared cardinality {}", + *expected_cardinality); + + if (new_running_cardinality > *expected_cardinality) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector cardinality {} exceeds declared cardinality {}", + new_running_cardinality, + *expected_cardinality); + + running_cardinality = new_running_cardinality; + } + + for (UInt32 sub_position : bitmap) + { + const UInt64 position = positionFromKeyAndSubPosition(static_cast(key), sub_position); + if (position > static_cast(DELETION_VECTOR_MAX_POSITION)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector position {} is out of supported range", position); + positions.push_back(position); + } + + ptr += bitmap_size; + remaining -= bitmap_size; + last_key = key; + --remaining_count; + } + + if (remaining != 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector bitmap has {} trailing bytes", remaining); + + if (expected_cardinality.has_value() && running_cardinality != *expected_cardinality) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector cardinality {} does not match deserialized row count {}", + *expected_cardinality, + running_cardinality); + + return positions; +} + +std::string_view extractDeletionVectorPayload(std::string_view blob) +{ + if (blob.size() < 12) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector blob is too small"); + + const auto * blob_bytes = reinterpret_cast(blob.data()); + const UInt32 combined_length = readBigEndianUInt32(blob_bytes); + if (combined_length < sizeof(DELETION_VECTOR_MAGIC)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector combined length: {}", combined_length); + + const size_t vector_size = combined_length - sizeof(DELETION_VECTOR_MAGIC); + const size_t expected_blob_size = sizeof(UInt32) + combined_length + sizeof(UInt32); + if (blob.size() != expected_blob_size) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector blob size {} does not match combined length {}", blob.size(), combined_length); + + if (std::memcmp(blob_bytes + sizeof(UInt32), DELETION_VECTOR_MAGIC, sizeof(DELETION_VECTOR_MAGIC)) != 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector magic"); + + const UInt8 * crc_input = blob_bytes + sizeof(UInt32); + const size_t crc_input_size = combined_length; + const UInt32 expected_crc = readBigEndianUInt32(blob_bytes + sizeof(UInt32) + combined_length); + const UInt32 actual_crc = static_cast(crc32_z(0L, reinterpret_cast(crc_input), crc_input_size)); + if (expected_crc != actual_crc) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector CRC mismatch"); + + return std::string_view(blob.data() + 2 * sizeof(UInt32), vector_size); +} + +} + +void validatePuffinBlobBounds(Int64 offset, Int64 length, size_t file_size, std::string_view context) +{ + if (offset < 0 || length < 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "{}: offset/length out of bounds", context); + + if (offset > static_cast(file_size) || length > static_cast(file_size)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "{}: offset/length out of bounds", context); + + Int64 end = 0; + if (common::addOverflow(offset, length, end)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "{}: offset/length out of bounds", context); + + if (static_cast(end) > file_size) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "{}: offset/length out of bounds", context); +} + +std::vector deserializeDeletionVectorV1Blob(std::string_view blob_bytes, std::optional expected_cardinality) +{ + if (expected_cardinality.has_value() && *expected_cardinality > PUFFIN_DV_MAX_MATERIALIZED_POSITIONS) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector cardinality {} exceeds materialization limit {}", + *expected_cardinality, + PUFFIN_DV_MAX_MATERIALIZED_POSITIONS); + + const std::string_view vector_bytes = extractDeletionVectorPayload(blob_bytes); + return deserializeRoaringPositionBitmap(vector_bytes, expected_cardinality); +} + +std::vector readDeletionVectorFromPuffin(ReadBuffer & file, Int64 offset, Int64 length, std::optional expected_cardinality) +{ + ScopedPuffinFileReadProfileEvent profile_event; + + if (length < 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector blob length is negative"); + + if (static_cast(length) > PUFFIN_DV_MAX_BLOB_SIZE) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector blob length {} exceeds absolute limit {}", + length, + PUFFIN_DV_MAX_BLOB_SIZE); + + /// Reject before envelope peek / full allocate: cardinality alone is enough to fail closed. + if (expected_cardinality.has_value() && *expected_cardinality > PUFFIN_DV_MAX_MATERIALIZED_POSITIONS) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector cardinality {} exceeds materialization limit {}", + *expected_cardinality, + PUFFIN_DV_MAX_MATERIALIZED_POSITIONS); + + if (auto file_size = tryGetFileSizeFromReadBuffer(file)) + validatePuffinBlobBounds(offset, length, *file_size); + else if (offset < 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid Puffin deletion vector offset {} or length {}", offset, length); + + if (static_cast(length) < 12) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector blob is too small"); + + auto * seekable = dynamic_cast(&file); + if (!seekable) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin deletion vector read requires a seekable buffer"); + + /// Peek combined_length + magic before allocating `length` (up to 2 GiB). Matches + /// `readDeletionVectorBlobBytes` in the SQL Puffin format path. + seekable->seek(offset, SEEK_SET); + + UInt8 header[8]; + file.readStrict(reinterpret_cast(header), sizeof(header)); + + const UInt32 combined_length = readBigEndianUInt32(header); + if (std::memcmp(header + sizeof(UInt32), DELETION_VECTOR_MAGIC, sizeof(DELETION_VECTOR_MAGIC)) != 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector magic"); + + if (combined_length < sizeof(DELETION_VECTOR_MAGIC)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector combined length: {}", combined_length); + + UInt64 expected_blob_size = 0; + if (common::addOverflow(static_cast(combined_length), UInt64{8}, expected_blob_size)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector combined length: {}", combined_length); + + if (static_cast(length) != expected_blob_size) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector blob size {} does not match combined length {}", + length, + combined_length); + + String blob_data(static_cast(length), '\0'); + std::memcpy(blob_data.data(), header, sizeof(header)); + file.readStrict(blob_data.data() + sizeof(header), blob_data.size() - sizeof(header)); + + return deserializeDeletionVectorV1Blob(blob_data, expected_cardinality); +} + +} diff --git a/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.h b/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.h new file mode 100644 index 000000000000..3cc1bfbd797a --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.h @@ -0,0 +1,22 @@ +#pragma once + +#include +#include + +#include +#include +#include + +namespace DB +{ + +/// Validate that [offset, offset + length) fits within file_size. +void validatePuffinBlobBounds(Int64 offset, Int64 length, size_t file_size, std::string_view context = "Puffin deletion vector"); + +/// Deserialize a deletion-vector-v1 blob (magic + CRC wrapper + roaring bitmap payload). +std::vector deserializeDeletionVectorV1Blob(std::string_view blob_bytes, std::optional expected_cardinality = std::nullopt); + +/// Read a deletion-vector-v1 blob from a Puffin file at the given offset and length. +std::vector readDeletionVectorFromPuffin(ReadBuffer & file, Int64 offset, Int64 length, std::optional expected_cardinality = std::nullopt); + +} diff --git a/src/Storages/ObjectStorage/DataLakes/PuffinFilesCache.cpp b/src/Storages/ObjectStorage/DataLakes/PuffinFilesCache.cpp new file mode 100644 index 000000000000..93bf606df368 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/PuffinFilesCache.cpp @@ -0,0 +1,110 @@ +#include + +#include +#include + +namespace CurrentMetrics +{ +extern const Metric PuffinFilesCacheBytes; +extern const Metric PuffinFilesCacheFiles; +} + +namespace ProfileEvents +{ +extern const Event PuffinFilesCacheWeightLost; +} + +namespace DB +{ + +DataLakeObjectMetadata::ExcludedRowsPtr PuffinFilesCache::cloneExcludedRows(const PuffinFilesCacheCell & cell) +{ + if (cell.is_empty_deletion_vector) + return nullptr; + + auto cloned = std::make_shared(); + cloned->merge(*cell.excluded_rows); + return cloned; +} + +bool PuffinFilesCacheKey::operator==(const PuffinFilesCacheKey & other) const +{ + return file_path == other.file_path + && etag == other.etag + && content_offset == other.content_offset + && content_size_in_bytes == other.content_size_in_bytes + && referenced_data_file == other.referenced_data_file; +} + +size_t PuffinFilesCacheKeyHash::operator()(const PuffinFilesCacheKey & key) const +{ + size_t hash = 0; + boost::hash_combine(hash, CityHash_v1_0_2::CityHash64(key.file_path.data(), key.file_path.size())); + boost::hash_combine(hash, CityHash_v1_0_2::CityHash64(key.etag.data(), key.etag.size())); + boost::hash_combine(hash, key.content_offset); + boost::hash_combine(hash, key.content_size_in_bytes); + boost::hash_combine(hash, CityHash_v1_0_2::CityHash64(key.referenced_data_file.data(), key.referenced_data_file.size())); + return hash; +} + +UInt64 PuffinFilesCacheCell::calculateMemorySize(bool is_empty_deletion_vector_, const DataLakeObjectMetadata::ExcludedRowsPtr & excluded_rows_) +{ + if (is_empty_deletion_vector_) + return EMPTY_DELETION_VECTOR_WEIGHT; + + if (!excluded_rows_) + return 0; + + return excluded_rows_->getAllocatedBytes(); +} + +PuffinFilesCacheCell::PuffinFilesCacheCell(DataLakeObjectMetadata::ExcludedRowsPtr excluded_rows_) + : excluded_rows(std::move(excluded_rows_)) + , is_empty_deletion_vector(!excluded_rows) + , memory_bytes( + calculateMemorySize(is_empty_deletion_vector, excluded_rows) + + (is_empty_deletion_vector ? 0 : SIZE_IN_MEMORY_OVERHEAD)) +{ +} + +size_t PuffinFilesCacheWeightFunction::operator()(const PuffinFilesCacheCell & cell) const +{ + return cell.memory_bytes; +} + +PuffinFilesCache::PuffinFilesCache( + const String & cache_policy, + size_t max_size_in_bytes, + size_t max_count, + double size_ratio) + : Base( + cache_policy, + CurrentMetrics::PuffinFilesCacheBytes, + CurrentMetrics::PuffinFilesCacheFiles, + max_size_in_bytes, + max_count, + size_ratio) + , log(getLogger("PuffinFilesCache")) +{ +} + +std::optional PuffinFilesCache::tryCreateKey( + const String & file_path, + const String & etag, + Int64 content_offset, + Int64 content_size_in_bytes, + const String & referenced_data_file) +{ + if (etag.empty()) + return std::nullopt; + + return PuffinFilesCacheKey{file_path, etag, content_offset, content_size_in_bytes, referenced_data_file}; +} + +void PuffinFilesCache::onEntryRemoval(const size_t weight_loss, const MappedPtr &) +{ + LOG_TRACE(log, "Puffin files cache eviction"); + ProfileEvents::increment(ProfileEvents::PuffinFilesCacheWeightLost, weight_loss); +} + +} diff --git a/src/Storages/ObjectStorage/DataLakes/PuffinFilesCache.h b/src/Storages/ObjectStorage/DataLakes/PuffinFilesCache.h new file mode 100644 index 000000000000..1dd12205994d --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/PuffinFilesCache.h @@ -0,0 +1,138 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +namespace ProfileEvents +{ +extern const Event PuffinFilesCacheHits; +extern const Event PuffinFilesCacheMisses; +extern const Event PuffinFilesCacheWeightLost; +} + +namespace DB +{ + +struct PuffinFilesCacheKey +{ + String file_path; + String etag; + Int64 content_offset = 0; + Int64 content_size_in_bytes = 0; + String referenced_data_file; + + bool operator==(const PuffinFilesCacheKey & other) const; +}; + +struct PuffinFilesCacheKeyHash +{ + size_t operator()(const PuffinFilesCacheKey & key) const; +}; + +struct PuffinFilesCacheCell : private boost::noncopyable +{ + DataLakeObjectMetadata::ExcludedRowsPtr excluded_rows; + bool is_empty_deletion_vector = false; + UInt64 memory_bytes = 0; + + explicit PuffinFilesCacheCell(DataLakeObjectMetadata::ExcludedRowsPtr excluded_rows_); + +private: + static constexpr UInt64 EMPTY_DELETION_VECTOR_WEIGHT = 1; + static constexpr size_t SIZE_IN_MEMORY_OVERHEAD = 200; + + static UInt64 calculateMemorySize(bool is_empty_deletion_vector_, const DataLakeObjectMetadata::ExcludedRowsPtr & excluded_rows_); +}; + +struct PuffinFilesCacheWeightFunction +{ + size_t operator()(const PuffinFilesCacheCell & cell) const; +}; + +/// Cache for parsed content loaded from Puffin files (deletion vectors today, indexes later). +class PuffinFilesCache : public CacheBase +{ +public: + using Base = CacheBase; + + PuffinFilesCache(const String & cache_policy, size_t max_size_in_bytes, size_t max_count, double size_ratio); + + static std::optional tryCreateKey( + const String & file_path, + const String & etag, + Int64 content_offset, + Int64 content_size_in_bytes, + const String & referenced_data_file); + + template + DataLakeObjectMetadata::ExcludedRowsPtr getOrSetDeletionVector(const PuffinFilesCacheKey & key, LoadFunc && load_fn) + { + auto load_fn_wrapper = [&]() + { + auto excluded_rows = load_fn(); + const bool is_empty_deletion_vector = !excluded_rows; + if (is_empty_deletion_vector) + { + LOG_TRACE( + log, + "Cached empty puffin deletion vector for {} | {} at offset {} length {} for data file {}", + key.file_path, + key.etag, + key.content_offset, + key.content_size_in_bytes, + key.referenced_data_file); + } + else + { + LOG_TRACE( + log, + "Loaded puffin deletion vector into cache for {} | {} at offset {} length {} for data file {}", + key.file_path, + key.etag, + key.content_offset, + key.content_size_in_bytes, + key.referenced_data_file); + } + return std::make_shared(std::move(excluded_rows)); + }; + + auto result = Base::getOrSet(key, load_fn_wrapper); + if (result.second) + { + LOG_TRACE(log, "Puffin files cache miss for {} | {} at offset {} length {} for data file {}", key.file_path, key.etag, key.content_offset, key.content_size_in_bytes, key.referenced_data_file); + ProfileEvents::increment(ProfileEvents::PuffinFilesCacheMisses); + } + else if (result.first->is_empty_deletion_vector) + { + LOG_TRACE(log, "Puffin files cache hit (empty deletion vector) for {} | {} at offset {} length {} for data file {}", key.file_path, key.etag, key.content_offset, key.content_size_in_bytes, key.referenced_data_file); + ProfileEvents::increment(ProfileEvents::PuffinFilesCacheHits); + } + else + { + LOG_TRACE(log, "Puffin files cache hit for {} | {} at offset {} length {} for data file {}", key.file_path, key.etag, key.content_offset, key.content_size_in_bytes, key.referenced_data_file); + ProfileEvents::increment(ProfileEvents::PuffinFilesCacheHits); + } + + return cloneExcludedRows(*result.first); + } + +private: + static DataLakeObjectMetadata::ExcludedRowsPtr cloneExcludedRows(const PuffinFilesCacheCell & cell); + + LoggerPtr log; + + void onEntryRemoval(size_t weight_loss, const MappedPtr &) override; +}; + +using PuffinFilesCachePtr = std::shared_ptr; + +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_deletion_vector_before_equality_filter.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_deletion_vector_before_equality_filter.cpp new file mode 100644 index 000000000000..1d17d21d7d2b --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_deletion_vector_before_equality_filter.cpp @@ -0,0 +1,68 @@ +#include + +#include +#include +#include +#include + +using namespace DB; + +namespace +{ + +Chunk makeChunkWithFileRowNumbers(const std::vector & values, size_t row_num_offset = 0) +{ + auto column = ColumnUInt64::create(); + for (UInt64 value : values) + column->insert(value); + + Chunk chunk(Columns{std::move(column)}, values.size()); + chunk.getChunkInfos().add(std::make_shared(row_num_offset)); + return chunk; +} + +/// Mimics Iceberg equality-delete `FilterTransform`: shrink columns, leave `applied_filter` unset. +void shrinkWithoutAppliedFilter(Chunk & chunk, const IColumn::Filter & filter) +{ + size_t result_size = 0; + for (UInt8 keep : filter) + result_size += keep != 0; + + auto columns = chunk.detachColumns(); + for (auto & column : columns) + column = column->filter(filter, -1); + chunk.setColumns(std::move(columns), result_size); +} + +std::vector readValues(const Chunk & chunk) +{ + const auto & column = assert_cast(*chunk.getColumns().at(0)); + std::vector values; + values.reserve(column.size()); + for (size_t i = 0; i < column.size(); ++i) + values.push_back(column.getData()[i]); + return values; +} + +std::shared_ptr makeExcludedRows(std::initializer_list positions) +{ + auto excluded_rows = std::make_shared(); + for (UInt64 position : positions) + excluded_rows->add(position); + return excluded_rows; +} + +} + +/// Equality removes file row 1; DV deletes file position 2. Survivors must be {0, 3}. +TEST(DeletionVectorBeforeEqualityFilter, CorrectOrderKeepsFileRowMapping) +{ + Chunk chunk = makeChunkWithFileRowNumbers({0, 1, 2, 3}); + + DeletionVectorTransform::transform(chunk, *makeExcludedRows({2})); + ASSERT_EQ(readValues(chunk), (std::vector{0, 1, 3})); + + /// Drop equality-deleted value 1 without updating applied_filter (as FilterTransform does). + shrinkWithoutAppliedFilter(chunk, IColumn::Filter{1, 0, 1}); + EXPECT_EQ(readValues(chunk), (std::vector{0, 3})); +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_has_non_empty_excluded_rows.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_has_non_empty_excluded_rows.cpp new file mode 100644 index 000000000000..9903493c3413 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_has_non_empty_excluded_rows.cpp @@ -0,0 +1,28 @@ +#include + +#include +#include + +using namespace DB; + +TEST(HasNonEmptyExcludedRows, EmptyOptionalIsFalse) +{ + EXPECT_FALSE(hasNonEmptyExcludedRows(std::nullopt)); +} + +TEST(HasNonEmptyExcludedRows, MissingOrEmptyBitmapIsFalse) +{ + DataLakeObjectMetadata metadata; + EXPECT_FALSE(hasNonEmptyExcludedRows(metadata)); + + metadata.excluded_rows = std::make_shared(); + EXPECT_FALSE(hasNonEmptyExcludedRows(metadata)); +} + +TEST(HasNonEmptyExcludedRows, NonEmptyBitmapIsTrue) +{ + DataLakeObjectMetadata metadata; + metadata.excluded_rows = std::make_shared(); + metadata.excluded_rows->add(7); + EXPECT_TRUE(hasNonEmptyExcludedRows(metadata)); +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_bounds.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_bounds.cpp new file mode 100644 index 000000000000..0c036309b73b --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_bounds.cpp @@ -0,0 +1,63 @@ +#include + +#include +#include +#include + +using namespace DB; + +namespace DB +{ +namespace ErrorCodes +{ +extern const int BAD_ARGUMENTS; +} +} + +TEST(PuffinDeletionVectorBounds, RejectsLengthExceedingFileSize) +{ + const String data(64, '\0'); + ReadBufferFromOutsideMemoryFile file("test.puffin", data); + + try + { + readDeletionVectorFromPuffin(file, 0, 1000); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("offset/length out of bounds"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorBounds, RejectsOffsetPlusLengthOverflow) +{ + const String data(64, '\0'); + ReadBufferFromOutsideMemoryFile file("test.puffin", data); + + try + { + readDeletionVectorFromPuffin(file, 60, 12); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("offset/length out of bounds"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorBounds, ValidatePuffinBlobBoundsRejectsNegativeOffset) +{ + try + { + validatePuffinBlobBounds(-1, 10, 64); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("offset/length out of bounds"), std::string::npos); + } +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_cardinality.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_cardinality.cpp new file mode 100644 index 000000000000..9bae4c4511f5 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_cardinality.cpp @@ -0,0 +1,96 @@ +#include + +#include +#include +#include + +#include + +using namespace DB; + +namespace DB +{ +namespace ErrorCodes +{ +extern const int BAD_ARGUMENTS; +} +} + +namespace +{ + +/// deletion-vector-v1 blob for positions {2, 5} (cardinality 2). +constexpr UInt8 two_position_dv_blob[] = { + 0x00, 0x00, 0x00, 0x24, 0xD1, 0xD3, 0x39, 0x64, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x3A, 0x30, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x05, 0x00, 0x2C, 0xDB, 0x9F, 0xC1, +}; + +constexpr UInt64 PUFFIN_DV_MAX_MATERIALIZED_POSITIONS = 100'000'000; +constexpr Int64 large_declared_length = 64 * 1024 * 1024; + +} + +TEST(PuffinDeletionVectorCardinality, RejectsCardinalityAboveMaterializationLimitBeforeParse) +{ + /// Ceiling is checked before payload validation, so even an empty blob must fail closed. + try + { + deserializeDeletionVectorV1Blob(std::string_view{}, PUFFIN_DV_MAX_MATERIALIZED_POSITIONS + 1); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("exceeds materialization limit"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorCardinality, RejectsCardinalityAboveLimitBeforeFullAllocate) +{ + /// Huge declared length with only a tiny buffer: without an early ceiling check this would + /// allocate `large_declared_length` (or fail mid-read after that allocate). ReadBufferFromMemory + /// does not expose file size, so bounds checks alone do not stop this. + const char header[8] = {}; + ReadBufferFromMemory file(header, sizeof(header)); + + try + { + readDeletionVectorFromPuffin( + file, 0, large_declared_length, PUFFIN_DV_MAX_MATERIALIZED_POSITIONS + 1); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("exceeds materialization limit"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorCardinality, RejectsBitmapExceedingDeclaredCardinality) +{ + const std::string_view blob( + reinterpret_cast(two_position_dv_blob), sizeof(two_position_dv_blob)); + + try + { + deserializeDeletionVectorV1Blob(blob, /*expected_cardinality=*/1); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("exceeds declared cardinality"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorCardinality, AcceptsMatchingCardinality) +{ + const std::string_view blob( + reinterpret_cast(two_position_dv_blob), sizeof(two_position_dv_blob)); + + const auto positions = deserializeDeletionVectorV1Blob(blob, /*expected_cardinality=*/2); + ASSERT_EQ(positions.size(), 2u); + EXPECT_EQ(positions[0], 2u); + EXPECT_EQ(positions[1], 5u); +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_envelope.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_envelope.cpp new file mode 100644 index 000000000000..89c0f97fce7f --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_envelope.cpp @@ -0,0 +1,98 @@ +#include + +#include +#include +#include + +#include + +using namespace DB; + +namespace DB +{ +namespace ErrorCodes +{ +extern const int BAD_ARGUMENTS; +} +} + +namespace +{ + +/// deletion-vector-v1 blob for positions {2, 5} (cardinality 2). +constexpr UInt8 two_position_dv_blob[] = { + 0x00, 0x00, 0x00, 0x24, 0xD1, 0xD3, 0x39, 0x64, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x3A, 0x30, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x05, 0x00, 0x2C, 0xDB, 0x9F, 0xC1, +}; + +/// Large declared length that would force a huge allocate if peeked after full read. +constexpr Int64 large_declared_length = 64 * 1024 * 1024; + +} + +TEST(PuffinDeletionVectorEnvelope, RejectsLengthBelowEnvelopeMinimum) +{ + const char zeros[16] = {}; + ReadBufferFromMemory file(zeros, sizeof(zeros)); + + try + { + readDeletionVectorFromPuffin(file, 0, 11); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("too small"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorEnvelope, RejectsInvalidMagicBeforeFullAllocate) +{ + /// Only 8 bytes available; declared length is huge. Without envelope peek this would allocate + /// `large_declared_length` (or fail mid-read after that allocate). ReadBufferFromMemory does not + /// expose file size, so the absolute 2 GiB / bounds checks alone do not stop this. + const UInt8 header[8] = {0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x00}; // wrong magic + ReadBufferFromMemory file(header, sizeof(header)); + + try + { + readDeletionVectorFromPuffin(file, 0, large_declared_length); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("Invalid deletion vector magic"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorEnvelope, RejectsCombinedLengthMismatchBeforeFullAllocate) +{ + /// Valid magic, but combined_length implies blob size 0x24+8=44, while caller length is huge. + const UInt8 header[8] = {0x00, 0x00, 0x00, 0x24, 0xD1, 0xD3, 0x39, 0x64}; + ReadBufferFromMemory file(header, sizeof(header)); + + try + { + readDeletionVectorFromPuffin(file, 0, large_declared_length); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("does not match combined length"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorEnvelope, ReadsValidBlobAfterEnvelopePeek) +{ + ReadBufferFromMemory file(two_position_dv_blob, sizeof(two_position_dv_blob)); + const auto positions = readDeletionVectorFromPuffin( + file, 0, static_cast(sizeof(two_position_dv_blob)), /*expected_cardinality=*/2); + + ASSERT_EQ(positions.size(), 2u); + EXPECT_EQ(positions[0], 2u); + EXPECT_EQ(positions[1], 5u); +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_clone.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_clone.cpp new file mode 100644 index 000000000000..ff0193cb9206 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_clone.cpp @@ -0,0 +1,87 @@ +#include + +#include +#include +#include + +using namespace DB; + +namespace CurrentMetrics +{ +extern const Metric PuffinFilesCacheBytes; +extern const Metric PuffinFilesCacheFiles; +} + +namespace +{ + +DataLakeObjectMetadata::ExcludedRowsPtr makeExcludedRows(const std::vector & positions) +{ + auto excluded_rows = std::make_shared(); + for (size_t position : positions) + excluded_rows->add(position); + return excluded_rows; +} + +} + +TEST(PuffinFilesCacheClone, CacheHitReturnsIndependentCopy) +{ + PuffinFilesCache cache("SLRU", 1'000'000, 100, 0.5); + + const auto key = PuffinFilesCache::tryCreateKey("puffin.bin", "etag-1", 100, 200, "data/file-a.parquet"); + ASSERT_TRUE(key.has_value()); + + size_t load_calls = 0; + auto load_fn = [&]() + { + ++load_calls; + return makeExcludedRows({1, 5, 10}); + }; + + auto first = cache.getOrSetDeletionVector(*key, load_fn); + auto second = cache.getOrSetDeletionVector(*key, load_fn); + + ASSERT_EQ(load_calls, 1); + ASSERT_NE(first, second); + EXPECT_TRUE(first->rb_contains(1)); + EXPECT_TRUE(first->rb_contains(5)); + EXPECT_TRUE(first->rb_contains(10)); + EXPECT_FALSE(first->rb_contains(99)); + + first->add(99); + + EXPECT_FALSE(second->rb_contains(99)); + + auto third = cache.getOrSetDeletionVector(*key, load_fn); + ASSERT_NE(third, first); + EXPECT_FALSE(third->rb_contains(99)); + EXPECT_TRUE(third->rb_contains(10)); +} + +TEST(PuffinFilesCacheClone, EmptyExcludedRowsReturnsNullptr) +{ + PuffinFilesCache cache("SLRU", 1'000'000, 100, 0.5); + + const auto key = PuffinFilesCache::tryCreateKey("puffin.bin", "etag-1", 100, 200, "data/file-a.parquet"); + ASSERT_TRUE(key.has_value()); + + const auto files_before = CurrentMetrics::get(CurrentMetrics::PuffinFilesCacheFiles); + const auto bytes_before = CurrentMetrics::get(CurrentMetrics::PuffinFilesCacheBytes); + + size_t load_calls = 0; + auto load_fn = [&]() + { + ++load_calls; + return DataLakeObjectMetadata::ExcludedRowsPtr{}; + }; + + auto first = cache.getOrSetDeletionVector(*key, load_fn); + auto second = cache.getOrSetDeletionVector(*key, load_fn); + + EXPECT_EQ(first, nullptr); + EXPECT_EQ(second, nullptr); + EXPECT_EQ(load_calls, 1); + EXPECT_EQ(CurrentMetrics::get(CurrentMetrics::PuffinFilesCacheFiles), files_before + 1); + EXPECT_EQ(CurrentMetrics::get(CurrentMetrics::PuffinFilesCacheBytes), bytes_before + 1); +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_key.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_key.cpp new file mode 100644 index 000000000000..0252c8b61d75 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_key.cpp @@ -0,0 +1,33 @@ +#include + +#include + +using namespace DB; + +TEST(PuffinFilesCacheKey, SamePuffinSliceDifferentReferencedDataFile) +{ + const auto key1 = PuffinFilesCache::tryCreateKey("puffin.bin", "etag-1", 100, 200, "data/file-a.parquet"); + const auto key2 = PuffinFilesCache::tryCreateKey("puffin.bin", "etag-1", 100, 200, "data/file-b.parquet"); + + ASSERT_TRUE(key1.has_value()); + ASSERT_TRUE(key2.has_value()); + EXPECT_NE(*key1, *key2); + EXPECT_NE(PuffinFilesCacheKeyHash{}(*key1), PuffinFilesCacheKeyHash{}(*key2)); +} + +TEST(PuffinFilesCacheKey, SameReferencedDataFileProducesEqualKeys) +{ + const auto key1 = PuffinFilesCache::tryCreateKey("puffin.bin", "etag-1", 100, 200, "data/file-a.parquet"); + const auto key2 = PuffinFilesCache::tryCreateKey("puffin.bin", "etag-1", 100, 200, "data/file-a.parquet"); + + ASSERT_TRUE(key1.has_value()); + ASSERT_TRUE(key2.has_value()); + EXPECT_EQ(*key1, *key2); + EXPECT_EQ(PuffinFilesCacheKeyHash{}(*key1), PuffinFilesCacheKeyHash{}(*key2)); +} + +TEST(PuffinFilesCacheKey, EmptyEtagReturnsNullopt) +{ + const auto key = PuffinFilesCache::tryCreateKey("puffin.bin", "", 100, 200, "data/file-a.parquet"); + EXPECT_FALSE(key.has_value()); +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_weight.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_weight.cpp new file mode 100644 index 000000000000..f1d307b90f9c --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_weight.cpp @@ -0,0 +1,46 @@ +#include + +#include +#include +#include +#include + +using namespace DB; + +namespace CurrentMetrics +{ +extern const Metric PuffinFilesCacheBytes; +} + +namespace +{ + +DataLakeObjectMetadata::ExcludedRowsPtr makeLargeSparseExcludedRows() +{ + auto excluded_rows = std::make_shared(); + for (size_t i = 0; i < 33; ++i) + excluded_rows->add((static_cast(i) + 1) << 32); + return excluded_rows; +} + +} + +TEST(RoaringBitmapWithSmallSetMemory, LargeSparseBitmapAllocatedBytesExceedCardinalityEstimate) +{ + const auto excluded_rows = makeLargeSparseExcludedRows(); + + ASSERT_TRUE(excluded_rows->isLarge()); + EXPECT_GT(excluded_rows->getAllocatedBytes(), excluded_rows->size() * sizeof(size_t)); +} + +TEST(PuffinFilesCacheWeight, UsesRoaringAllocatedBytesForWeight) +{ + const auto excluded_rows = makeLargeSparseExcludedRows(); + const auto key = PuffinFilesCache::tryCreateKey("puffin.bin", "etag-1", 100, 200, "data/file-a.parquet"); + ASSERT_TRUE(key.has_value()); + + PuffinFilesCache cache("SLRU", 1'000'000, 100, 0.5); + cache.getOrSetDeletionVector(*key, [&]() { return excluded_rows; }); + + EXPECT_GE(CurrentMetrics::get(CurrentMetrics::PuffinFilesCacheBytes), excluded_rows->getAllocatedBytes()); +} diff --git a/src/Storages/ObjectStorage/IObjectIterator.cpp b/src/Storages/ObjectStorage/IObjectIterator.cpp index 7ae3d1cbb1b2..b03ca71b325c 100644 --- a/src/Storages/ObjectStorage/IObjectIterator.cpp +++ b/src/Storages/ObjectStorage/IObjectIterator.cpp @@ -170,19 +170,19 @@ ObjectInfoPtr ObjectIteratorSplitByBuckets::next(size_t id) auto file_bucket_infos = splitter->splitToBuckets(bucket_size, *buffer, format_settings); for (const auto & file_bucket : file_bucket_infos) { - auto copy_object_info = *last_object_info; + auto copy_object_info = last_object_info->clone(); if (has_cache_entry) { auto filtered = file_bucket->filterByMatchingRowGroups(matching_row_groups); if (!filtered) continue; - copy_object_info.file_bucket_info = std::move(filtered); + copy_object_info->file_bucket_info = std::move(filtered); } else { - copy_object_info.file_bucket_info = file_bucket; + copy_object_info->file_bucket_info = file_bucket; } - pending_objects_info.push(std::make_shared(copy_object_info)); + pending_objects_info.push(std::move(copy_object_info)); } } } diff --git a/src/Storages/ObjectStorage/IObjectIterator.h b/src/Storages/ObjectStorage/IObjectIterator.h index 47febf269ee4..c922f0a70243 100644 --- a/src/Storages/ObjectStorage/IObjectIterator.h +++ b/src/Storages/ObjectStorage/IObjectIterator.h @@ -53,6 +53,10 @@ struct ObjectInfo FileBucketInfoPtr file_bucket_info; + /// Polymorphic copy. Used when splitting files into buckets so derived metadata + /// (e.g. Iceberg equality / position deletes) is not sliced away. + virtual ObjectInfoPtr clone() const { return std::make_shared(*this); } + String getIdentifier() const; }; diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 8521f6c309af..c79da9e5100e 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -122,6 +122,19 @@ static void logIcebergFileStats(const ObjectInfoPtr & object_info, const LoggerP #endif } +/// Count-from-files cache key is data-file identity only. Skip when row filtering can change +/// independently (DVs / selection vectors, Iceberg eq/pos deletes) or when the task is a bucket subset. +static bool canUseCountFromFilesCache(const ObjectInfoPtr & object_info) +{ + if (hasNonEmptyExcludedRows(object_info->data_lake_metadata) || object_info->file_bucket_info) + return false; +#if USE_AVRO + if (hasIcebergEqualityDeletes(object_info) || hasIcebergPositionDeletes(object_info)) + return false; +#endif + return true; +} + StorageObjectStorageSource::StorageObjectStorageSource( const StorageID & storage_id_, String name_, @@ -585,7 +598,8 @@ Chunk StorageObjectStorageSource::generate() } if (reader.getInputFormat() && read_context->getSettingsRef()[Setting::use_cache_for_count_from_files] - && !format_filter_info->filter_actions_dag) + && !format_filter_info->filter_actions_dag + && canUseCountFromFilesCache(reader.getObjectInfo())) addNumRowsToCache(*reader.getObjectInfo(), total_rows_in_file); total_rows_in_file = 0; @@ -747,8 +761,24 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade return schema_cache->tryGetNumRows(cache_key, get_last_mod_time); }; - std::optional num_rows_from_cache - = need_only_count && context_->getSettingsRef()[Setting::use_cache_for_count_from_files] ? try_get_num_rows_from_cache() : std::nullopt; + /// The count-from-cache shortcut builds a `ConstChunkGenerator` without opening the read buffer, so a + /// requested `_headers` virtual column (the HTTP response headers of the data `GET`) would have to fall + /// back to the metadata-probe headers (usually a `HEAD`), which can differ from the actual `GET` + /// response. Skip the shortcut when `_headers` is requested so the real `GET` headers are used. + const bool headers_requested = read_from_format_info.requested_virtual_columns.contains("_headers"); + +#if USE_AVRO + /// Equality deletes filter by column values; need_only_count emits default-filled chunks. + const bool effective_need_only_count = need_only_count && !hasIcebergEqualityDeletes(object_info); +#else + const bool effective_need_only_count = need_only_count; +#endif + + const bool can_use_count_cache = effective_need_only_count && headers_requested + && context_->getSettingsRef()[Setting::use_cache_for_count_from_files] + && canUseCountFromFilesCache(object_info); + + std::optional num_rows_from_cache = can_use_count_cache ? try_get_num_rows_from_cache() : std::nullopt; if (num_rows_from_cache) { @@ -919,7 +949,7 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade filter_info, true /* is_remote_fs */, compression_method, - need_only_count, + effective_need_only_count, std::nullopt /*min_block_size_bytes*/, std::nullopt /*min_block_size_rows*/, std::nullopt /*max_block_size_bytes*/); @@ -937,22 +967,22 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade filter_info, true /* is_remote_fs */, compression_method, - need_only_count); + effective_need_only_count); } input_format->setBucketsToRead(object_info->file_bucket_info); input_format->setSerializationHints(read_from_format_info.serialization_hints); - if (need_only_count) + if (effective_need_only_count) input_format->needOnlyCount(); builder.init(Pipe(input_format)); - configuration->addDeleteTransformers(object_info, builder, format_settings, parser_shared_resources, context_); - - if (object_info->data_lake_metadata - && object_info->data_lake_metadata->excluded_rows - && object_info->data_lake_metadata->excluded_rows->size() > 0) + /// Deletion vectors (and selection vectors) address absolute file row numbers via + /// `ChunkInfoRowNumbers`. Iceberg equality deletes use a plain `FilterTransform` that + /// shrinks the chunk without maintaining `applied_filter`, so DV must run first — + /// otherwise later DV filtering maps dense post-equality indices to the wrong file rows. + if (hasNonEmptyExcludedRows(object_info->data_lake_metadata)) { builder.addSimpleTransform([&](const SharedHeader & header) { @@ -960,6 +990,8 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade }); } + configuration->addDeleteTransformers(object_info, builder, format_settings, parser_shared_resources, context_); + std::optional schema_transform; if (object_info->data_lake_metadata && object_info->data_lake_metadata->schema_transform) { diff --git a/src/Storages/ObjectStorage/tests/gtest_rendezvous_hashing.cpp b/src/Storages/ObjectStorage/tests/gtest_rendezvous_hashing.cpp index b00c1d609fa1..fdd534a1278f 100644 --- a/src/Storages/ObjectStorage/tests/gtest_rendezvous_hashing.cpp +++ b/src/Storages/ObjectStorage/tests/gtest_rendezvous_hashing.cpp @@ -1,9 +1,26 @@ #include +#include +#include #include #include +#include +#include + +#if USE_PARQUET +#include +#include +#endif using namespace DB; +namespace DB +{ +namespace ErrorCodes +{ +extern const int UNKNOWN_PROTOCOL; +} +} + namespace { class TestIterator : public IObjectIterator @@ -231,3 +248,225 @@ TEST(RendezvousHashing, MultipleNodesReducedClusterOneByOne) ASSERT_TRUE(checkHead(paths0, {1, 5, 6, 7, 8, 9})); ASSERT_TRUE(checkHead(paths1, {0, 2, 3, 4})); } + +TEST(ClusterFunctionReadTaskResponse, RejectsNonEmptyExcludedRowsOnOldProtocol) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.data_lake_metadata.excluded_rows = std::make_shared(); + response.data_lake_metadata.excluded_rows->add(7); + + String serialized; + WriteBufferFromString out(serialized); + try + { + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_FILE_BUCKETS_INFO); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::UNKNOWN_PROTOCOL); + EXPECT_NE(e.message().find("excluded_rows"), std::string::npos); + } +} + +TEST(ClusterFunctionReadTaskResponse, AllowsEmptyExcludedRowsOnOldProtocol) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.data_lake_metadata.excluded_rows = std::make_shared(); + + String serialized; + WriteBufferFromString out(serialized); + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_FILE_BUCKETS_INFO); + out.finalize(); + EXPECT_FALSE(serialized.empty()); +} + +TEST(ClusterFunctionReadTaskResponse, RoundTripsExcludedRowsOnSupportedProtocol) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.data_lake_metadata.excluded_rows = std::make_shared(); + response.data_lake_metadata.excluded_rows->add(3); + response.data_lake_metadata.excluded_rows->add(9); + + String serialized; + WriteBufferFromString out(serialized); + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_EXCLUDED_ROWS); + out.finalize(); + + ReadBufferFromString in(serialized); + ClusterFunctionReadTaskResponse deserialized; + deserialized.deserialize(in); + + ASSERT_TRUE(deserialized.data_lake_metadata.excluded_rows); + EXPECT_EQ(deserialized.data_lake_metadata.excluded_rows->size(), 2u); + EXPECT_TRUE(deserialized.data_lake_metadata.excluded_rows->rb_contains(3)); + EXPECT_TRUE(deserialized.data_lake_metadata.excluded_rows->rb_contains(9)); +} + +TEST(ClusterFunctionReadTaskResponse, RejectsEqualityDeletesOnProtocolBeforeIcebergMetadata) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.iceberg_info = Iceberg::IcebergObjectSerializableInfo{}; + response.iceberg_info->equality_deletes_objects.push_back( + Iceberg::EqualityDeleteObject{ + .file_path = "/path/eq.parquet", + .file_format = "PARQUET", + .equality_ids = std::vector{1}, + .schema_id = 0, + }); + + String serialized; + WriteBufferFromString out(serialized); + try + { + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_DATA_LAKE_METADATA); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::UNKNOWN_PROTOCOL); + EXPECT_NE(e.message().find("iceberg_info"), std::string::npos); + } +} + +TEST(ClusterFunctionReadTaskResponse, RejectsPositionDeletesOnProtocolBeforeIcebergMetadata) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.iceberg_info = Iceberg::IcebergObjectSerializableInfo{}; + response.iceberg_info->position_deletes_objects.push_back( + Iceberg::PositionDeleteObject{ + .file_path = "/path/pos.parquet", + .file_format = "PARQUET", + .reference_data_file_path = std::nullopt, + .sequence_number = 1, + }); + + String serialized; + WriteBufferFromString out(serialized); + try + { + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_DATA_LAKE_METADATA); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::UNKNOWN_PROTOCOL); + EXPECT_NE(e.message().find("iceberg_info"), std::string::npos); + } +} + +TEST(ClusterFunctionReadTaskResponse, AllowsIcebergInfoWithoutDeletesOnProtocolBeforeIcebergMetadata) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.iceberg_info = Iceberg::IcebergObjectSerializableInfo{}; + + String serialized; + WriteBufferFromString out(serialized); + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_DATA_LAKE_METADATA); + out.finalize(); + EXPECT_FALSE(serialized.empty()); +} + +TEST(ClusterFunctionReadTaskResponse, RoundTripsIcebergDeletesOnSupportedProtocol) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.iceberg_info = Iceberg::IcebergObjectSerializableInfo{}; + response.iceberg_info->data_object_file_path_key + = Iceberg::IcebergPathFromMetadata::deserialize("s3://bucket/path/file.parquet"); + response.iceberg_info->file_format = "PARQUET"; + response.iceberg_info->equality_deletes_objects.push_back( + Iceberg::EqualityDeleteObject{ + .file_path = "/path/eq.parquet", + .file_format = "PARQUET", + .equality_ids = std::vector{1, 2}, + .schema_id = 7, + }); + response.iceberg_info->position_deletes_objects.push_back( + Iceberg::PositionDeleteObject{ + .file_path = "/path/pos.parquet", + .file_format = "PARQUET", + .reference_data_file_path = std::nullopt, + .sequence_number = 42, + }); + + String serialized; + WriteBufferFromString out(serialized); + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_ICEBERG_METADATA); + out.finalize(); + + ReadBufferFromString in(serialized); + ClusterFunctionReadTaskResponse deserialized; + deserialized.deserialize(in); + + ASSERT_TRUE(deserialized.iceberg_info.has_value()); + ASSERT_EQ(deserialized.iceberg_info->equality_deletes_objects.size(), 1u); + EXPECT_EQ(deserialized.iceberg_info->equality_deletes_objects[0].file_path, "/path/eq.parquet"); + ASSERT_EQ(deserialized.iceberg_info->position_deletes_objects.size(), 1u); + EXPECT_EQ(deserialized.iceberg_info->position_deletes_objects[0].file_path, "/path/pos.parquet"); +} + +#if USE_PARQUET + +TEST(ClusterFunctionReadTaskResponse, RejectsFileBucketInfoOnProtocolBeforeFileBuckets) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.file_bucket_info = std::make_shared(std::vector{0, 1}); + + String serialized; + WriteBufferFromString out(serialized); + try + { + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_ICEBERG_METADATA); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::UNKNOWN_PROTOCOL); + EXPECT_NE(e.message().find("file_bucket_info"), std::string::npos); + } +} + +TEST(ClusterFunctionReadTaskResponse, AllowsMissingFileBucketInfoOnProtocolBeforeFileBuckets) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + + String serialized; + WriteBufferFromString out(serialized); + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_ICEBERG_METADATA); + out.finalize(); + EXPECT_FALSE(serialized.empty()); +} + +TEST(ClusterFunctionReadTaskResponse, RoundTripsFileBucketInfoOnSupportedProtocol) +{ + tryRegisterFormats(); + + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.file_bucket_info = std::make_shared(std::vector{2, 5}); + + String serialized; + WriteBufferFromString out(serialized); + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_FILE_BUCKETS_INFO); + out.finalize(); + + ReadBufferFromString in(serialized); + ClusterFunctionReadTaskResponse deserialized; + deserialized.deserialize(in); + + ASSERT_TRUE(deserialized.file_bucket_info); + auto * parquet_buckets = dynamic_cast(deserialized.file_bucket_info.get()); + ASSERT_TRUE(parquet_buckets); + EXPECT_EQ(parquet_buckets->row_group_ids, (std::vector{2, 5})); +} + +#endif diff --git a/tests/integration/test_storage_iceberg_with_spark/test_deletion_vectors.py b/tests/integration/test_storage_iceberg_with_spark/test_deletion_vectors.py new file mode 100644 index 000000000000..127df6ab993d --- /dev/null +++ b/tests/integration/test_storage_iceberg_with_spark/test_deletion_vectors.py @@ -0,0 +1,265 @@ +import uuid + +import pytest + +from helpers.iceberg_utils import ( + default_upload_directory, + get_uuid_str, + get_creation_expression, +) + + +def get_array(query_result: str): + return sorted([int(x) for x in query_result.strip().split("\n") if x]) + + +def expected_complex_ids(): + ids = list(range(20, 90)) + list(range(100, 150)) + ids += [x for x in range(200, 250) if x not in {205, 210, 220}] + return sorted(ids) + + +def upload_table(cluster, storage_type, table_name): + default_upload_directory( + cluster, + storage_type, + f"/iceberg_data/default/{table_name}/", + f"/iceberg_data/default/{table_name}/", + ) + + +@pytest.mark.parametrize("run_on_cluster", [False, True]) +@pytest.mark.parametrize("storage_type", ["s3", "azure", "local"]) +def test_deletion_vectors(started_cluster_iceberg_with_spark, storage_type, run_on_cluster): + if storage_type == "local" and run_on_cluster: + pytest.skip("Local storage with cluster execution is not supported") + + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + table_name = "test_deletion_vectors_" + storage_type + "_" + get_uuid_str() + deleted_ids = [2, 5, 7, 100] + + spark.sql( + f""" + CREATE TABLE {table_name} (id bigint) USING iceberg + TBLPROPERTIES ( + 'format-version' = '3', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ) + """ + ) + spark.sql(f"INSERT INTO {table_name} SELECT id FROM range(0, 200)") + spark.sql( + f"DELETE FROM {table_name} WHERE id IN ({', '.join(str(x) for x in deleted_ids)})" + ) + + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + + expression = get_creation_expression( + storage_type, + table_name, + started_cluster_iceberg_with_spark, + run_on_cluster=run_on_cluster, + table_function=True, + ) + + assert int(instance.query(f"SELECT count() FROM {expression}")) == 200 - len(deleted_ids) + assert get_array(instance.query(f"SELECT id FROM {expression}")) == [ + x for x in range(200) if x not in deleted_ids + ] + + +@pytest.mark.parametrize("storage_type", ["s3", "azure", "local"]) +def test_deletion_vectors_complex(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + table_name = "test_deletion_vectors_complex_" + storage_type + "_" + get_uuid_str() + expected_ids = expected_complex_ids() + + spark.sql( + f""" + CREATE TABLE {table_name} (id bigint, data string) USING iceberg + PARTITIONED BY (bucket(5, id)) + TBLPROPERTIES ( + 'format-version' = '3', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ) + """ + ) + spark.sql( + f"INSERT INTO {table_name} SELECT id, char(id + ascii('a')) FROM range(10, 100)" + ) + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + + expression = get_creation_expression( + storage_type, + table_name, + started_cluster_iceberg_with_spark, + table_function=True, + ) + + assert int(instance.query(f"SELECT count(id) FROM {expression}")) == 90 + assert get_array(instance.query(f"SELECT id FROM {expression}")) == list(range(10, 100)) + + spark.sql(f"DELETE FROM {table_name} WHERE id < 20") + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + assert get_array(instance.query(f"SELECT id FROM {expression}")) == list(range(20, 100)) + + spark.sql(f"DELETE FROM {table_name} WHERE id >= 90") + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + assert get_array(instance.query(f"SELECT id FROM {expression}")) == list(range(20, 90)) + + spark.sql( + f"INSERT INTO {table_name} SELECT id, char(id + ascii('a')) FROM range(100, 200)" + ) + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + assert get_array(instance.query(f"SELECT id FROM {expression}")) == list(range(20, 90)) + list( + range(100, 200) + ) + + spark.sql(f"DELETE FROM {table_name} WHERE id >= 150") + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + assert get_array(instance.query(f"SELECT id FROM {expression}")) == list(range(20, 90)) + list( + range(100, 150) + ) + + spark.sql(f"ALTER TABLE {table_name} ADD COLUMNS (label string)") + spark.sql( + f""" + INSERT INTO {table_name} + SELECT id, char(id + ascii('a')), 'new' + FROM range(200, 250) + """ + ) + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + assert get_array(instance.query(f"SELECT id FROM {expression}")) == list(range(20, 90)) + list( + range(100, 150) + ) + list(range(200, 250)) + assert int(instance.query(f"SELECT count(id) FROM {expression} WHERE label = 'new'")) == 50 + + spark.sql(f"DELETE FROM {table_name} WHERE id IN (205, 210, 220)") + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + assert get_array(instance.query(f"SELECT id FROM {expression}")) == expected_ids + assert int(instance.query(f"SELECT count(id) FROM {expression}")) == len(expected_ids) + + spark.sql(f"UPDATE {table_name} SET label = 'updated' WHERE id = 25") + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + assert instance.query(f"SELECT label FROM {expression} WHERE id = 25").strip() == "updated" + assert int(instance.query(f"SELECT count(id) FROM {expression} WHERE label = 'updated'")) == 1 + + spark.sql(f"CALL system.rewrite_data_files(table => '{table_name}')") + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + assert get_array(instance.query(f"SELECT id FROM {expression}")) == expected_ids + + assert get_array( + instance.query( + f"SELECT id FROM {expression} WHERE id % 3 = 0" + ) + ) == sorted([x for x in expected_ids if x % 3 == 0]) + + +@pytest.mark.parametrize("storage_type", ["s3"]) +def test_deletion_vectors_puffin_files_cache(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + table_name = "test_deletion_vectors_cache_" + storage_type + "_" + get_uuid_str() + deleted_ids = [2, 5, 7, 100] + + spark.sql( + f""" + CREATE TABLE {table_name} (id bigint) USING iceberg + TBLPROPERTIES ( + 'format-version' = '3', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ) + """ + ) + spark.sql(f"INSERT INTO {table_name} SELECT id FROM range(0, 200)") + spark.sql( + f"DELETE FROM {table_name} WHERE id IN ({', '.join(str(x) for x in deleted_ids)})" + ) + + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + + expression = get_creation_expression( + storage_type, + table_name, + started_cluster_iceberg_with_spark, + table_function=True, + ) + + instance.query("SYSTEM DROP PUFFIN_FILES_CACHE") + + query_id1 = f"{table_name}-{uuid.uuid4()}" + query_id2 = f"{table_name}-{uuid.uuid4()}" + query_id3 = f"{table_name}-{uuid.uuid4()}" + + assert int( + instance.query( + f"SELECT count(id) FROM {expression}", + query_id=query_id1, + settings={"use_puffin_files_cache": 1}, + ) + ) == 200 - len(deleted_ids) + + assert int( + instance.query( + f"SELECT count(id) FROM {expression}", + query_id=query_id2, + settings={"use_puffin_files_cache": 1}, + ) + ) == 200 - len(deleted_ids) + + instance.query("SYSTEM FLUSH LOGS") + + assert int( + instance.query( + f"SELECT ProfileEvents['PuffinFilesCacheMisses'] FROM system.query_log WHERE query_id = '{query_id1}' AND type = 'QueryFinish'" + ) + ) > 0 + assert int( + instance.query( + f"SELECT ProfileEvents['PuffinFilesCacheHits'] FROM system.query_log WHERE query_id = '{query_id2}' AND type = 'QueryFinish'" + ) + ) > 0 + + puffin_reads_first = int( + instance.query( + f"SELECT ProfileEvents['PuffinFilesRead'] FROM system.query_log WHERE query_id = '{query_id1}' AND type = 'QueryFinish'" + ) + ) + puffin_reads_second = int( + instance.query( + f"SELECT ProfileEvents['PuffinFilesRead'] FROM system.query_log WHERE query_id = '{query_id2}' AND type = 'QueryFinish'" + ) + ) + assert puffin_reads_first > 0 + assert puffin_reads_second == 0 + + instance.query("SYSTEM DROP PUFFIN_FILES_CACHE") + + assert int( + instance.query( + f"SELECT count(id) FROM {expression}", + query_id=query_id3, + settings={"use_puffin_files_cache": 1}, + ) + ) == 200 - len(deleted_ids) + + instance.query("SYSTEM FLUSH LOGS") + + assert int( + instance.query( + f"SELECT ProfileEvents['PuffinFilesCacheMisses'] FROM system.query_log WHERE query_id = '{query_id3}' AND type = 'QueryFinish'" + ) + ) > int( + instance.query( + f"SELECT ProfileEvents['PuffinFilesCacheMisses'] FROM system.query_log WHERE query_id = '{query_id2}' AND type = 'QueryFinish'" + ) + ) diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index 11fae0771496..7fc845218de4 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -128,6 +128,7 @@ SYSTEM DROP MARK CACHE ['SYSTEM CLEAR MARK CACHE','SYSTEM DROP MARK','DROP MARK SYSTEM DROP ICEBERG METADATA CACHE ['SYSTEM CLEAR ICEBERG_METADATA_CACHE','SYSTEM DROP ICEBERG_METADATA_CACHE'] GLOBAL SYSTEM DROP CACHE SYSTEM DROP AVRO SCHEMA CACHE ['SYSTEM CLEAR AVRO SCHEMA CACHE','SYSTEM DROP AVRO SCHEMA CACHE','DROP AVRO SCHEMA CACHE'] GLOBAL SYSTEM DROP CACHE SYSTEM DROP PARQUET METADATA CACHE ['SYSTEM DROP PARQUET_METADATA_CACHE'] GLOBAL SYSTEM DROP CACHE +SYSTEM DROP PUFFIN FILES CACHE ['SYSTEM DROP PUFFIN_FILES_CACHE'] GLOBAL SYSTEM DROP CACHE SYSTEM PREWARM PRIMARY INDEX CACHE ['SYSTEM PREWARM PRIMARY INDEX','PREWARM PRIMARY INDEX CACHE','PREWARM PRIMARY INDEX'] GLOBAL SYSTEM DROP CACHE SYSTEM DROP PRIMARY INDEX CACHE ['SYSTEM CLEAR PRIMARY INDEX CACHE','SYSTEM DROP PRIMARY INDEX','DROP PRIMARY INDEX CACHE','DROP PRIMARY INDEX'] GLOBAL SYSTEM DROP CACHE SYSTEM DROP UNCOMPRESSED CACHE ['SYSTEM CLEAR UNCOMPRESSED CACHE','SYSTEM DROP UNCOMPRESSED','DROP UNCOMPRESSED CACHE','DROP UNCOMPRESSED'] GLOBAL SYSTEM DROP CACHE diff --git a/tests/queries/0_stateless/04077_puffin_happy_path.reference b/tests/queries/0_stateless/04077_puffin_happy_path.reference new file mode 100644 index 000000000000..3c855ff323d9 --- /dev/null +++ b/tests/queries/0_stateless/04077_puffin_happy_path.reference @@ -0,0 +1,34 @@ +--- spark_deletion_vector.puffin --- +deletion-vector-v1 -1 -1 4 58 ['cardinality','referenced-data-file'] ['5','/tmp/clickhouse_spark_puffin_9eiw_i48/default/spark_puffin_source/data/00000-0-15387e93-06e3-48bb-ab87-ea4bfedb0c1d-0-00001.parquet'] +/tmp/clickhouse_spark_puffin_9eiw_i48/default/spark_puffin_source/data/00000-0-15387e93-06e3-48bb-ab87-ea4bfedb0c1d-0-00001.parquet [2,5,7,100,65536] +/tmp/clickhouse_spark_puffin_9eiw_i48/default/spark_puffin_source/data/00000-0-15387e93-06e3-48bb-ab87-ea4bfedb0c1d-0-00001.parquet 2 +/tmp/clickhouse_spark_puffin_9eiw_i48/default/spark_puffin_source/data/00000-0-15387e93-06e3-48bb-ab87-ea4bfedb0c1d-0-00001.parquet 5 +/tmp/clickhouse_spark_puffin_9eiw_i48/default/spark_puffin_source/data/00000-0-15387e93-06e3-48bb-ab87-ea4bfedb0c1d-0-00001.parquet 7 +/tmp/clickhouse_spark_puffin_9eiw_i48/default/spark_puffin_source/data/00000-0-15387e93-06e3-48bb-ab87-ea4bfedb0c1d-0-00001.parquet 100 +/tmp/clickhouse_spark_puffin_9eiw_i48/default/spark_puffin_source/data/00000-0-15387e93-06e3-48bb-ab87-ea4bfedb0c1d-0-00001.parquet 65536 +--- compressed_footer.puffin --- +deletion-vector-v1 -1 -1 4 58 ['cardinality','referenced-data-file'] ['5','/tmp/clickhouse_spark_puffin_9eiw_i48/default/spark_puffin_source/data/00000-0-15387e93-06e3-48bb-ab87-ea4bfedb0c1d-0-00001.parquet'] +/tmp/clickhouse_spark_puffin_9eiw_i48/default/spark_puffin_source/data/00000-0-15387e93-06e3-48bb-ab87-ea4bfedb0c1d-0-00001.parquet [2,5,7,100,65536] +/tmp/clickhouse_spark_puffin_9eiw_i48/default/spark_puffin_source/data/00000-0-15387e93-06e3-48bb-ab87-ea4bfedb0c1d-0-00001.parquet 2 +/tmp/clickhouse_spark_puffin_9eiw_i48/default/spark_puffin_source/data/00000-0-15387e93-06e3-48bb-ab87-ea4bfedb0c1d-0-00001.parquet 5 +/tmp/clickhouse_spark_puffin_9eiw_i48/default/spark_puffin_source/data/00000-0-15387e93-06e3-48bb-ab87-ea4bfedb0c1d-0-00001.parquet 7 +/tmp/clickhouse_spark_puffin_9eiw_i48/default/spark_puffin_source/data/00000-0-15387e93-06e3-48bb-ab87-ea4bfedb0c1d-0-00001.parquet 100 +/tmp/clickhouse_spark_puffin_9eiw_i48/default/spark_puffin_source/data/00000-0-15387e93-06e3-48bb-ab87-ea4bfedb0c1d-0-00001.parquet 65536 +--- mixed_blob_types.puffin --- +apache-datasketches-theta-v1 -1 -1 4 16 [] [] +deletion-vector-v1 -1 -1 20 44 ['cardinality','referenced-data-file'] ['2','/data/table/part-00000.parquet'] +/data/table/part-00000.parquet [2,5] +/data/table/part-00000.parquet 2 +/data/table/part-00000.parquet 5 +--- file_properties_ok.puffin --- +deletion-vector-v1 -1 -1 4 44 ['cardinality','referenced-data-file'] ['2','/data/table/part-00000.parquet'] +/data/table/part-00000.parquet [2,5] +/data/table/part-00000.parquet 2 +/data/table/part-00000.parquet 5 +--- dense_range_100k.puffin --- +100000 0 99999 +--- subset without deleted_rows --- +/data/table/part-00000.parquet +/data/table/part-00000.parquet +/data/table/part-00000.parquet +/data/table/part-00000.parquet diff --git a/tests/queries/0_stateless/04077_puffin_happy_path.sh b/tests/queries/0_stateless/04077_puffin_happy_path.sh new file mode 100755 index 000000000000..f41522da71f8 --- /dev/null +++ b/tests/queries/0_stateless/04077_puffin_happy_path.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +DATA="$CURDIR/data_puffin" + +run_happy_path() { + local name="$1" + local puffin="$2" + echo "--- $name ---" + $CLICKHOUSE_LOCAL -q " + SELECT blob_type, snapshot_id, sequence_number, offset, length, compression_codec, mapKeys(properties), mapValues(properties) + FROM file('$puffin', PuffinMetadata) + ORDER BY all + " + $CLICKHOUSE_LOCAL -q " + SELECT referenced_data_file, deleted_rows + FROM file('$puffin', Puffin) + ORDER BY all + " + $CLICKHOUSE_LOCAL -q " + SELECT referenced_data_file, row_number + FROM file('$puffin', Puffin) + ARRAY JOIN deleted_rows AS row_number + ORDER BY referenced_data_file, row_number + " +} + +run_happy_path "spark_deletion_vector.puffin" "$DATA/spark_deletion_vector.puffin" +run_happy_path "compressed_footer.puffin" "$DATA/compressed_footer.puffin" +run_happy_path "mixed_blob_types.puffin" "$DATA/mixed_blob_types.puffin" +run_happy_path "file_properties_ok.puffin" "$DATA/file_properties_ok.puffin" + +echo "--- dense_range_100k.puffin ---" +$CLICKHOUSE_LOCAL -q " +SELECT length(deleted_rows), deleted_rows[1], deleted_rows[100000] +FROM file('$DATA/dense_range_100k.puffin', Puffin) +" + +echo "--- subset without deleted_rows ---" +$CLICKHOUSE_LOCAL -q " +SELECT referenced_data_file +FROM file('$DATA/dense_range_100k.puffin', Puffin) +" +$CLICKHOUSE_LOCAL -q " +SELECT referenced_data_file +FROM file('$DATA/cardinality_exceeds_materialization_limit.puffin', Puffin) +" +# Corrupt payload is still readable when deleted_rows is not projected (payload checks are skipped). +$CLICKHOUSE_LOCAL -q " +SELECT referenced_data_file +FROM file('$DATA/dv_envelope_crc_mismatch.puffin', Puffin) +" +$CLICKHOUSE_LOCAL -q " +SELECT referenced_data_file +FROM file('$DATA/dense_range_100k.puffin', Puffin) +SETTINGS input_format_allow_seeks = 0 +" diff --git a/tests/queries/0_stateless/04117_parser_system_query_variants.reference b/tests/queries/0_stateless/04117_parser_system_query_variants.reference index 2ffe7e9fb91f..037f0692ae0e 100644 --- a/tests/queries/0_stateless/04117_parser_system_query_variants.reference +++ b/tests/queries/0_stateless/04117_parser_system_query_variants.reference @@ -15,6 +15,7 @@ SYSTEM CLEAR QUERY CONDITION CACHE SYSTEM CLEAR COMPILED EXPRESSION CACHE SYSTEM CLEAR ICEBERG METADATA CACHE SYSTEM CLEAR PARQUET METADATA CACHE +SYSTEM CLEAR PUFFIN FILES CACHE SYSTEM CLEAR FILESYSTEM CACHE SYSTEM CLEAR DISTRIBUTED CACHE SYSTEM CLEAR DISTRIBUTED CACHE CONNECTIONS ON CLUSTER cluster diff --git a/tests/queries/0_stateless/04117_parser_system_query_variants.sql b/tests/queries/0_stateless/04117_parser_system_query_variants.sql index c48bb44537f9..e52cb9f54caa 100644 --- a/tests/queries/0_stateless/04117_parser_system_query_variants.sql +++ b/tests/queries/0_stateless/04117_parser_system_query_variants.sql @@ -25,6 +25,7 @@ EXPLAIN SYNTAX SYSTEM DROP QUERY CONDITION CACHE; EXPLAIN SYNTAX SYSTEM DROP COMPILED EXPRESSION CACHE; EXPLAIN SYNTAX SYSTEM DROP ICEBERG METADATA CACHE; EXPLAIN SYNTAX SYSTEM DROP PARQUET METADATA CACHE; +EXPLAIN SYNTAX SYSTEM DROP PUFFIN_FILES_CACHE; EXPLAIN SYNTAX SYSTEM DROP FILESYSTEM CACHE; EXPLAIN SYNTAX SYSTEM DROP DISTRIBUTED CACHE; EXPLAIN SYNTAX SYSTEM DROP DISTRIBUTED CACHE CONNECTIONS ON CLUSTER cluster; diff --git a/tests/queries/0_stateless/04257_puffin_errors.reference b/tests/queries/0_stateless/04257_puffin_errors.reference new file mode 100644 index 000000000000..f8b8279a584d --- /dev/null +++ b/tests/queries/0_stateless/04257_puffin_errors.reference @@ -0,0 +1,38 @@ +--- overflow_offset_length.puffin --- +Puffin blob 0: offset/length out of bounds +--- negative_offset.puffin --- +Puffin blob 0: offset/length out of bounds +--- length_exceeds_file.puffin --- +Puffin blob 0: offset/length out of bounds +--- blob_overlaps_footer.puffin --- +Puffin blob 0: offset/length out of bounds +--- invalid_roaring_bitmap.puffin --- +Failed to deserialize deletion vector roaring bitmap +BAD_ARGUMENTS +--- invalid_bitmap_key.puffin --- +Invalid deletion vector bitmap key +--- dv_envelope_length_mismatch.puffin --- +does not match combined length +--- dv_envelope_bad_magic.puffin --- +Invalid deletion vector magic +--- dv_envelope_crc_mismatch.puffin --- +Deletion vector CRC mismatch +--- cardinality_mismatch_large_bitmap.puffin --- +exceeds declared cardinality +--- cardinality_exceeds_materialization_limit.puffin --- +exceeds materialization limit +BAD_ARGUMENTS +--- oversized_dv_blob.puffin --- +exceeds absolute limit +--- invalid_cardinality_non_numeric.puffin --- +property 'cardinality' must be an unsigned integer +--- invalid_cardinality_negative.puffin --- +property 'cardinality' must be an unsigned integer +--- invalid_cardinality_non_numeric_subset --- +property 'cardinality' must be an unsigned integer +--- invalid_cardinality_negative_subset --- +property 'cardinality' must be an unsigned integer +--- puffin_wrong_type --- +Unexpected type +--- puffin_unknown_column --- +Unexpected column diff --git a/tests/queries/0_stateless/04257_puffin_errors.sh b/tests/queries/0_stateless/04257_puffin_errors.sh new file mode 100755 index 000000000000..1ef976a39770 --- /dev/null +++ b/tests/queries/0_stateless/04257_puffin_errors.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# +# Deletion-vector / Puffin payload error cases. Split from the former monolithic +# 04257 suite so slow CI builds stay under the 300s timeout. Cases still run in +# a small parallel pool because clickhouse-local startup dominates runtime. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh +# shellcheck source=./data_puffin/puffin_errors_common.sh +. "$CURDIR"/data_puffin/puffin_errors_common.sh + +for f in overflow_offset_length negative_offset length_exceeds_file blob_overlaps_footer +do + launch "$id" puffin "$DATA/$f.puffin" 'Puffin blob 0: offset/length out of bounds' + id=$((id + 1)) +done + +launch "$id" puffin "$DATA/invalid_roaring_bitmap.puffin" 'Failed to deserialize deletion vector roaring bitmap' 'BAD_ARGUMENTS' +id=$((id + 1)) +launch "$id" puffin "$DATA/invalid_bitmap_key.puffin" 'Invalid deletion vector bitmap key' +id=$((id + 1)) +launch "$id" puffin "$DATA/dv_envelope_length_mismatch.puffin" 'does not match combined length' +id=$((id + 1)) +launch "$id" puffin "$DATA/dv_envelope_bad_magic.puffin" 'Invalid deletion vector magic' +id=$((id + 1)) +launch "$id" puffin "$DATA/dv_envelope_crc_mismatch.puffin" 'Deletion vector CRC mismatch' +id=$((id + 1)) +launch "$id" puffin "$DATA/cardinality_mismatch_large_bitmap.puffin" 'exceeds declared cardinality' +id=$((id + 1)) +launch "$id" puffin "$DATA/cardinality_exceeds_materialization_limit.puffin" 'exceeds materialization limit' 'BAD_ARGUMENTS' +id=$((id + 1)) + +# Oversized DV blob region is generated at runtime (sparse) to avoid committing a 2 GiB fixture. +OVERSIZE_DV_BLOB="$TMP/oversized_dv_blob.puffin" +python3 - "$OVERSIZE_DV_BLOB" <<'PY' +import json +import struct +import sys + +path = sys.argv[1] +magic = b"PFA1" +blob_length = 2 * 1024 * 1024 * 1024 + 1 +footer = { + "blobs": [ + { + "type": "deletion-vector-v1", + "fields": [], + "snapshot-id": -1, + "sequence-number": -1, + "offset": 4, + "length": blob_length, + "properties": { + "referenced-data-file": "/data/table/part-00000.parquet", + "cardinality": "1", + }, + } + ] +} +footer_json = json.dumps(footer, separators=(", ", ": ")).encode("utf-8") +flags = b"\x00\x00\x00\x00" +with open(path, "wb") as f: + f.write(magic) + f.seek(4 + blob_length - 1, 0) + f.write(b"\x00") + f.write(magic) + f.write(footer_json) + f.write(struct.pack(" "$NOT_PUFFIN" +dd if=/dev/zero bs=1024 count=1024 status=none >> "$NOT_PUFFIN" +$CLICKHOUSE_LOCAL -q " +SELECT blob_type +FROM file('$NOT_PUFFIN', PuffinMetadata) +SETTINGS input_format_allow_seeks = 0 +" 2>&1 | grep -oF 'Invalid Puffin magic (header)' diff --git a/tests/queries/0_stateless/04612_puffin_metadata_footer_errors.reference b/tests/queries/0_stateless/04612_puffin_metadata_footer_errors.reference new file mode 100644 index 000000000000..ee92652feb71 --- /dev/null +++ b/tests/queries/0_stateless/04612_puffin_metadata_footer_errors.reference @@ -0,0 +1,40 @@ +--- inflated_lz4_content_size.puffin --- +Puffin footer LZ4 content size +--- lz4_content_size_over_absolute_cap.puffin --- +absolute decompression limit +--- missing_lz4_content_size.puffin --- +Puffin footer LZ4 frame must declare content size +--- lz4_trailing_bytes.puffin --- +trailing bytes +--- incomplete_lz4_footer.puffin --- +Puffin footer LZ4 frame is incomplete +--- missing_type.puffin --- +missing required field +--- missing_blobs.puffin --- +missing required field 'blobs' +--- null_blobs.puffin --- +missing required field 'blobs' +--- null_blob_entry.puffin --- +must be an object +--- invalid_blob_entry.puffin --- +must be an object +--- missing_properties.puffin --- +missing required field +--- missing_referenced_data_file.puffin --- +missing required property +--- missing_cardinality.puffin --- +missing required property +--- invalid_properties_array.puffin --- +field 'properties' must be an object +--- null_properties.puffin --- +field 'properties' must be an object +--- invalid_file_properties_array.puffin --- +Puffin footer field 'properties' must be an object +--- null_file_properties.puffin --- +Puffin footer field 'properties' must be an object +--- invalid_file_property_number.puffin --- +Puffin footer property +--- missing_footer_leading_magic.puffin --- +Invalid Puffin footer length +--- oversized_raw_footer.puffin --- +exceeds absolute limit diff --git a/tests/queries/0_stateless/04612_puffin_metadata_footer_errors.sh b/tests/queries/0_stateless/04612_puffin_metadata_footer_errors.sh new file mode 100755 index 000000000000..ca6bac8941c0 --- /dev/null +++ b/tests/queries/0_stateless/04612_puffin_metadata_footer_errors.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# +# PuffinMetadata footer / required-field error cases. Split from 04257 so CI +# stays under the 300s timeout; cases run in a small parallel pool. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh +# shellcheck source=./data_puffin/puffin_errors_common.sh +. "$CURDIR"/data_puffin/puffin_errors_common.sh + +launch "$id" meta "$DATA/inflated_lz4_content_size.puffin" 'Puffin footer LZ4 content size' +id=$((id + 1)) +launch "$id" meta "$DATA/lz4_content_size_over_absolute_cap.puffin" 'absolute decompression limit' +id=$((id + 1)) +launch "$id" meta "$DATA/missing_lz4_content_size.puffin" 'Puffin footer LZ4 frame must declare content size' +id=$((id + 1)) +launch "$id" meta "$DATA/lz4_trailing_bytes.puffin" 'trailing bytes' +id=$((id + 1)) +launch "$id" meta "$DATA/incomplete_lz4_footer.puffin" 'Puffin footer LZ4 frame is incomplete' +id=$((id + 1)) + +# One missing blob field covers requireBlobMetadataField for all required keys. +launch "$id" meta "$DATA/missing_type.puffin" 'missing required field' +id=$((id + 1)) + +launch "$id" meta "$DATA/missing_blobs.puffin" "missing required field 'blobs'" +id=$((id + 1)) +launch "$id" meta "$DATA/null_blobs.puffin" "missing required field 'blobs'" +id=$((id + 1)) + +for f in null_blob_entry invalid_blob_entry +do + launch "$id" meta "$DATA/$f.puffin" 'must be an object' + id=$((id + 1)) +done + +for f in missing_properties missing_referenced_data_file missing_cardinality +do + launch "$id" meta_re "$DATA/$f.puffin" 'missing required (field|property)' + id=$((id + 1)) +done + +for f in invalid_properties_array null_properties +do + launch "$id" meta "$DATA/$f.puffin" "field 'properties' must be an object" + id=$((id + 1)) +done + +for f in invalid_file_properties_array null_file_properties +do + launch "$id" meta "$DATA/$f.puffin" "Puffin footer field 'properties' must be an object" + id=$((id + 1)) +done + +launch "$id" meta "$DATA/invalid_file_property_number.puffin" "Puffin footer property" +id=$((id + 1)) + +launch "$id" meta "$DATA/missing_footer_leading_magic.puffin" 'Invalid Puffin footer length' +id=$((id + 1)) + +# Oversized raw footer is generated at runtime (sparse) to avoid committing a 16 MiB fixture. +OVERSIZE_FOOTER="$TMP/oversized_raw_footer.puffin" +python3 - "$OVERSIZE_FOOTER" <<'PY' +import struct +import sys + +path = sys.argv[1] +magic = b"PFA1" +footer_length = 16 * 1024 * 1024 + 1 +flags = b"\x00\x00\x00\x00" +with open(path, "wb") as f: + f.write(magic) + f.write(magic) + f.seek(footer_length - 1, 1) + f.write(b"{") + f.write(struct.pack("= 90` +- Insert `id` 100-199, delete `id >= 150` +- Schema evolution (`label` column), insert `id` 200-249 with `label = 'new'` +- Delete `id` in (205, 210, 220), update `id = 25` to `label = 'updated'` +- Spark `rewrite_data_files` compaction + +Regenerate: + +```bash +python3 generate_iceberg_dv_fixture.py +``` + +Query paths for tests: + +```text +data_minio/dv_puffin_warehouse/default/dv_puffin_source +data_minio/dv_puffin_warehouse/default/dv_puffin_complex +``` diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-14-ba106366-1379-49df-a585-047546b341d9-00001-deletes.puffin b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-14-ba106366-1379-49df-a585-047546b341d9-00001-deletes.puffin new file mode 100644 index 0000000000000000000000000000000000000000..956042e717a548bc296df89f69e032150c44dd42 GIT binary patch literal 2141 zcmdUw&x+JA9LFmjW>=gy5FBz+nAoOG|BWoT2&>-q;z?x4pI>*vcG_-|jxNJqeGK2l zClJ|_2)@Cdd;x!55%e&7n8U#QQc9bA^UE(ELcXUb$1Dhf$KS3?eGfnW#`7o$@c#f` z2iQ)6U$1azlWdH#2f;Y_{`t$z9kL&c+)m?1r?agp2cFFMl+@6i`30FJ<&<=`?ZA2in08GpRA7gxnrh(% ztxP=>6E4~%ghn*JZCoeAm$gynuiM^1=)rjx;(8WSgwfo2;jwh*!Xd$YX&TdpdNuF* zsxr;|gRro;ov5J_eeM4lQ|+>=;p6SZ(CE4B)j9a1D2#@nj8R^eS|@o)6CRh8YgN#a zr#UTUq*IZJ3`$8OI>^&!ov>JzM{G)1Z5mM^Qw>?6UegVocXfQD0SO#w&X z00@bSo+HrWywyt30NMhyBNyZG%y|6C-*^1vh$cybK7HP*ThY0)HcOJL{k=o*`^EbY z(fMQM?d0RzFZJJK-buc``L;Wb6YP-~t?;OQzDMC~2Gk-2BLr;?-hm9(;T=UO>NSjN zP0J~(SyNQ3e32q=`0NK8uvVzWGzQ@10!9Kh%Ot>E25sMa2&iQ#8o``|A-Z{rW@l$` zK|^f2M-F;$YpKGD2abC^uEUVQPCzs_!5Tjc@_5wp^r>AukN`e-67O@d1<5)ec!;BW zPu2$Aq>Z*j?YepCdu_T`+`(`tRM7Kf3_qPoJIy1!J`@tII~dj4?cE{+ur4Igzli;tn`{5<^!I9E3f z==@qhv;Pfr)7&(m*|mT!ehYNefnTXC9YiK7EY?4NG;=w?=9z@32Pc~)vKXX}a-uf( uBkd*~HnkGG7p!OiC{&HgGFA%^Qj|@dSEZ^|6BnlsB7NT7z0}E7yM6$>E~FIz literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-20-04b90311-de4e-4d07-9633-95670a571224-00001-deletes.puffin b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-20-04b90311-de4e-4d07-9633-95670a571224-00001-deletes.puffin new file mode 100644 index 0000000000000000000000000000000000000000..ce4a05ab2431ee61dba5b610026f085560a79817 GIT binary patch literal 539 zcmY*WOK#jS40U>f9$}zW1B@!$kw0Ugi=aT!?y@feLmw&@VoQo7Pa6z!hhC}&C<2L4Krlx9X*1|KGI`F_QY72^#-de-tqpT<<0e;Bqu_ee5O{DfmWD+yA_-Q>efX< z{Fn5`B*f|X;+A|v&b*w$cNth1i!;}}+BtQ)_H2-8!c1|!xvzxq>t2{U|u zDr8oVa#gpGKa{N8jB;wAH|C%P<#p3i-UvYtpexGzx-2r4n2VmRA5NtSW`=R0o-fRqwxR$ literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-9-dcc19f77-3570-494e-b336-f4d3301291b4-00001-deletes.puffin b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-9-dcc19f77-3570-494e-b336-f4d3301291b4-00001-deletes.puffin new file mode 100644 index 0000000000000000000000000000000000000000..7df1cec9ff2a84f8a891ed255c8a40b01403463d GIT binary patch literal 1351 zcmdUv&2AGh5XapD2QEP21tNnGNHO~n?`9Lkp+e&0f+}$Xq2P~kV|BCM+TK7F^#OPU zUV=y91vnutJPxyokOERJJyfc-hw=D7#0(D=KiDIuHv9=U_g zyW}4E`0o4*nh$qt_t#oNZjsyM>&x#yP<{V@_0vt}n`9qPuaN_CgS`Fy=HLhmSy5R# z8R+!=ipH}R=#*-xAzIh4C8+2;TV`}fjfGnGvOMEsUX4m#d>GQuh&Ie!WLDED%g_O* z9W)A9(=8)Pn)vhiaxp6wKJVFD9A01teB9 z(KVr!$@AK(lUJ?_kVFVEBzT|46iGS@QA8{~grrX}TiC|BL@#G;XN+xTpM;0`I#7cV zT^+a1lv}hld|n48R?lRoP9Q$ZlC)opGXs?|6D=6ei<0p|2sQzoF<#|q%6XBN6T?#0 ztFtGq@K{wuYD#!*8&TVMhX0b&^Yh^)V6L_}msf!^zX<2iPR^st;XK)X&!elrIll;J zzLRs7B50ff|2DGnfPJW$?4SH$E7TkYuNB}lI39G^!=Nk<6Fqo@O}DUdFxFBS$ww7H i24$tOpT-iTl0`Mn%R-NJg`dbC<7MBj;{e0ePp&^TI-~Ib literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=0/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00005.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=0/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00005.parquet new file mode 100644 index 0000000000000000000000000000000000000000..fb364a125c2fa75b550e84d7a49497b2ed3e9d01 GIT binary patch literal 779 zcmZ`&&ubGw6n?Y2>K=k9bcS76N*J1u!Zv18^OGFvO?oP#Ac)}pN?l0OWU~>q1d8O~ zNvsMAVntCY9yFkW7fmDw@1A<~=FxvZ@J-eLguWv5_3r2i`KM`3f=HH-GufTuy~lgcUcP<*afDO=0kVJv0gD2-fF%JLRgu=l z*PZKja zY{H6vCIIY_LKoRLndX2E6~@yl+mRPUt#&+cea8{D#-%)l1u4oLDAr*-cT8JClcgrx%1iCXnr$vtEs@)+XMWCE&U!^bH9FSpr?QC0#e|{%YBa9z=_jA_}FVkYs0TV7qOzSw-63 zil-jLgU8~@b9?Y2_y-8yDtOt0-g*&xP+AXyZxXky6fuO4nK$3}-uLDu^VjaU2)K%K zxc6;$?}CL{+UlqS6y99@1q*$ESNre@zQ6%|g(LV0=`(y+0hKWTqTj%W`0n%jw-?W; z4F*JkvV^*X>k@8Bn3J#|VUYkeDxAlo{_*!r0~OQ;_3HpCaXR9OWJ#Wd$3G5tRar|r z>+}-QR3e#|71Pl7X(u0A4lJW$axi(8-ju}3rok4^g;gqM8WR6Hvbc^LT&A@67Q<~u z>m-%M`wGq`yKx}N?VCff=eGq8lig7;l(2bE0nE@+YkILa3Kh+=C{Yp`YJ0wDQ*1bi zkv+?7KP9tZ@K*=B<|1%fh&xX+AI~rDjA1geY*ZGb*A}`&@hJo-!v{dW{ zkHoTPd|>N#{B%{=Gj=zQdO>VEc6u}sHtm^`W4A)lZg%aN`|GxhxQ^ZNmL;Edjva}$ zUKj~FAhmD?x!Nn8C419JC6gx-Ik|s}$Ut~THgzIPru%>0gH14ISG{PhC*q0aDEC-I z-EgIobBk`J=;jK3FKjn+L2a^9^Xo0I)v8pdr-eJ^`$4VlR%_GEpfuGg*8|rrO?qQy QgTAlyg%=9IJpJas0pwQ6<^TWy literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=0/00000-20-04b90311-de4e-4d07-9633-95670a571224-00001.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=0/00000-20-04b90311-de4e-4d07-9633-95670a571224-00001.parquet new file mode 100644 index 0000000000000000000000000000000000000000..6dfde23d8eaeb51ba7d8cb37aefe5efe95801946 GIT binary patch literal 969 zcma)5&1(}u6n{HS(;zV*^bI?*hcM8PLK?G4)3hNM!9yuM1VJcR5Bo7`wrtXLvnklp zs|bQ8{{TVo;K7@>o(ooiLGS3%00_acjGtco{*zrG983uWg!3_4ViT{vy!?J^ z0sVZ6ULo{o;98+_i6ibXp=YKk=S*aRgWyaQsH_;JoeNz%UyQFz)Pz2@Zek3cL7P)n zVen%-B#UGt=UT&V?B#$M`@74bW%~-#fT=a6GtYg*0JCJKs9I|c>M6q@nK-oUTek8k zmX2dory*JeFfb+Rkj{vRj#>ChtC_&X;=I*HV?6yD`O8CGscjoXQ}+Pqu1)>_Fh6N7}O z15RS|NpiC!uTyC7cT2bn{F}yEA;aIY)#w29oaZQ81xMwf(^Wl7|Eyie@aaIwrW}UB z$O&ahj<+^ES2imp+3}R`4rTN4g^VI*N%pN?geR{g19dv`0!14g%O9%JRLno=AIZzo zm{UzS(Lw%$MJ0H1yjsGGPy0WJPcA{<8(6{VNQE1{Ab+NUp?A{Hn`N_JHuFV$MHZFK}Cd+2u2Y_vpcgjHEy<-By=gG z0;7xc)Wt3=Xwe-kDuR$K>%oXF4-o}DL=gq~AW=}%nO#?fAR2}_|KdL%L?Ipj-x2;#2&p!{0eX{P2^aZK*vnH|?*vItOHVS$R+WkTCP&nptljm0liLFU>1J;CV&E8t+e5dDWvnQ;=g_GAQ1eH@0>p|KM68N_Pz$@58c|?}Zv36#C?ReXlWqm19JPBNRM ziy<(4ise-Y?VQj?K?oBQGE8I;&4%F zz#}ds;a5%+0X*o&s^5*DJ7(L$3p31^5jB#VD^sy8Myn<~WjaWDSDQf^NGhGo#L`42 zg;a&9lZGmp#7)D}Q>0;i2eBiHOcGkFy`OqANg6vcX40UExMrmcx!BCwp4d#Xa-k+T zQDYz9KdLAMb4p$HL|y)j|D$k27KBx8T5?CmNLREbeLIb0%G{ptDV0j7Qt<_%8Pn2z zvASTWE*g$&@p!1VzTQx(qtRGhSgEb6*JD-H@tSZUDzwJ0z^n z4nBYUPBfOrSY|Rn{{5w&aLXmIMCU*O70$tVxBwR+4?0w#0ZY(?7Tkn*rIslq2mwI+ zTf&|Yw)OO2eEOe;9|$Z{EQ3*G)+6(PqS+fieDwI)^Osw%-n^T}JEtUi*PGb#=_2sfhJN7T2BJOvs8?sTw<0$67*aZhM5Fva z2nF~N?9L`A&At_^4ZU!o8)WW#!N6bbWsHJRE*P1-J@h+n#;KIb6}#HD+U>Ggt9izv kZ9A2!VODCcQ(SCct~!QMELroaju#i*-4A?1EqwC70NR(C!2kdN literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=1/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00001.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=1/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00001.parquet new file mode 100644 index 0000000000000000000000000000000000000000..43f7e9c338c66ca5c0cccdcd8aaf0090520bf278 GIT binary patch literal 766 zcmZ`%&ubGw6n?Y2>K;N#r8Dfx62d}40^7KonxEvL;6ZvSq9BOi{+P5YNt$jpf|MSM zU=LnO@gzA(LBVAe4W{VRjo@W zV%O_%ssL4Tph;nHRRWqqR|(wM%|S&m41^+wp<8({sG{dUMV2t2DVAX)iH%5JZB0c@ zl`<$COPxkh2F?uGp;b(z13Q!{W1l6u2!bM&P)rm|NInpDK=Qk!RfPgxPb29+nlStK z1c3dNX@F|82{evtWB^Va4co4P3{F_4&yD=H!#l1K(mq+{4j5`*_&M=}%W0%6E2$!v+x`5+FtWHue6ec`SI zi0Qu;uNkIi(-!pA1|sK{Z*o8r81iMKX0$<;mt`ahQ*qBQZ4YxTVGi}llAZ!|C$)hf z1!$UJGMnYt0!az_k)b1(4AO@-kFqEfM}&$%Z;xyP+vh1`8z>io7Kx}us48somaut} zS|WN5(qvF19mhCAioh_ixj4PZp71o&WZ_Y0WF(|<0+#7m>J2A{7a6`#WN@ir43dh| z<53#w3qCq49gGIp%Q%(U3Quzw@=Q(%Y$bLfxOTDCwQSS&hci{{u03Li>ts@?O^n$} zuTrh~b*rYRN_5oemX+QvRdHO~D_51?+mlL=(Nv{kj0E+#Qx)GHue-jD2Z*-SCYLu_ z8CIs$2-1BB31;_i5DIXQ!Fmp1;e7v(yE6r1-&8h@ZS~`&D=R4mW=J?SQ8UnP&qeTWizA~}Q$tX-50&)X*Zy)#%UaR9^`2D^4L$4-UE;N6B zUAZbZHU3;QvpV@p&F_2gokNq+(x~vmos%2KzB>Blk*TGWS@>|oC|DSdDD#)|yPlrD z_3P;EnvZHmX0P4)&ihxbngg6@3~1l%UhXmgr1=*dT4&I2?|k}a_AWMq92`4`ay#lj z(C*$)9$c6ZL+?%3eq2(~)2#pa(!korm_Cy;v^aY2vO1kwJ>5IdzFd6x>#h|~^B)Dh z@7DkAIy|+nZ+Q0Gms<}?-&+$6g(GA4v!3K$T4lir;;rI0?8}{nIQ<7q)U5P zNpxf4qz^Gy(iR|A!XV+Wq(V5I#gs@QdvAPqfFqqHk*+l6cR+p_BpJKWAWLf#GsU8D zYD>7A9*11$4d>KAJJr47O6rlAg##>=5lCc8c9g;084&?bXQ5m?Eg*lU5Kq~r$8jp1 z#h)5GB8}OqnB;z#!1p$Bf0uHNg{EsV# zsxM~Sh83~mr%DsXn^vnX{>1bs$+tT!rCLcO28vLyci_smaLz?P z2k`Yi5NkU0_?b>CxxY1@_m&k;*zIk3TB#N&)$+WNPCI7i8I}G(Wh5BYqtQS`Rh6Zc mMmOX@m=nT8Kmavexk!_ky&8Io&K?p^p6h)y3?pNwUlD3<~Ahmu_ z4^lm-6cqK;gE#S@77r4O{S(?_j~;sVDuQpircuOUdHeRw`+dyE&YYXI7_cqM9uW3p zZ+$1OF#`)SlL3s6lRx3=2Lj(HhvRSp#vuVGAq}Ts60%T$3Y>xSZ~-pDEL?#(a3C5- zFHwjQ0)R~Wgl!OZ?@ND&{6lS0q{cSriEvwlJ0jer7{1LGYOcK1ce6PWeO~gspQ!mI|+G7(-4Xpnrfy%W0IN#8C3$0 z#ks-1Q@%-3a&>5G=#+zTXmwac8@P3-4%KvI9r$C3asDjEra?%=BZ&hC2O=*i-zBL& zsTRP%MuJ8Eae5UQ*@SC=>PL}n~ ztwmq2=nmmi$sIz(R{kG^1_A-QwgSrZq8%=E{mw))Ox*OtcCgS&SQ#syu@Z*U4H{m; zE#~q?r&PD=^?adR_N^($af>CZP%L|HcB+25jqYid38;=xl9(VOV87cZi-NlgTC7??M2-uJ$5-kX^@a!SE~ zKQnk4;kB#pS2+>O)WKm6Aoz0fC!GC&z~?GJ5h`HA65N1gxCyu5HZ5nqNY%^Mxd35zzyWNhS zym0OMr-=JqzVY^Wqj#{e{%R6d9&uM5E}d%{vj@H(^03U|*Jz9tFZP#@O&=u__A&y` z6?QdHb0~)q2na-`a}b1(z%#uL{@X7S4tEiuXh9^K2yZ*KK(auiGFf*`)rux&0keMK z8Ou1s0QOLmHPo8whZrl06iPWnK@Nb3S-}q+Ww8wGAu_q+B3%^VG9ytVr38XQsUt6y zL90jEAu~RZ4&*+I32C|TG&mHV0YzUypQO!@Z4}roO|c@Q8B#hd zOF7Fd8dWKMenE0PiY%4Yk`qsNvSgd{b;~yC0Y#hgpIpUCIWH~B14v{DaZ34r5EWoK zu-Qf6r{`3AzHZhgO1AfsX;-aE*{g(0d&Hw-a literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00004.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00004.parquet new file mode 100644 index 0000000000000000000000000000000000000000..cb037bcb4efabf66e7d228d53eee43670eaaf802 GIT binary patch literal 1043 zcmah}&ui2`6n>LrOTeWlI70$KmO#TUrfbu>KUx;li!4>K6c+^XkYs0TV7qOzyZ)f` zZ%~SO=)q$VJgOH3>CIIX#G{CM5vzz)D5#*~n~B?2iWtJj%$xUp^S$>%rjAW}6zCeE z0ljnk`;U}G4czNg2gu$#^b^jy1U7o`1fIcbcmwa?J$!^u@EN|qCM3_EGYY7L03h)) zy+-JVtB;vhpDFLSiObe(XP>l{!hkkwaZ8Ke= z3f1}^%K$3owD={FCGsr#p!fKpG@~i zG>uEG?sxnlC}@_2LNb(U`T=iZY(U1)pJi4Yu&8DZfo=*mO*4aS5VlMly|P`iB(R^P;L!(I{Ql`Of5NB0&UMiMsqy#i6nkv^|x;RP=t;w09LstDW8elI{)MD!z zEdu+cGQ-i?y1HMq-588Q9`JZ3-wrSES>Jd{7VXZJd2UbG?M~bcJGN^lN5fIwo)~fM zM#P)-wmorf(H0TUwOjtI$j6;)$9$n1#T*Zi&E39MbFMXGFS!Y2bc-Tp_iqv!h<4D% zwrFy`|L2`w0%LOCj~BYUGdvq-F7mh?&9yRK-Yew2Og897&3Yy*jTTBlx#2e&h2r=) n_r`)CES0@tX}lhej5Q9IL(dx-^>bzgzp&(m>k7aW{tkWvOij}0 literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-23-292ac3e8-88a4-482a-9a88-e08ad8bdd2e6-0-00001.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-23-292ac3e8-88a4-482a-9a88-e08ad8bdd2e6-0-00001.parquet new file mode 100644 index 0000000000000000000000000000000000000000..690316c0bced6e4729a5e503715c5549ab0386c0 GIT binary patch literal 1197 zcma)6U1$_n6h3ozc6K9C8k&1MBh$E!&9IAWy0dY!o3Ka>t!&e@##B+X(ag@yuA`g3 z-HpX0VwRR*rKqiKS||zW4?g6jLb3iLF%g?qv3-!@LnPrg<-dzfpKLj|YNzk5!&mCZzd3vT*L~mK-oNIzFXPJW7y9a%(KC(1 zjbCj}ojku)b>&`KQw|Q%_&~e%q{(s|0G)cufj>j|(*1j>B`A0vJ`&1b9C%|;BVF3~ zmbFKBW{r3A(rNDI^7q~rsyE#ZI){6Y*8Kgouc>vcrTNumEywqLJo|e8kw5>EHcW&c z@_X(&`bXZp`oj*-kFU(O`8RA{8+E<_zKM-(&xyQ{pZny+$(g`jbZ=bynV-rVqa`OC z-`79y32%Q5F^oK3fmQtP^uq_GQH(HO6e$AW0DunjodOdP2k@!i|9ONM5q`0pZ2$t@ zxe^E`0yO|q8InYIB@MENx(^B-U?dTcNK8^8R@lY)BoR%1w>06PlO>T^mJ8ZNL0T*m zH$H|4>noEXgr}G3hs)0>viaB$7JZ)luvaCcRvJ%@6;8N=H2|cG*LWq}T z+Q<$$(|X)Y)0kk#C`MU|svu)t3W4CJD6Ir8B{A}dC&!TLjxMlTpi00y&vhY6q0y^A zAwkqP#~sLJ7bfJ0fSZ5~DV-Z)1GC5c;viE<9YWC>@KS+9&bO!y=`D%~X=i#!^^gwwjn_;)7JJu^3xOLDLx94eWIkHs5g-NE-aHl7mfhA zSwdo}n+yo$Onq^MWted@ld8%Zd(C!T{E6#RatFIirB2D_GCf94@hgSYIxC^n1^i0V zGSi8yQn#y5VG+%*bm;ADKVA8ijM?2|WlWk#>glZMFE&eecBGU;{z6S~zJ{IQQ>rKg zYf)YGe4TyJ&rsNU2*k!NJ=5J|=GL`mJO|87*6Qr=XjNLMO7nQ*Jyts5F~Y%6I37vr t$z-Uew${|D<8dP#(Q3lA2_sOQTpuwsEfCawQk34c!V4n^pqc)t9s`uDBd7oX literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00004.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00004.parquet new file mode 100644 index 0000000000000000000000000000000000000000..56882346e595e3067aaa68451a744b5903dab4f8 GIT binary patch literal 806 zcmZ`&&ui2`6n>MWHPEG{)EN>8vIKY8Vj4HwZubX!XhB2^V$m%Wv}iWT)DD*arhJ2t$y7B%FdQIUHu;C(g#af-OoM=%IyWIVk$`6R?!EQL zPd1)CfBAZI>&@Gp_UhVg9&fh>VM5AF?Ys(X4-Z$x3!iF8BnT#fjy zUQ8Q$)%OCop%pWr^`hs2K`FJw)&|7Nyp@$cU_Kd??!Q#TazG8GBnIi~q<^Mq_ zz~`{J6;P(?cDU5^qLEq{zv+bye{mshnO4@c;;Bm058Swu%VcwveATX2v*U$=XN^`W iPA+ea=L)Wq9<7e$9m`5*>|xcw!SUIB!B1!!|Bzp-n5bX? literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00002.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00002.parquet new file mode 100644 index 0000000000000000000000000000000000000000..f9661fe1b8f3dc6051661f59daaa1111584c3f3b GIT binary patch literal 800 zcmZ{jT}TvB6vxlKJIV+m1>M_B46$5rS!>*8v~^WB(1)a?K%y)N!TGqW1M8}@yH+?# z^-%OGLeNt|4`mW4q(VfdB_ZK!PklT{g}(Y&Y@bGzCvm;}hp z2$@9W>%G~}RT?owLMAdm{Kbi1aAgaD8{v?HB86&PZK!X46eWptp^5}~3M>4eatx@@scux+w{={kz7p>z|(#r710|7b!x zb|wIHNg_w-&j>MqZd6bsCex|Np>W?yjpuNi`+bSL)6WMiWfG0D!i^!$T3No}7o7sr zS$V16b6KmQvy8{HZl1MX9c5w0)LG6N2>sY^^AeiIb`|6MP8^M@S`K#&wGQpsF^U62{RhEi(b}^I;nv&8Ad~>llrhlg*B48Vy#Rs*8gQ&+do#g}TJu_zjbYzTf}= literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00002.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00002.parquet new file mode 100644 index 0000000000000000000000000000000000000000..c9baf49eb9c946293095888191bed65921c0f8e1 GIT binary patch literal 1002 zcmah|O=#3W6n>MWC5L5E=o=CUvIH8oC0(1Y`^zGTpi)FyQ4mBS$BI@K z!GIW0lu(jTlW<j)QJWHD-90KVy8&(Hn>D9QMc|f6(M%lzh^Cks)YIq)h|g(r}EwP;nMy zg-lu@F_qMa`37?atWkzMn>SeTNQxwbY!~>hhG+O!6$`^88re+SNV2r}6Xt9y&LhQ2 zZZQG!C~q`bgXz-fG;)>gub}JI?X4kIYUbSV!bM215DEYK=?O3ez zqgc=Zsf9bt)n4i>*z0a8nK_Wi$^BbI2BIUf;(;ug?*Dan*1?!v_T!bl=uIugx%(pS zMoXQXSMW*&FP9JcQM;K7D>J1^P;L3GR;gU839lFgVWsMoE45}gU2L7JhMqS)<4>3k P`o7W^9xDKI^q2nu>F&sB literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-21-55307a4a-f583-48b6-983a-c2993f261fec-0-00001.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-21-55307a4a-f583-48b6-983a-c2993f261fec-0-00001.parquet new file mode 100644 index 0000000000000000000000000000000000000000..4a5ea28ead0cff6cee3f80ddb56c71f42cda3ca8 GIT binary patch literal 1148 zcma)6T}TvB6h8B_(?XkNcrP{>efn6%|Gj1eWwOJ3Cub>#DQ6R%Bp3 z1!ZaY5J3e|LO%~-ND;lHB@|*k6b1QEQBO%k81)i%@2snW5DmlGbI-ZoIp25Au+1BG z>Il$d4h=Er$;|cdoQ%RGARsS5V0hghXi2l6Y+NSYJ5q8C9oTwNxpJ}9%8zuF`%7P# zH*YjCGr@eYxAEG9n!VMeT-f}nS>!XryQkhb&-Sk>15Py0>0|Te(ozP1G#FvgCkCAz zx_yRS)KTN$^wZ^XUu;xtf8Y0L&E4R1|KTf*MKNzR zodJe2;ZX@v(XYwh)4n)j5nogt1>gW6V;wEzLH2S%>`Xh&ZqNAjDm_%C`vZx*olg4ANKGh`h^CBGDin^zEWI|7 kFe6bt9El~(>e^IY)YSFr8lysrlW(o?!c`Wanf&kk1t0AZ0RR91 literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00002.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00002.parquet new file mode 100644 index 0000000000000000000000000000000000000000..82ba5ea76d3794b3f1e8b936bca6642e3d813d41 GIT binary patch literal 772 zcmZ`%&1(}u6n`^W>ssO`onaRiSr!vg*u-p_Zz5Jtr63eh5Jc!^cT$&pwApL~DS>+R zpa(w=B2usyp;Qlgm*Tnqgr5gb;zQUFWzd$UD@{+DzBB`AMikBNz55RyJ$bhI{KYuhQ58D>mP{$e z@_hc|5iHYBQe9K0u~aT}_k}R3bI`eoJ1!z^U8B1@f8`CXvMs_T(%{>K?MYwfMZl`Y z!lB4*b_7#^DJjrs;9sS{lISYF8@VYcDTaYizo?LV6E z;_nFnzpJp*{9S_Ofj?HbnO6BXIwS9`9Sc0s6yZv4;N28mS9?g-t?2feuuDM}J$UfYU*HEgldVZX9F}L_edm4N=b4Y4y>Q9GfWJz( zkMP>hFQ1MWnDT^*Re;p?+23&WGlDa61AMpzE$Bc9cVH2Ea2M{ueRv3~@B-H16}*Nm zcmvz81JS^jC6EyU;N5;8yo2!bZy%SB{nNfL!3SC5oZSaQeAHiFdGhqx>e~9tjkg2N z2Svc7fGGhd1YiL~z^S8(bp9QhRE!%3uXM=s`f*9u)CMmQgNGwul1@M;ls{1sQglt) z9pJ-xgQ$3bh+rCI58-_g>!b#}s(NuEw4Ely3ScDxn&f*|B)~BqDxMwM3CJsk!Kqxs z&}kA3tmp|)xr*=M7@_2`Oa>^aR>!8sP6?z&R>xUxgKr~OhiW>q4rEioglx&U20|hh zNemqfMK&d}kCJ<+T7VQ^o`#e|O-Suv2Y|d+@M-c(##tbP2||-9*^p0)b!$_;%N!Oo zG9C9OYuVZg>Y3r7Igf%cHGKX z`HU4$IbFZ$#ob~qUvx@!yI#*1%4KHFIF4H^S%qTRbF(w`b0ycZvN=1gQ+{yb?7s6W IG|T^uKV)>MN&o-= literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00003.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00003.parquet new file mode 100644 index 0000000000000000000000000000000000000000..4d8d2c59eee2d2bf3da40564de2e5cd68abc29d3 GIT binary patch literal 1028 zcmah|O=#3m5T2K0C6%Qp>^u?(vIH8oknT3E`_m;J3JOv@6j3UQha{V=f$g@6KM zWqfz%`R4%xbF^1c1t>fz{enwb2HRWk2HwIByoV3)5kA2e_zvkaa6tl@F#sYp@D^@8 z__}-akj7mGlmaCIRRMJYrvyw2I4$6efGGjf0%izM!9ld}$IkAztcwyVeN#06nR43U z4Ur}C41D?Y=e>+5tsJy}jM8$Hq#u`bP2HlMcxc)0041G+&U5spNvx!6Z01l{reeA# z@UJ6-tGLRz&@}iO!ws=cv<$u?<5aS9fjH;RX1bo=3h@Z}9Rf{?8W$zNEG?CW*Y$>= ztQZCfC812q^TQU!jwCViXXvg41}gdxsJdWdmgo33$rt=y)N15|>R7qz*P33lS*}b>gwD9{2i2NW fsZKP4(edWVTHrXNW8Sdt(ifJ#a9^TdjsE}Nu!qfx literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-22-cbc4c1d2-88dd-4d8a-a31a-6db198e852d4-0-00001.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-22-cbc4c1d2-88dd-4d8a-a31a-6db198e852d4-0-00001.parquet new file mode 100644 index 0000000000000000000000000000000000000000..9fcb8cee529f660beb1a0695fc96e23ebfe95a93 GIT binary patch literal 1176 zcma)6Z)h837=P})%QbefEVb{;T{&yH(OZgXnro9ZDN#0&Aa%0NDGbI&FUh65E$!0e z(lWPUP+?<=Xw@2!O}2p_vJWk*;)Eh?$2#dI^^^2Nl_FHOFQY=IVZ}f1U6!_NFg%Xu zectEq@A*B)_3nOM#ek1&r zpMRgu{5UpqcJy%LvGnX@Brw!EQv3GIuIce_Z~by~a`8%jpX&bOr-_d;6Fuj9F7ECB zOqu<6Q4uGswtHm()zH1%jL;(L%P#k=EjRn%a2VMlniaQ~?QdVi0&I zMQJ3li^Rx1j+{e^GrBp|=BNZb8?$a&DKxrw3JIdE4`&DRX$>aiR|nn$Ruk95CN?m; z%{)skA!QUrL*StV2~7KzGQ_Ns2&7bln@EYr9o0gdg|;$|UFj`SVw8eI`I?h#K5an) z9*bLB2hmG6%29k2q;*6|sLD5xA8N2fmUtWna`h<^Q=H^y2VJcw33Wz7kL%ffUoLS( zAJBva=A>+%$mnvnoHMiegeiOFimS~?%H4jioHF!uGADN*JSkfi)hiEb16Dk}dF8A= zlsB?Eb)>X(PWM)erH>Ex%cs4SoQ{ngYllzBq7aNNd7T@1_M-ns;-S+Z^klT`P+m9N z2C~h=dNyYqA8b~AYS5=Pd*XQ`oor5Ybp*TO;gpt21w)aDu6D-biLS63>WU;2{?1e& YoKRJNht?`a>9OV literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00003.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00003.parquet new file mode 100644 index 0000000000000000000000000000000000000000..50b6c3ed85e6464c819efd6129ded6d9fe2fca95 GIT binary patch literal 766 zcmZ`%TWb?R6h1RqWho&dbcS77$}*6U!Zv0Tb0tW{2c^DLQSgG|W-oOimv*x$NGJ&X z10sDYf(Uu=ArDe06n&`V7x?H85G_9V4;0U2ZL5gGa`v1v-*?V8XJ+-vEt>&*Mp?{u zzi#hk3}#}iG8G`dfAI(0jR|-(1M|S(0tm3743}XUuEI6A0XN|u+=oYytaG<1qzD0o zvID}N6Sn{H(@W)_lAj5TDZ-#H%}|<6X+{*ybo<55tJmWRhjl#fYHZJmlb zRmvfMBK0(jGH_zp9a@Wtbl^uR#`$;3RzS8$CoN_QW=*~kJ|(#cX;mSQ$J0pvk0QMO zdji0Jrq}{MNUp&X5X}c8Av3lmdQMm3F|tJDZw$l8#|>or)^V*)zqcl~tb|fJp~zAG zpM(NJNqe{j+RC~UZ4CW*sT-M_el!UCJ<~4Om4a>N+rzNqnQpCAskQ5Y69koNqv6}- kcH6Dh?P{&zxy5pDsqWf#vE(f37G9jZyCZx;tN8MN0nc`u;s5{u literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/0c03abfe-66a8-43ab-a765-a9e8072b59ed-m0.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/0c03abfe-66a8-43ab-a765-a9e8072b59ed-m0.avro new file mode 100644 index 0000000000000000000000000000000000000000..50388b8a2ebb6cfce380f516f8023dddb1a5cdd1 GIT binary patch literal 8221 zcmcIpdyExV83$G?lopMm#kiW|Sha2L);lx%?!tlsEfMLK-8Lq$Ue29+?(QseXKrWa zvUj^Bn1<9xYB2?*NnL_Aqz};I);6_K(?}abTPY>czWl>Sqo$Bh7PbaN;&;wG&bf2% z%-+4M$%cfPGvDL)J7RK9Fhuz8J0ODZBKdMDB(9b?Bay4 zzyzIsH@N*%c$;nA>{;@BZ)l@UG|Q%XeH#kc)bqq~GsJZ5F?cRm>mPAk?3NsO<-4hy z)JJyCbCFsZoHcRL8MfRE^UeEUlVp|*13qCc4W^_V$>WmYO5OCiC`lG20WQSUNX9G` zvcwM6*Dp;Kq~&#kWSlD&*(!*nk;qWFp=q%o+e!eKmj*sqnq|5Gd-%QNA)B)_NRF?0 zFCAp%kvmKf)zppB%cIvp}S+jvp_2gldOu?GeC;i z9p>8o+=2Ko1J+v_W(w}yxDJi#?l4;0k61oYk9qjzbXq`#$<14K0SkUbB(DWmkaq@A zD7q2N)fBR9jsAm**oN3wCYhWTlLJn;%=^&#chl~5TmJqf-P(A3@Vn-Daa(vDCOXyLi7ee*vz2~ctbLFqLK8g z3=8TU0;tFoz}mecIoA}iHO-i7iqcT7F{GGq$U(0w#Ali;wqz5Va}Ji;31F#R4P={I zRRl@drgk+57M&Sry+U3ghFaJjy$ViDHdSmMhBZp;OcpSQAueb*SJ2axcA94$4M3-~ z#}{gi&a8AagdOu2F@F<5QrZM-O2y*bf<(KS+DJE3#6}S*#u^~0jkPo8Z#^K1*$!PK zQy?z(XZXm_;Tad+H)~AI9xJi|48B9bcPMLE*~GZ0VmES%Fb`&|eBL2${n;DbkT6*m zU_kZ9N0ncO{Zy=rvoObT+41d26-S$3NCk>83yZwbxYD3@ZqS-w^3|*Y3|YDZ_c6P) z(7M%BOM-1;8atK^XxTZ)jy?o)o`p;@N}nft*D4}!lRU90r^;oj$xG1DD2t^OGEBs^aM_L_ zhA$wRLq@<0Zm1>(H`11iy)l@Mp)mAf7h$;LeU<{Vs&7LwV&cH+(DMP|^$1 z&ALY5njG@C1n;+olQ`r+9h(f;Z3MJGSP}4T21ww`3|ct!wy4~HI2FqR{%OJ@MCB^0 z$b4M_T0;ShsaCH9>XK5qj|sK_{8Q2SYEW79goTjGjq=k1a-g=bxnB{4VM+6FGHCqe zK#Uy`N}seW;2QYqi)jeg1oi1yTj$z<>*HYxvp*KunZpF6KTIBRdBCl&Ok9@qbFd}g z{_yON><73BV&(`pRJJK$6QBPIba(?`cYuWVmaCv>g=?HT89w)+EA)ZMXoc z9~4Nrx$z;HdWTNQJjZLX>{)uIK*h&-2E}=$o>g(;&Lcz;t&7reZQ_+5kcSHP7N<2bp#8sC9JiFu4ON6K&c+I4W=q_- zW}c6MX1bCWIa=KY%$BPG`+~pbaJ*#X>f!nDQ?Gyfh=ZbQW7DBSFO;G8)6=H5L8w;8 zD%l$GhL*@LYp|VK;ybmFa`t(r&i-*=@4I~?>o@KkxmIa1ufF`@Kb&0t^(J}QN8aDO z@Zf>ldRKkt_z%{NzwxKfG_U^9t)HCOwR!j3kA7vey=!{+Li5258*ct>-{S1Vi+65r zcF#9;ez&}Dh}=9X1&4wnc2`R(b| z{T=q}Tkxj;zIEt769-zBl)iBGxxe2({iD8D_MCXG_{7eZcmDC`!*9L$%a_}qEpK1+ z&RqxZFYKGT`q1Q)M+Vp5eebU)mmT?*+w#njUravptEZp6cGX{pKPQz(4t%+3_3i~X zz1XyJ-V5c27QgWLPunKW>`%{*5>7Ke_ayukJX$=BX9(iN161 z-m&7?(ckxUO`Q2s&vj3~q84FJ5)e8$D|V)=!;&aO{W2_kCjj<b>20_0fMjM;E;|eR}HQC(1AP_k8-v9j~2OX$ncIWIRd%MT( zo?Qn*lh6bKif}+E6n=m}B5FyBApB`4B|iX(ZxF32R9Zk$DT1mX_`weZ;+>h@nYq2& z_4(pTR;29Cz8}x~G4J!r%2_qaeLU3Gyf zc8dj& zt3~7o7X1g+aR9MzPBOiuCI_5|nD@f^Z0z+?d-NTF1(6xNBAg3(Z5 zPGPqp%QW#)aY7$r z;dK28$HMMb-p*^d4vjQH;~HG2SaT1NW24B2KZ+0ohZdp}ZU>99yIW}}Z4(dh!ay|) zP#L9epc+n_C2@^2PJzncncp2$ET4;zNtWBN;i5wH20+-e(MG&6A3N1Z`c;_)bpZi1 zbP8bmUXh<`hT59u%r(R4D%TiNOgQ8q?+V$OrmHR4!`^~}Wex#Y=5QC;X0{YTUbdOT zU4lht23oI>SBRk|+9R*v++;J==HdF&Bq&z^a~R^1hI0}<%^0M4wx$c{jKS8Z(NM?8KjQQIMNNTo27b(|BgyR`L zHFR*}#C>zQX9Y8LHh{r*DESUmT)&bV7gOy8*;fFfl1~LgqUX&@Tiq zrrNp^=txTQJ|@`$$WK-0TR~;fQx-y|G%8LD$bt647Jfw_!cvmqWYEORfebrRls<1+ zAT`L<7tu_x#^~o@a?9W7Y?l1xA7s;b852W?YiOZ6H0k#CvADR8B{eUz< z&Aj9q=lh)eeSuUWlaW_QYE#gOs?5!S)G{QXOzjM${3bL-cql6`YA_P0LC!~AxW}XDsM*IL;OuW%cPDp`G z5p6^8qv$8N!?=91C+zH(tt`#!G7OFH5FV>$ekY~M;H0Qr zme3gSSZYUdW@tx%ZvZHwXLge&MDs zR;+q#?LCj(ebKI|e~;gH&c)xZ|L>Cj9q;{g^3}6G>Kk4%uw>woT|JHEk8c<@{ZB@B z{`S(nkG_e%^T_YN ze$Cqb*5)IBzUu02??3$6^Upc^pSga=)77D0^$slmTi^F@7oUFp$>VoDIQuMF+jHt! zy|3KA*gpS9zgYLX#pmuTd^L1nY~TH_|MBPtXRe+8%_Wz7_eHq=;NPEk_vMEQg*yu0 z+;Q*i->+Z()O&~DJoM?N#~-zO&RDwtz>T{r!~4H} zar4o)9=veTxeuJT=j_=ddoCEpp BrX2tP literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/321000ac-b08e-430f-9c79-e40d5e4aa42d-m0.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/321000ac-b08e-430f-9c79-e40d5e4aa42d-m0.avro new file mode 100644 index 0000000000000000000000000000000000000000..0cce769e50f047c4764f93674602e98fe0811c84 GIT binary patch literal 8145 zcmcIpdyL$~87Co1iV_h7FUnfv$y8F_MYv% z>xG;INIWD!YLvFL=^qHBAkwBS;*UN+0t6)mnyLsSD%3%j-E=(a zBRj`kq*f+pJzNi~eq=J=_y=Q}*Kl3q{4L8z8=yO$4NtFb+5YHl}*Qk|A zAk=p*%}}IO9G95kd5i2*MAAr@RIV^B6XZZo0OM&8gJt<%1h9+mHIA(9TaX+#@s}a; zD=4ZGL^X||;ufm5fJ$5He&|Gw?;)GGBqCt}>07`*>^jkG@+{HH!c;O;>lq+*995)#QK^5%XSHpN)N~*dBeyHp>lblIck$Dvd*Ko)*F9@wo)4 z+Y)OK4GxMLb(a)4R2Y?UNNfagqCpgPqp(h_6pXq8BQI0a@*8S?umVc7?qqdA(}N!l zk;9eH5Tr!6mV&dNn2u+bXCZY2h`d4*1$+DmyDaZe2*;`TSBRk|+9R*v++;J<=HdD?BrwZ>ISg@0!#R(hrVr3O8|(r)eIUC~+jM4q zuq*7Czliyp3zFWiSktK2mlq`3&5UlknXWd9R57*z$>^?~F@HM&NzHcXB4&+5IG*8C zLkBlb+&9}iE10db0SvxF$#o5yump?vXxwbjfHY`ZGWlA54TdZyg!`CXT4?*tR7;L+ zY8r=r0B8j^ZqVbiieewwogF{H{CGT)JaJ|TcH z)z+0jM^c*iG07G{eyTd(3Mz}9vJf()QE^&84zw4x{3`+xmZA(NgC<@MWZ03S^m)qy zsX?y3mBF$_fytFzR4VAq2M|OVW*Z-LNV(taAM(x0cjE6O>OA!61^}Wkutn6Aav*N#iC?Y_@(y? zzf!z(i9YEAnLg=&UE)*O@i6;YfIaAyN*1jgl}1YQW!a|~qm`Z7V<;ulz4Erd^CY-7;s?-T;*DN%LJDk( zXd8kj&67Ep;&M`0`yjTp4iZsX$D8;f#^sYeVP~IgWocfQVQ74Z@K`nTJ1JELCq?D5 zgvN-+Qahq_Qk#0^C*+}ty=7@l4QT%_7AGy0b3+SZ%d)WvnWYjpsafV@qM5GbRgTuN z0ZZj-z`o#bqZlt4x%%k}_^CI(ebhlQ3a}T_p%==K|MWE0KMvIz`i;OMd}vAivIg6! zi({vN+OZ${F8JEO4ts~aZ{wNGKbl{C^6j6m{^zPw2cO^hQt^`GW52xd^785aUr%4U z=~ruR|KaA_H?RKw!zUMx?Ys5RZC`)k!lyo6<*qAy{Ks{dU3l#U#`+gd+_ho*+RZ1% zpWOeuQ~x;g=5I#5?|t&6C%*HyL*s8edG6uYZ@BKpJM0h6o#>f+=8?np?;m}n=jcrz z6}I$Fxv%f)8Qb*5C#t=B-+1*0wti@Lqvs1p-n#n1zrMd=&HT1|_xW3M`MkN$n+ z+E1p}@3*b_xo7`%#p(C9p4@QxzyEAM;B8u%I(~!ojmgKZs&9ShuKRc0bN%}(&MZ7M zb>-8qp{M>Z`25~irjERJ@ahlyUpY1Z@##-LJ2!vyxzz^_+ea48l7&@kuiJFG`j>^i X!(Vy%Pv(8=&9gsyYx|kCX$b!Zz!tjv literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/321000ac-b08e-430f-9c79-e40d5e4aa42d-m1.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/321000ac-b08e-430f-9c79-e40d5e4aa42d-m1.avro new file mode 100644 index 0000000000000000000000000000000000000000..c9facfde4c040fac3af8044e19b6889af039d5e6 GIT binary patch literal 8132 zcmcIp4Tv1o9rw;s^ei!cOHJW922r~wJ2RWh-O@BkZB8xid6>jh!!dn3`*v@<*_qAE z>|HM(+O(#EC=#V2O7LiDrM8Gf6K#Uu)Ng`{AP8Emii%J_3M~Xm`+x7vy!Uo?XKrtE zg(HxeH~){{|Kt6C|GlPi{PXKv7pr) z;#NbgynUBOpXde6!wq^K{#VcfvZLh$1YXpz8-1w355FI zrJ0Jfn&T2HJYkU~MI?=cMdb?9GC>ZE1TdZkF<7?mMF6|_UgOACeH)VF3jQ)geho#_ z1W`?8R9r{Z7LdNK>4#3__#Ubfmqa8iAY&c)hg~O{Po5=OS(tQFwVnae#L=|W9_J3k zhZ*o=-^NVA)@!dub51mk#y2CDPt;=>e!W;!P+@Xwe$c>*Un$As;0oer5QU$9=fi`~(8s#c|C&oDiyM5S@a&C??IJhdl5YBI3~ z(cqw{)pSXLLxoWVhop)iPBe(ZZWK0&or2L&VB}?L*?vpS4^}{l)(KV@G(Gs?5II~4 z4M9qDXDK*4iRE}!Wgb#TfXFK}QLx32u*>oeg>amTPmZh6SF_6Y-B!c1=y54EsaO#r z90ZuQ8YdzRC!XP`ZPhbFsEmxB1Yas>&SJME%QW#)aY7$u;dK4F zV`Fy*ve-t4G4lP6{oB@lnyIXNIZ4(dh@<25PP#ML3 zpc+e?C2@^2PJzncncp8&ET0RINw(Xn!bOGX4S=v$MH6@;A3N1Z`jy3kx{LrCIt8$M zugK3eLv2lq=9*#jm1_(sCLD5*cZKXs)76&jVQ<;NGDiU{bF7bSGdqePFWbzqKEa|h z1FcucE5uL}?U7e-ZnBwb^Kkt+5?B?$9EP~0;haEEGe&5ht?dIkVB(uME#{BICBsJTii&za3;dq8m4ISJ# zao=q6>|nmh1~B*zCEuZn>sNB)VyfLJY{EQf`L$X|qTaJNxglY)Y`}mTkB_Fj3^!A; z5w5@-CuL{1BU2r1k|7l+!V)avqj9@IBhsKv$>eMM4H&YV5bk4kX`!osNVVkHrlxV| z2Y^;kg4J<^;gD0)MWn20@Xyiil2@a81H-P(nuz z9EMIE<~$#HWR5=1_pYQOACMXeD5t7go#Z9yXq3fL3MmqC94^~=#P9_~tH@1w!6lu< z;8NOhF&~5J7z#r#ju3`BK4vK}tHw6uBc=|lF-?lI7`{stPCI5>V9e$@sm{Y_ZW>%? zGk7cx0j&*KO=RarseQI(Y>f@KoujjWuhSZrcy(Y;Y*0@E0CzZ(23Gh${#8@+fmy?P zx}3-8FmX1c`nF#s_6l&u#SvM>>DBNo2|^%=gQg~^$1tTHP|}0NRgO*InyklLf-g&A z5{Dee_eHaryFu9V8N(H_70D>sP?Bas~DYra6o(F+q2DZ>jBLU-a`EJ{{|U&b2YSBjS| z(I;ah(P{H(w=q!hBO+ zV2}?gm&-6;69(`eoJ;DY8hLM}`CEn_(H&hUq9>lKJK_W`)coSd5xO}oF>@3MvmgaRChQ@aYk5xOrlTu}HQdBNWXpDF) zwI@m^wW(KrLLS=KTb9<;fcF1lane#bH*^rTEF0U9St@apnq@vFn(0bj%`k9Qt;7#fG~FC!QS`9r*3)^_NUOc;PEE_4i)e zb^Mh1n|AHK@2S7t{N(KS?%chy{LsZ$-BFm@`cC@Pod?9I>5z2H~( z9DJ+%*&`cf{&?5KAI{o*-)#@w|Co1ZgzWt81*?W{A3Efo_V4;V+wMN{%+Y=O5B_Dt zFJ1|)Pgn0P{^%e=*#^3nh^Vd(^7CrpZ$Jf2_**_X*ANbjK%DXpx^7|`}Jw1>{ F?SG;LvHAc2 literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/39ec1ceb-2dbe-4ab7-ab4e-0d18bed2d2da-m0.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/39ec1ceb-2dbe-4ab7-ab4e-0d18bed2d2da-m0.avro new file mode 100644 index 0000000000000000000000000000000000000000..2f84bfe01b743ad7f90816345f78231c4dd7fcf5 GIT binary patch literal 8230 zcmcIpe~cVe9Y1P8C3sc^dO(jp$6qw=!OqOy^=_p|fu1zT(c06f zd)MRAXpr`bhGIx#nu3K0XR$>RsJR$uY6Zjy{z@?gi50*@!rnv z%w%G!i)~H#994WLpUU^U}ZvOS4QDU=P2SJY)-&2FY=~ z_tHUD5xGMIQB7HtUPaZWki4pFIl8M`CMpnvxWq{zWfk~`4c#3Jo&{Q2m}FJ7o&i$E z?hx1R=MKb&8Sqt0!%V@~zqS^Q>h2Kg-GEp=QIC1}<#bv=g~=^ib_ol9MI`rvE66*8 zC=^*lb2Wu5TciJ=GPWW1)k!9&#pHk!F7rOLJ|25HT_1hDkjq!JVWuaNs5B0Vd71{F z2et%AtqZI{G&snulns*dpu#AR9a2CLCmO`iZse4S7J*R_VD!o~r&$#-KUe_;TKiaC z(Dd*QJ4pAG&=5pK*Or2QJIU#0E4l0(^d&qBNCj3@IiYa?tAv@tG!zE!o88f`g@Y16XQL6WONL z6hTt9sXa}CMP~+DuaH-Wp%%7BuYwbkO%mM3VmB`8Hn1(*8m+4#q)46$ zmrj;&O~P?dLPrklIQk&Wc@{FsD1DynT}DLSCPiXXPF2X&l9!;PQ5H)nWSEG(aM_L_ zhA$vmKt{j|uBasjD`?Bb-WW{BP#Ai#i!j{rK1+dF)wdxTF>zq^X_BAC@Li;Esxj+_ zG26>YaUMpaL*P1_!F_QEXsyj^B0e`V)w3;QYizi!9i2n?YOOJdISA~D4eCJv;0|Zf zzzPekB(Lt#Pl=PzX zGF>BZO%D27g4s+si9-%Fu*rbkMnL<66#?I7fCRqGpoK$ki|YM{Q?)GMpC&9qRIajy z%r_(;6AEBVwRR=Ykd*3uOt1yupNh`cg36*NEQC~Ul%E!m1NDV1{E8q9OPYt1LE|q6 zV(f@e`lMw6*T7d_OhdRPXiP_MgKGn>kB2GD{#az^4ik|6FnPq~0k^(7aaqzYz?Ojf z!?QoKAK)g4nHOB+-aaRJU%-{{Wb`T|vMFdpRqW<~Yv~T5jO+}!{3Wteq* zP$1O%yH&jRq|8A;7f9L6i3W*fsg$kiN zdoSiBtD#>?XXsaim(J5Cr7PAa1+WWz%IhX(KMSx29bCzrl_SzfZay#j2xH`u8!8zk zzGPIPWSTc$BzU3uD!;%WA4Dz}W4;s`;Pv1{QpeTsdn?M{V)TUF(a;^bH?QVNaBcVx zpvCwbz2F2FSQXJa1dp4?b1=f?xUl*`tZN-4BDcP=?-5Pa8W2pjsWP zVr#@3S|Y!!!FHvLkD+y7{Wf_GLFLUb=cs`^R27dENCde*WDnr=Dv4bL);HzkOx>>vukR zSND6%7f-ZGt;fIX{O)f{_V#`7lhQGtIu^VnR)qVckC|ho!GWmeOHTCa+mBtldi4CZ z<@OazuW4D-vTb7WqVy$;Pb*KMu$e{s)|)YX%VKc2cknf}e8^?&`* zi4_lCcmH6|@i+f)>phzo6`;*U>n=$n1`-~Yhlk6oDAdG*-Nd)_I0=JH!kKBJGn zHM8q04{V+I?B;c+Rv3R-w|e=#d$<2^(;oNP>E~bDH8tbz7<>M}?7u#`XY2dlDjwN0 Z^WRqww4M4@;f>u>Guh|f`A{@8{{z{c>aYL+ literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/41caec64-2cad-46f5-a966-dd75e3635abc-m0.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/41caec64-2cad-46f5-a966-dd75e3635abc-m0.avro new file mode 100644 index 0000000000000000000000000000000000000000..3f7df28d60e9e941028adf7999c41026606465e0 GIT binary patch literal 8135 zcmcIpS&S4#7+%&FV-$D*JlJT_i3H=U&UEj>?kdr!U_?a}G%Av|rn_dh*`Ds9yJy!G zHyU+~2jLAQCivnN6z}G9oA$+!c5hF zeD(cR|M$-|<;`>EpB!2vq>3LsVXGER)QC2ug;7wqBCVkDZ+abD8!8pFibGsG)P^>0 z)#6V)T&3&be+4ZdTk1|gU}*)rAt@xrT;ChkCJTIzUeMw_HjZ%4E(pO<_8DTIi_64q zW8^T(@Mkev1U;;i@(u|jc5svC1d==y4coZDA zj$u=z=_PADTnk71$Yh}K4I4GD?$V$c7Mlv1O)9wVM%oblT(wkEEdij!vxw={t7Q@h z_w8FV6l)d7C1!ZsCi@hV^axFASA>=ca-b)GasI?$S-uwm>|(#hk+ppbSam7?GDLm_ zMI!`JPeZ79Un;gn-?7bd-5Obp(jKLK6oY{0O^@cPNDA)O@mBJ$;o`mhaZ9o=KNWt4YO* zBf>#|=~d%IqzVNgK2P6GqovXcbF5SvNU~|Do>4+=WZX%xQ$ceKyLDNnNtQ|yr2%%F zu0QNp*xkV2fhw+PlavOm9InPSiZ#y=IW~%X_@f9haA+Yq;V49ueY+J0(lPM^o*k$L z0V=)N2~>mWuq3WA;}ocL&iu}xVtg(DlPtGx!wZM#3xKd^qqTT#K6Yx5beG8vbru2C zOBBG?cSSzeboFYQmTS7+5!V<}%s6n6-wN4GE2&qqhrL+`%NPK#jKL1rW;7K+9@~t; z4#A>51Fcu!6=JA~>ycM*F4+wA@^Jle5}0Mc9J;^c!a0sRO*^yS%w+Tv!` zS9F9O%NMbHb3xMkm1yd<+U$ZvZ!@El+e}w4ic~SS0Lkd=I%D~^1Cq*ixJAq=iE!Nf zrt~ddxt-v2|=|M;&?joX{B|MX` z9GuXW1BanA41J!DJTgu{&(E$;MLr-E5>QQLo6X`SxzVWnrP@;vaWy=)6Nupph}y_a zbiuM_VQ>z;axrg%X&VYnFOCp~H$GM=(5uEVQ^| z<^$8u?sVCY(Pm-+llqo#6KgIcCsE22b6SovB$9pJd?xm zl;Gz+(TM{GnnobShS>lefxst;>`z7nats0%)oqRqq(6B~g8iAuPJc`w`UQE^;em|4k+_WXvoIx)`AGJs z<^wVWm3hfC&IN4#yg)jUWaJf+niRC-Dsyrmy$lE_Q!@i;zX45gJXDnzG-!!zRY1z$q7BB_$(cq^4X!_0K3_^iyJIFC28CQU5TFn|{EE7L;$&M*rDh?J3q0Y`WISxhTd zMP7Qp$Sb8wXPA?|JToU9uuF0(J04~~3$PD*rITqZN0pH>d|CD>#V9SeRWf>Z$rwt> zbRNDc@I?5Ae87MYs+P+LUlRfFJ2+R=NjLJ{O6#`_KjL92HT$a%o@z`oxl1_S4pZr8TG^n>Muc-#@|59->QrR~&3AU^l8<3f)aFd=T9}~}X zB(G|;wgZ@{Rs;0~e?6kTWa4Vq9QchmetpzNF$%C3(xw;6kpJ|wv40a(Yv|Vli}0o; z^~)Mer{=~ zzh&t~uP>RJy8qUv4~<>2%l1Be;V;8@tn85L;d}n<-uc${MK?TRFJ64jmcPFFYTn#K zpMH7di@!hk<&Tfg?7BnSv+urFcRu&i&7XdMZ13m$e}5yq@71%dQ{Jxkjp(=7@2|PC z|MYiv?A1r3XD)dD=t~E8JmI~(b#Pt(Ll3Swy!r2^&fWLqp2K+2Yd73<^ugW();s$b zEIMiFp$ivYJHNQ9^x};V-223lW#z-0cc1v$n!AhJ4*c5F^ILG*@S*^wjLO+tWSi?%8#X zi;>_F6ctqPfOvsoLJ|@YQR9V*F$e|`1pT2X1`!nTg2eMxRd-cQ&vftX>U&l1dvkT^)&)z>46HFy#k&@6)5575(T21j^y^lr+#LtD0K z(I+mh()IAaoaU3Qb=xPfw1S<0BRQ>1D!A^1+7SI*wNy|o0ieXSi0Rg=CE^SB z?OQVxYZcoeW-w=yJ&H+sgeJ8sLQ4hN*JHpqeDC-0{5LZRQBC=>?bY#OR(oKPDncM|MW(44?dU6yH_rIJKpfE}mf zjoKD=Ht~0$ifh_5r2#94t8k5C%`-%H8HFDFQG^&cv=HrJ93o1;-SPv;nD_zD4^)Ey zm7ebes=;Jf64!`v3RF60erHfIKIec*mQydo3y0_nfUsLeMO@6rP7RXoGTEWdBY=8= z0@(Vl$mW`^UQIJ{P1if(8bgX12M+REA)RRj^-6ZJJMUl_0|1sW*a6#&rXt8b2VZf<$jKqm$cAS1*c0F}487=&<~nkr4o>^efGvLBy^TlXi%f}(U6bf zMrt<1CFtX%?ey!&P+Ob0Nd<{8i->q@+!#=w3}`s+d@ZjEO_m+Nd(0j!v|%mvk`bFq zld*rU+rdB`P`^z-cOdQ{|nQXxLoRAsYSyu>#emA_Pb3L>t9$94)ad;!ria$;R@ zezP#RfL^(nx52axg{BvW2*Vp6sTAl{qZqOgQybPuCPgfUeTl+p#B39^*_@MVKa3{F zAa&M*N9GX7TA$TKIyZV6dt0W~SaaLlIt%!kqj88k3hIe9>hS@<8_ujj6dp_s)KIg5 z>1B7i?8j&`v5ZN5%PSLWF(l*QkgVkLs(XU?0f@vdLle@Yn@SHT>8|`r+amBxjz&|0 zpL;|n4jgD2f#e%z19SuepTx319udef2w0F)5L!6&wV=CTloIc17#7H_850ya327Gd zZ5ikh-kDPE&`RSVC!PYx0Mz80L1m|>Op**4R74BlKxIhl7#_BH}@=N6ssaH zy;tOw(517?N$*R|NeArWoJzKf+0O#(gKp_$#>!D;qzqq{eL^ux%Waj6o?bGBQZk)~ zuL?X7z9Anl;Df5=Qo`3n0Q?Tl6m`;#e7BPNEya(x9UVKMbMr=>1kXnF0bDWBL@!<; z4K|!;3xOxYlN?NFIcco*A-0STh$y4um-r09Ws5x#XOFzflDaM>(C7`}XVs|hq*Zx1 z2`QImG(tSG+Lok~-qa^Q77q>TEzN79LHoZ{oQzcV4NZbAEyf09W-HvJXUWIdGabpR z8m;XBW~b^yI(~|gQ z4W?6zqohFEu^;+Q+I_HIdhEL`uN*nKfA>X?-9OTGY~cqFy;*!OzwGWEFD^cGWUqJD z>X+^--2U@ckbW3=lH6L?N96+{ra*KT@M^Oef9b)k1jgzwjH~E|LglZ9$53! z(_fi4KYj7r#pmvOv+&lBE50ed`t10$`OC{+Y<}Zc_qX>K9{>9IkL!1=`sBdRmz}!% zqit?@bokpFKi?YM@Z0mR?CbvN_e;NgZS3$52NwT(X!rPuC(k{meYoLa^vwBZzp(Yp zdluc5e|*Q0PnSLVvHzL=&?9T#S+?t}-gExR-T6mxzuWiJ{Z}kM@V9ni>7N6qJZKCL zUcdT`@xO+b-eQxz`iixhvE;h(Ys#f1dlp@L+Ri5qZa1E~>5tn74xQn=yYKA|}){keRCg z`0D$s{_mev#XDx6caCq4lM3EHYnv8KREajI`GHq611+QRZ+fhfHkiw3WsBG)UmM)G zO$$G9aD}di|7A3f+*z|c0!z!-_DLo(X1mUqwmrl5=ou~CQ^EmG*##k3%05HvvvHBw zZEW1M{UW@@bC>WGUb#*r=MmHOXux4073^4L;s*ud1l|OEuA~_*@dF&xeE2HF$8sn* zY#qZUOVf+iI=JeOyFr10!Z&QxoSIF8W>{>>XeCm{H9OD->F27YoN5UGC5}l7POVZT zo^ap3HAAshwro=HPupZ(F-ecGK<$dq5<&L#2r$l{5G>Pm0)SoY*Eq5z*92DG#J}{B zTSmb+LDbVA$}XdBGbp#L>iSk-xeh83n*_wqAblC+hixmEh(3$Fata%2^bC+H4#uVT zkaTc9EPz{F6Eg=J)~-NxD;P(^>k#7;%~&3PE~^rN*)6+X1uJ>QEDu8}h_68uia|uV znnA8-(tl7DdvNxRB+F%0a=?m!WiO&n#y*#AjlNYX6l>;Y<|jr}dJd^P%|gzbwns>< zj-o;Ia8OXI+9boF!l;OSQbKS}^bked$gdJJ2BWT!ifYZa$JmrJWr z#fl@so`>mGV+EuF1tC6<-%Xoyxuu0Ixm8Cd)L+Qb{7$ z&yLe}$1D@u>-jrS!BuTLr2#945nQEM^9+$yLV*i^6d?u1uC60zcZ*9pEJND)2@}^g+ufOK-ejvQ9PQCof;(FRbYoY zjR5L73SjHIBAsixdNobTHC^wBYYZu795~2tg=D7X)GOJ+&a{JN^aEJNKnH9ynu;Ke zZN@-{V9}m|)+_J|G1SEM$SXLNY=(My*lwM8g(6@M-CuO!oW`A|_feiL?EpHxFS$@# z+|2sYj<93-B9?C|NP4dlO|4p;UXbW*W^{6!>FPxhE5;Te8J%5cEZ=rOQrQl-NTEUk z95(;4rh^A3-kTxE^d_pT0fT)g+J}m^TTDF{L%ogsD)fT|w_Nr~&^~*k7ZN(l3N)x; z`)J6=a2+)p;3D*K(suH7WT>r8)TDw$m_kInHEs;3PX;s;b-t!sfhNoH;XP)L7Fw~K zdP#{*rLpgNfRZ;UK2lxe)!Vpid7!pDx8Bl&kVxD`Ks!r# zCSf@^p)CjYeQON*JQq2nPCrl2F0Uf*kuve9rYaSh#Y=RfQTa=?ry$}mJhl^v;R}eC zkR9oQ#m&N?n_jt?x52axg{Bt=2*Vp6subu|!x+*LQybP$CWS19eX+u6#B4pZ*_@MV zKaA?*kUHzZLvsjZt;cF2nH%}W-j=B~*4#F?&H}#XXl&w)fqG(%dUOErhBIprg$q*y zHPm!qdfA;W`!U)~%wtmDbW6mX3(43xAPcy>>e)g(A4Fo8p$X~HL!}3lbXRtPWfFKM z$HFPWFL}|40|%N$AlZi503CtACz0%rMg(#U0v6;Hgcc5cE$A*7r9`_Lh6QqK$^=DD zLYl>VTL$vNJ5#D1T4@~QL{k76fSPn?5g)P9zz5g~TQW?YK&u97r$y0?OFTK-zCWQydRfC!3Yr1vG}qyu(QPDRVX>}LV?L63AYY2~OgQidL!b+i6{{?o5d zuKa%C$*sTq`RgAqPwe>Y-yd(7G4s?JxBk_mFB<%$K0N=&XV-l9&w=`!eM9ECuh#n) z=XZVo_Suh|d&ygej%n_OBU`V!%=_t?m9UWR4-Szohhi|<0>kIXdzI@33=8YF#t8QIz?{7PRL&lpJ*w%|DzwMk?Cjgy;bv!+ znb|w`$ngguqOFQeBeca>XbA>LgzJR_-3H$UFn z*`2w)?U`JfboagQkI(nV`+mOr?aZBvmM^u{JT54|{M0>yQ!ZkmORycQq&h-EaKCBO zvO-rXA>=fyXKkTt`#pm9iJ=t8dH7#Ku<%_a&BAarr|33L_`-D47#1cH?j129cz3dj zqr}_-6EyCg6x`F54A$$oI5^Rwj9TUz*M;BS7&vEP)wGDgo*@Or&~n&zWNbKA89wLS z^p4n$;*@Oo%EL!92src{#U@GI^UoSe(axKWOhIukY!{4@PK>5l3?_su&M76`5xU6d zyrdK_2>`{0ie;lz$Y3i_U%xbola|wTEZY|?GR=u3A(V;Sz_dt^Ezt+e%^wde)ifM{ zJ-9Ep$YxCyvZ}}Z(ne+uIeCnTrY@B1CTbHX)m=1g&CyH)WwDMOY$uS|4gM**=9K-< ze64N@OT6_IkfP$`nRYMfAU@Q9ep6Mbg70qWLt~ngN4*1x<`eOlg+G<#6M)LinN~sJ z{0d3#1y_)J1`#N_5Xsd9GA)(-gNlj;v9CMnsSe+pis~7`L`FnU zf}0$gql#W)W$Js$Ig#q1;ndAxO;z-*?mJLWioyg*16mHZDn){|>mkyz$T8uMAVk3- zg{avhU{Q2;OLl~9as#|DP;~-SVzLpaI>Tl$T)m9rpc388ZwxA$&k4vRRWD`Xg+t^8 zK#Gw?o0ZM+*zrb^TQUvm0s<(e2!OSFMSQM_d~2FB*F>?QT%$-);gExFSBTEE6yK5! z#aM8#qz(W}>TDp}q^cr_%QmUAL9ocoKkXHBVCTv(7uHM*Xb^BtF}ri^v7+DBkcN8aiC#g7;>lp<3l49l+oo^6#OHZf0WRBJthGF2X!0 zo4K5go%*xae<5MAEWm*3jgJz03)6!aOW;N8?I^+L=Kc z{mEA~3ovA9HoV95(L#OSBwAu@AlRopR{jG-{}Dh^WMjrUjz%&MLZ@rdyQ zt4EW(EQWib!l}e;D~#E0PV)0G8q0(0bO!gtA)vJut%>N|NLS9bl&#U>wt939;H$Pq z$Hp+QCpxJ60f0B0N&_oQ*c$Mr#sjmK_H;Inkzt~lQhn9TVzmjJ(G>@`y7a1T6k9fs z#Kn>jP>(i3J)oo)CtEcY!!tSTZ3!-g`R2%)&%wG=&f^Y!1S>&1=$~o?A&1j z(jO!bzdT^pS0*k^`UTh$Fn?I~hxP-^1U~cpXWS*Q@%sX%ge9X}A)!q{J*px%2TV&x z0A*-rz~on;8H9(h@`L~*kvo;7*3tivH7wnf(239c3Us&wV3&`C`z=#J&3|>4gM|6yb#gp}Tl5<|M0uU*g)puMjVtr%z&gq)#GX=lhh=42AwIKtE_> zO6IH_o<=hBS=omeBa>WL$%xS)a+j zhjH;_FR(MsT3MLaMHuS6A#SWH`5lwWf)k>0Q9?b$J*jn3I;M?(@_q79!QP^@h6c3% z7mH(-vbmv(utnKefy{h~8`I44(br5?@;pbY+kp9UHDF)x*BXqMlw9py1iuk`uMa;c zI+kMCWax!5bbor3_U1v-;i7c0V!l z%nhwyzvB33-keEodSyfNWf$Hkw>Dijdi2C&@`hD+Oh5isQs?J*6dr?Pu}{#Z+`mr>OIfxAM89;oOtlc-tIHIuRYLSIRE)07xuoE z|H=3@ll{3bmEN5k%&ffnb1&Zh+{#0^{oFUw`IAqbd;6UOQ#-etGiUqKx9D%I{`b#& zUf2O_{rQuDSG7%S(^0*!{yJ&n{Xrbji>=PslGlyL=`!n*Z7r zFP5Lbap|J_r%wKU*^f`369&F247_@D=%YmUzwW(x!~45$Yr%K5Yabmew`8W)FZ=k) z+{?$B_n*6JpQh%frpx)3C1=O8Us-keroO2)e_J*A=PxW9yDIm+Kjt5Lf77he`sI(?%RxGOLsl+k;lw|@4|n-0t2$o&r>T?L!~ literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/5b66c606-c296-4fe4-a5f8-d8be592f6b96-m2.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/5b66c606-c296-4fe4-a5f8-d8be592f6b96-m2.avro new file mode 100644 index 0000000000000000000000000000000000000000..97a79ed3f6d9c64cf7b4d06301d69181ff4a8f9c GIT binary patch literal 8298 zcmcIpe~c8>9ao_UcwA_KtACuA*J-&Nx|p4ryW?&fJScFHsEB~ImRvS(XW!n9H#@U8 zGkcfQUfQ0ewNwS6RvV4Ut@V$ZnApU&2qleC($LcHd-LPH zo!yz+Tdv84gx&YPKR(|d@B8`g{kcbOT-xMlL!^jbn)jIGj+cm(ksQ}9Yp#@%yl+

q2tvs}y(J5*o4G*ysR&<&Dxu3KcMAd-eKOXY^9#e!@r0bpMK_+V+4=>qKG zeaS;MZ)uQK8@w+aWEGG*L=e@KLFu(rZ3@Y2OO~U%x@Dp~F^Ef?6jIiLf7sC7@!+#S ztCzy6Xgvd@gxw*o-A_7*4>RCiOT$dTgWEQvG2IM|$>o)4R>%A!LPHP{U0n+Lqa>@F+1xm!P6(n`p@D)OmWvIRcgTU`RD5t; zjJ}vvnq`!WW|ki3QiF;OLxgP`(^jLqqzDBOeje>j!?L_4J1WaPfj1S^Gfb$An4SbT z1vE#oQRZbDcqup`_popp)}XFoV~6(+6mdzKplQI$;XYiVSbH8KJ&#-q{wP8W99oFF zGYl5Rceiv;)Fv;$a|2Z`K&7M`fvPuZ7RS}kH~}if%lyWmV)>kcOwx>U9$q*^UjT&7 zJlcY{Bx5HUNpEFYQ0EXpMWz7O?iI1K-9C?dsJ10=Pv zcEr91E*vquYU-bl41*e0g2W7&X~orhemMPP!tkPQUe;Vew6a29kVs^}EEaY46% zZQ0gXO${MM@>IBVvV><6j)M|9a$v{N2Vu^$kV(eq=gHo6ipblfKy1pX^4V(g5_B|5 ze<}A26R{s2+i}G31w`}62zWt9H8EI4TQ2s-U^<4v(2HG!;f?oM3e2j$4ata!1FKJy z{49ohk;19OYzK_lUQUYhFd7>I*Vzp2i$g$bZB`TUxzSlU+cLJshTH1VIfSp;8iSaF zz@FHk9s~g1a3&3`uwZK-nwktu7whSK9;3s=az^zvD^Ij$aK^wcY4zyU>QQ1lKoW0K zrI328rqlyU`lfWNt`T@92mLL<<<4*tha9L{fut*T19Sxfp9Hi&SP}3w2uR>t5L!6& zwV=0Pl@i>lSQhZB2^$o?38^OYbqVMUwKJyLpp?o*POt^w28hmAgUX^OY?4%Nl%E!m z1GReN=`4Lxx?_D(0K34aoNi+FvjF>GHCHlY z<%l$ro6pNW!Wg;ax=KcgFBw%Rna0f*30`Qv${#Su2a(Igm@kC}cs)3g)NwWZ-HP(J z7(HQkG<1jV%`15lJRAN8&|>_JUT}g7tcYk0g2&C{IT+z`Tv+WO*0c^1kz40A@fnOu zCVQcsoxGJrd0mX5{u|=Os*>MvsXRClDi4;Ab!PHSX9`+u=G zZYiG|st8-0jTOkumbh`vJRbwibR{oxw7LzLEms5f1%IvKc*)4s^Ebk8#Qy6e4vMag zO@|J>P=?-5PdmE?pjsWPY-_|DS|Y!!!FH^H3Mc=%e&`Q`(6H}&Sf+_nC| z;-4)zbFT0H6YE+Ry#I+UZ(KNX?y2;z4?pyimFb>ar$^)mrl0P<`PeN>c1dF^MsI8S z&Gfv-r{&j=_w8Twi|?NP<<$NW`&Vmocl>YX25kSBZai&NU3kBt+13(C{IBMY@tCt6=99@UPX z%E(9CULJkpqYv8j-H*3Sz3W^#U@dv=z4EDbts8$!-Z|F)gUb&t(T{I)ckR3V<*r-q zc=Ytajw9bbwC#&uecr_U@{ z`egn}$36dTzu7syVRGO6$usA6cMdFV+IZL1Go`EVpZ(SkZ+&sgm+m>(vuCT^K7aA_ zlh;1@RQblTuYBff&#(A;^R9nOf8H?l-1)mZo?G{YiMA>G-i~LlHD}zFCl4)s^W@c6 zw{|bLH{G>6^Kt8)&g?HZ~+Lf8_hWyRH1U_Rk&J`=5vUqEY%Eh~w|w literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/5b66c606-c296-4fe4-a5f8-d8be592f6b96-m3.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/5b66c606-c296-4fe4-a5f8-d8be592f6b96-m3.avro new file mode 100644 index 0000000000000000000000000000000000000000..e96733a33279ffda09104a288feb1bd9755ad7ff GIT binary patch literal 8297 zcmcIpdyG_9880;jQLNS!vPBY3FA|r{Y%_E3>@vH}Vq0Wg1eV8cO$uGdGk4DHT=ver z<9)Ch>59ZZT5Gj2m{{vZs;s!NP2(m?Qd>zW)!GI$#TwCQLTc%QXiE$UE&u3u&V8IS zcjn%m9oA$+!pu3}d&O4l1B8RE%om^DEXflxI)jv|5B1gzT#>Yfukj?+awhT(@kSsno48lm>>R@;P2CI58n|kgO()Ml@h3|FT{Gc3kZQjwIft2^LUvSk4rYDl9 zG!BV*ng*Z8rUImP1=b)M9OSrFouoXdFe+f16cNOU1~If7*;S%OV3dXA^2#)?ny#21 ztbhWo!>lf7dU%Izq?bx?3^w^f;FqRBRX`Y+0DL8qFaUD2VX$Xm6UxWH#mxWHPzHo4V?mAXG+7PlB5Q zng_7%@-hv)6r9N9SU7caTvM?=>b(OMT$QG18nANMfvXg2&qJgYkz>LiMTmhz3sJKt zz@qr>md-_O@&dd#Q1t;+a=IO;`l4oWT>Xp_ppw1JZx1S#&nd_xRd6#g@nnn0t2c)J}Ue%9HC+zT!1-_%Z_hHia6Q?Ln=^&1z6;b#`OmEa)Y)6ldo!4 zV93&Jc#qkmg|>c-$04=Ksx!jDv1al!92u9#6OsjAfbR??Ciru)RS-`d| zbF!(1kRo|196DLTGYQ8*2^~4GZENE&=b6YLll1ds@3JED7AX;na;jp!k-P*QjnZGr zJ;OvCgvYjq7`}jL5$ORh=xihg8)(bL-WW{BP#Ai#gD|}DK1+dF)wdxTF>zq^X_BAC za4%9g^_Y#qnC<1HI1i)AGPurWa96`foN(nFg>iN^LdO86Sp#|ubM@o-VDy@*dc2?dewb^ST>Nv8x<*} z9^I6BKuO=2UZbf5p2=~4OK>?GPU4UQ4J(jz-EM%cK;V;r_6I8hz6Jpad<#MghrSl{ z7OYZ&TXo9Gzp=MNK*{xEsO zsbRQ4&6{7E&Nxe z3H@E57YZa&j28-o?)tr$m#l_<$vvT85nj4LpXA7m)ugx$nhnk2qn|L`69s!%~$vX2KgXzxft`M&;YLoCz3j@hQC`;{uZMr z?2fu-)4h2;Pl9K|{{UKyztIa$aDjCZZ9?$4c{~RrT#gHC9>k{BK_YVNye2-6ami#a zv@^?FS(Mks80xWTuI*WJeB0Ps^ZuRNhCVX0 z^z6C0FPy%sW7*|5ax+g19erZ+Thre^p-n7ZcFU>VTi&X^JXR_H?Zr2LvT5_$=U@Ku zv2Q%|(6)Q`-~H6Lau4V3y!7UU3tzVW{JT&7Z1|ar?;g53RlWL8l3>iD z{nsD4vb_3_imV@5HM#HP`0FcQI zj%UfUkG*@_mv%k**KYObFYg=6?)l=n!>9i{eV_3s`~2bZi@iN(p5LE7IaBCaJ-h3! z$DbbA`}0e)Gb`Ph-4nOGv-g>YosplCJ-4qs{==ayyKA*m>;8Rt?*m_(s@dv6_oG7} zobJ2f)ZB`rWW)V$;7yNzx^pYqTO81`FARM5y+g`#zga$c&o^f;-?M7{J95`{GCI_8 z`0szw-@o$oiU(ilTxK5`boB47{NRq>OTQi5sb$X{m%G+KBEN#JeSYe;lgroL)OBR= zPIX=9e~A9tqu<_it~@&Z%FwosyDwgO>FtY$j$J!HG(G;{+^eqs+D#j>T|3C=_Kvf& iudn?WdB1DL=r4-M6}`SdK^Y;6w=7Y8DeRHf?h4HVCuDz_NukDCbXsh_vYiho!!~n zTdv84gx&Z4AHVnezxV(BcUg49*r}5ZF(D;|7sqYp%=VPT)o_NXr$v(saMm}aa*V49 z1-Q5@DKUeqS-+XHKT(AwUJw5ZaJsZHE$b32jSGq)1sr3FrZ#gO0c#H*;Ospy!4&fB zf)He8U)bHJ2vJEXU}IB9iO{NRbFCEKyw6Qemqbm+0o#Ej1yznqh8dAmQ*VdQnKbPs zhAEh71HQ8HkyQ*1Sx2x5;`E%gs*o}gni(OWSR2-JYFfcT6D*nnTuh1!X~pDf@aL?h z5Ninll~hrRsOe->(p~q3TMIMR;<6$|jGH!D#+bxM7{PYk(0oDGc?U3yKQ>sRrkVh| zyPva=jcFpVYLWG&fwVX>6B5FnYEW=Kb{jyU`6Oj0CMG6IO75BUj-Y_>a} zIbJOahuP=}ASuC2Q13SB;Cx5`^_nOU2df`mirQo|ftIX9giknQdiio+%yfIk^JUUQS15|vl7^tefVNqOd#xYRw7W0dPitsrAOcIrJ3|=^d zUjQViG1MqD=Esf=67PzTLmfo``49#$_g#^nYdm{34a+s2FN$jfDPkNrXx$3_ObfAB zvMQ*f4pz7dzzSCv!M1Q#5#(cAxVlKNaL<726?laRYVP%DRd8Oih1ttP(b^8>dMHi7IXJ;BhhP|TGxT{HQl&QhdH(FmSmbpnF6mfP z#Uk0_#ktY2{KeYSCE^lzY}*mR7Z8mh#nA=#WDA3P@s%rBZ7^;_q3IP&B)}VQs}$%} z?HKYS#x|_BOtM)F`#gn{iPvn@`t{V-}vK)nL zb#~#)jz*ExW>8P0QFjgi-f&_KqR?P!z=oP1m~wKb(|!y$6Vr**7qyrqPJv_;!IWlN z@~X5|(hU%aw}d%YdX!@60VRD)aHcFu@Ju$_Q-V*++)f-gkTn7cX3Pfg2n0THWWO^a zpkolQK&K$MaPVt^cflyd*_AOYpj-1yQ0OEiTg(?^pv?77lxmSyG6y-%6o3Z6CZ7!| zIXz~Q6sAGhv;Yp|Ha7Yr0{2*g^f(DL_He)_4v(bIH!PqY=+P4o?wFu3A4>`x8&H4r zm|XVzB0Kysf#`S1!wwH<^qIsZq#uPT0nLZ9-!mVeA+XGIo^eaS=ARc(CzOmLCByrdOqeN|;=^YJo*RCcK48EHR?GQ>&$$6u zcW_=&N8QkO%d6jf{J6KHA{%&ao~e`I*|0x=E5@GaIV-5aj1$cv@Mw6HgB~qMjpaVX zoY4UhX>`^lK1^`=#hx2y8NJH9y3Qw1`wg+qDpTK4tMqU@QqIq)jks;KAW28Pu}{7u z9x~LMpI1+V_Wx3GG*a3(WC=FE7&DL=sc@s7DIXorcqGqiw1NW|sa6B^1%ET$_L7LJ z?y>M2vHkk6jiRXws)3tcC`0R~r}gDcP_2fR)TN@l#>c%<*EEiLP>PJH$?Bi`Elz;l1CT)gSn z>}USHY2Jk6-~asMlgh>g*U#TSe%GsYL+4|$%XPb6JyZ3Q{-NW-OS?Aj{!Y0XzYs|8 z862D)I&kgQ3-$4M;=;vC{O%vD;2ZD$*@AD@8!fv(yHfvR_r6c|R;(F3`Ip}E^4lA> z4?dbc*YFSYt4n=nhL*oSXZ*P_iMm%NJ~igf+2iis-}&IgvHf3rc*ok74ON?#tN+rL zeY|vIk88Mp37UFO8xD3Qr$VD^S)WUR=xQ0$@v%Nj2mCuS+jrgZEDq_t>+u= z?;kw;(0>o_{_;21SHHI~(c676@Ih(l7bW7A)6f31`yrL zKdwv8SoELXR|cw2>{xgB>FY1w=pDGUb3mB&x^k+dc3#QF=z>n+%4>J+er(aS>E$KH zlXYm}`Fp;7@eS>4Yv@E**AL&>qFw#^lgAF8sv7KkKf1l8@7g!|j!b(iaq164$GRWc z@x)`3f4=S7)csP)+PD6^@W8&-g)g6Yd}6#Oq0KOAK0n$&cioqFao?KLH)ZIV#?sc7 z($?gx{yX|U+&NHtbkDLKwRaqUcf;W|`}RgpKi4aN)+OIsKj~5Pr}MU68+f{N+MSn_ sLmzy%ZN z?w*Ykxhg{W_#5R%0thf)K?H3IirOC$rF^_IvpX}l zce_4cT*-=*yP5anndhB(pZB(v_FlI1iohBpRs6`p16nv$BU(WVLceZ>T2ABNth38n zLCgh_D~ue_qEB30rR(8;In5{g>b6f{X$3n0$;HMT&mGmKb9|4H)1p0P9O8^! z5Q44jGsQj!mx$BG#y!*D#}mHy6Q07aH;UwZVtGCdI0~eSUAsbput?m{pMuYoG@~U! zh{JjSUq$%XE(M3JW7y=dvTwwIasEVLS)LmL>|(#h zku7@`uxbgp~9$y15!qCPV^8(+$g9KD*>aSkc*dT(emmlKUe|9 zUWZv-P&WmZpF@0z_V+v4fpnh#kf|6u@$7K3=Y#zRD`g zbLv&MNS8~iam9)w!oH8`Rbz*w3I!oPPu@-Ay53uy(DlAJn}+HcC)7sDodi1-G$*iA zmt`7fsU)HIvEy{SQQN}KPW}#5aZQ`1G+^a$JFZczd4|X?qtJstiVy>b7NQ-DLqzGf zTfQ$D6F=bjfvO*%GV+~3)t?MY;uucleKW*8lDjUmO10|)u7kj^w+y^>w*&O2CUAAn`{ zcfdBYsR**zX7+ao7VQ~my#lWgLrq+dyn-{yW~!Hm<4qF3SOUzU`-?A}OSsdF4U}iS z9YAMnNH5eDH?z^(5q2zJ#PZDq$yl#MQ?J$L7bJR{nVsBbhI&yXim?SqW@pzK%eNhn zRJOw{Qmm2?N6mkt>EOYM_h!(w{HYpiz+fMW_o0&Gl`_x8RBxl82K`{st5gCKw$I-9 zg@n$s3Jq%1KAQ3|+(FHTxCDKiw4Ht(nQChjH>n^I<`5BYjT-~nAOjkVJ73GILX%|& z@E)^A3vJmg-)yB-yJpHzrXHC4ITEMDRpjmlrDJp~bm z;IW-T3|~OBjGR~(tZfzs*V8K(^EQ~aq0sc=5Mg-ZBb5TZY7|2@Vrs)0$)t$IurEnjS?jZ!NasdRV{gmU8f$KwTW0}Zb2JWdM?pQYMm;_N zc*B`Bh{A)Zff{NyFzeZ!F8eXsOe|wk-}1`DS^~*9I3(R%UagxTegGnIfvE}Uv5ra) zDCq_HZrdX8OpZoVg3o$HCk`BF8iC{+W&?Bt0-waPKOPatF$h?YQxIA>^tGV7V3ZQ? zY8V#Ctr-&(ISFYN^KBXE5#E_n?a)f&ASa#z$N<#jn?Yr#r%aMe8B|0I;6Q6*^FJaG z$C8)F$)Jgb11WJNBz@MfKzfiz&pe1RL3=)i+8i55fAW|F`%{sf{g^=X3-YML0~viI zaT)37VM-wLk?c>*2V@8;^WtZm3)t*=fpj9t$SWif}Ir=@U>UW(LxJ1DfJ^ zs4CBC&=T>kMDJz)p<`G!DPbL7^cDEv4PYTgf`2PrP_)Dtr%i<~bLba2FuPKVN(b%b z0EVc-Y>|Tjl$)O%vZc33N|zjOrLyOknE@4_mKhZ1rDoQoiA5SF&?0(eTFBoyW?=%6 zQnE1N=q^2rS;eZz%UCb+O6bx#=45P0&B*}l;+#shi`mZt?1OdE$*h&5%19Z$Ec=9F zl$P5n86&-9Or>Nx4__5{B79RmV891e%cX>`i2(Q=oGI$08~JV}^;?P`aXUJ8KqBfA9S~7Q$1m|&g3A_rBF-Lpl_hmu zN}$mj!q2Kv-$|?Ta1v53&1i&pWVJ0xC%vgpek>ju)LWX@M1%JKQgJd;**7!^wzL=< zkeRD+lb$6XW6yLXuWGcm1DLB;1N8-e-J-o@;_Ad@@EdXT`lyX!=wmmaO)r!o|LJLD z{T`^+z^nTf;Y~~8mo=D9Es2r>X~%x(>pJ_8sITk0@7;Iw;Tvj4ez9O* z*U)=taqxpvzr1kq;!pQ4-T&WvcfUGtaiZ{>V=LY|JovqXpWJrw>Epe- zMt=Y0>0ADBDt}x4ug1}XGk0|@>i%eE-}2WNk3WCa+Rck^-TK+y6Zq4WU)=P;zYg4d z{O9NY`T4gCKfCgcbKYZbfBieXuw(kwuF0#un)&=MtB!2>_)q?+f#ue+r?20ZKeBeo z1H;Qc&`&PxUis3Q|Gdz>{IMUsvwGl*cb_`!J-O!Y3orj&A9&)^KVMyaa?L##rrzIl J_QoVu-vC$Bxe)*W literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/b68e784a-6b9c-4894-827d-c56978599ecd-m0.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/b68e784a-6b9c-4894-827d-c56978599ecd-m0.avro new file mode 100644 index 0000000000000000000000000000000000000000..2dbdc71cc3cd474bd49c68c44ff4c06be986f429 GIT binary patch literal 8301 zcmcIpdyHIF8Sj?TLTNTjQ#XQ3J-JH2I7?>ko$Wr_wL;xZDza_53o5(o?3p`fW-fc@ z-r0Na>~>q15d5R98wnUNgvz5e#X_~IQDa-wV%nlu358{tF}wdbdg9slI>U})sbS7`%Sx+ zk$Ms_DXS4ZV@o~5+a>Q40~hFd_+L!2$V5rA2pr8~-6k<#m~I+5X(s00kzzW%-dWSj;;@!vxsV1)L_q$0yeZPv7Hn#9BUdr7u@uY z*ba6|HhksbqZt$&c8*~ar|$V@4P3PIrjufzxEF>cqoh-#85W~4DMPZjq&rd%{alok z5G4Vi#864fC>7Gg3e?vxO%bGJHJzmFD;Aj)MA8tZsNBG`NRTbr2h7bM4=mL*9DqHz zFS*EOOckrDBere#!#ZSXxf^inFh)bojAmfA-Na)!@A~7`=9w* z-4s?t>lq+L?Buz2FX zAFP0UtpltsXnMGZZKSzMXb3{0D@#G!MpBxQN>4-T1R%N<>MIyB9jvpwLpB_z;{D@d z^u?@FO}$hwQuH{N>Q`(KB5YZhwi?YL1t^H%^Kfq(Pb4~1lZix^?@d|tj1wv&q9?&k z0nJIQmw1`_UJ6bmx>z`MGpDIoA9CM;0xn83G!0le+=Yu2Yu7`hWsqaSA4Q0PLkm%} z$HAiL?iTL~+vEm#VW8>;sN{GfP<4mR;<$PlCqN~;nco;xET3bLNvdATzzc`y3xKeZ zK?8W8Hg=+s^j3-mbpZjC6BNMey`pxm$zp4oGuLFfp#O9%!Q^ZQ80dtuC{DyM{ zJxyLi^Q^N0=;Srgg<7RE%bg8j$NWXi-d@lO;Tpa2%A-kptVdmV-IZL!7ZA-L-RA|jRuY5H(3XqcF_?~_F!W*vVR+*`mIAY?XG3ko#DUeL zNnRGiy-?wlV>Se1wwsgUJdCFD;5wVZJ#h$Vt;K30IyaK#vn^w5Y`Co)odftPt`hBIkkg$Y{&(bU?&bg-Vz=P^1=tYTDOH8VtQ24{5akT#cIwNDbu29mf~ zkpk+`PN@f!^x}A%rV@B2bKaKVQZkstAqOf}An~%@09}E=CqC`>R|I?w0uuNZgcc5c zE$A&+rTDkXmIeH3jSULlgjACGx&$Nx?To25D5ZRn<8J}D0iyGjpt9%*nooz zi_jBvM_se&-n^VA!L#9g04>Jb==mqOz_N%|A$Z(8o`WGS$Awi7VpZ!P5xI446Q9Gl z+GH=VGs#<7nAb%Z>b)UutjhTvm&$_^qH<9}J;Xh!bx}I5O?>iw@=(U!qO^tvwEq{2 zCSr5qqzXI4C+6 zHf%cdLK(V0Jq>q^Lbcjv$x?|sw1j?HgY8ta=hQWnvoHGk?zQ*+{G%0HmJHo;X0!44 z7nc8E=__ya-g*4Cxp%kJ5p zuGvQinn#`14+gus{&4d4lbg;hE4DoMjj=PY&nC~Ee(u!|E*;x`?qcV)cKqjG z2X?GnG`3{padYtepH{#Bi)RLJ-rfGQvw!*U_ya5ZuWEhxdk;^J-q`Z{<&VFTpIzEC zp=XB+Z`w2G+lHS!^tGwp_fA}V{iV$5J*Bn!kJjfpj=iP5cIIU?anpet&pz?!V?%rU z_vO3$I`W_U^6w6O@vhST^nFh}ziM{d?R%z`@%+G^=l}NAzPtCn_s7c*y>-+2*c=%(F>^!-@>6)=cV{bg0dhsVM$;5|$efiV>zVPvS9adHsfhnkI(l^5e-9qaHLEA?{p7mXEkC`#DIBQ(0XO{l A-~a#s literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/c21c4572-6006-4649-85b1-47e82e73a396-m0.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/c21c4572-6006-4649-85b1-47e82e73a396-m0.avro new file mode 100644 index 0000000000000000000000000000000000000000..4f57b9fff1fe963049330c0f05a33da882383112 GIT binary patch literal 8157 zcmcIpe~28_9q)yn2hk>%oC#u%@|Z}Cc5~gGxx3t*xk$~ACMKRXxmqP$Hg9L&-c4_I zW;-)`Kbl4(5|h%ZEwvC^4T5O}QArD>wfKh;MM|haYy%C7Pz8VgJp2(bwBPsM%zJNV zcjopsS2zNhdGr18`Tls{&v$Pw{HU$-GT)paW&FgF16nX!A=;4U2VT_-w2a2zthb8V zP&T8LEMgaZZD{WSjXud`GzXXIdH7#O^T>~@mPg=43EMu&M8a&>8P^VG_&2)vricTa z@)ek1(eK6%uEtZI+rusS+x5amkC?7U^@cW-v165pALNM>c(d?au{J#72RNwu@G5lE za;T5&9Cwjg8J%@-#h-A4JoAlzuvc@cwhf;!m&P($k(6-N4zwZqT$PklB>^tPF-hL3 zmJ7ra>YJBlDAG!nP4fOli|kcI(n#c~Twz)w$etbn#?v4K%XFOpU>DzO9ND64LUL^8 zFMZ^eP%uFd)ii{%!&Ges<%TP+Zv~d?pdztJK>Q5ShrvH=TfuDfEYixtlrvQ886XuL zOi1lv?m&E)0lQoiGX+1`y#q~K!35g22eEvj9?S6OvRMTcCb#5zWvuuWle`UFLHrD& zPz)iOs~O~aCjAFhum`cPO|o28O%6B_Fz24r&;iM z>|lh{Es-^d1_uSzicK;cDvS!)Cq)EtqCpgPBfmn-7>v3CBQMju=~mVJUZlDC|EVHQ$HfXFK}QZVWU*k*Z$d^k?UN5|FZt662bcD3x} z>2WDFs#p;s?0J~B8Y>`WCzITgo+t6cY|P$h$&vrsdR@>|kfn!7>H_EMu^RY%}VL zAT8UB!4|=yGXt$x$ScH96YY^#aB8v{YV)w&Y2xJzfH@3tQNy{2o~HNHJloI$bb5bs zp*HBu`i7RUWBwxMZz@Q7pJGk5QdwM(Xg4!j>1MjxC}PFf03@TecEX_bag$-cv9g4m~1=}s8#>G&(kzav%Fz=R1J_(x7-spye$+8Rs zYB)X`@-o~*#Rj+lbDWf&+>Q)&w26jPpa=`Fh>ym#2K7sWwnme$>6T&0vV6FY*`4($8ZILvu2a>z7&p6*?*io8ck#G{<5n6D=kxfmt=QAssPwU=3+fn8om2tZ-^E8-+2O z=cGChqv;87oz38(I0UrTV>OYS8@;u&En{nJxUCX$Yr-v41(2Vr&ewy=qNgl`3~5xD7LWstg)RPyK!hbL!^xltmjelQ z#3+5*vOsE(t1qTOtO=Uav8~Cqfz&6%B(gsd+4;i+q+cYDx;&89*CsAY`bF3hNPlGZ z$Myr#1U2)bYn<XoSA9P5$ z#qlAXdJCs=GRGUS>;-zJL&Ya~2E}=?p4D+;;f4We5#CJ==HUd5>4072Q^9gD`&ocJSTB{#TRAF?l;+E_k1<9mxv7%TlS{@> zN~U%5Re~qXH{=Bd`Ji&S1oJgv0Pn%6q)w`l_g0+0CFl{|(YAcLH?QSMaBYMSpv8n6 zz37A#SQF6(1W%eLb1=r`q_D<8Y-k-MqO^`T@p+6(Cws!qUfIgxye`4e@DAazs^xc5 zstity$|VU65f7y{Md_qA^~#UPLk)XN(i$7k{$DIkS}Ny;I>MG@V+}G3C2mr)%*RMG zUCFB)t!V=m%GH2JX4R!n{Hp7^$N%-y71ur6e%nCDnui|g>cHma!aL7zU%6%f zOE*05mvbMTzwp_w+h%4i{PoTQcbt3ww$(Eo6AwPN@|xohZhU3y(UGoIPrmf&jXR4c z`mY@O;h#o#ukD^Vb?>edXSY{Q-91XK{oa4RweInOTW_zN-ubk>=I}3Oe{<~m@$VmB z{j+B#*Q{`su3Wl#ZFV5Ti842TzG6SWsdVF@S)eRU`;07{og>JNM4)tGoQ%nLFn@ z_dDl&=i9{%O@qfyWLL@3EQJffFuR^wL#goyq`Z_2IY|ysg_OvKwclY@yRGAkW(eHr7p*S@X$Ev&}g8HTNL5x>JV4e@?=s0Q- zAE3N0nr?5S{7M9J zeuRt@=S3A~?L#lY{7Mj2U;wBRh{?Q~oUg+YmXkyk>fa|=um-CzUMfHe0yPAINv#*q zWdT-Y3jtFWfC5a(oN-$q6xbYvA_x6qG~57XlDCZ#U*Z%nPJvhmiZ)OXV->�#LiT zUQ~l6#$~|@^N63E@M@$OY7$@`HepdEtT#=9DzQN6srVI1M`@LBid;R30L!qiaK2%kgQRw5+}0+wvunO zLSW#ym3q%)%Quy1S&F6PYUA0UKs%O>0mj<&n1KOxCy@ zlLuo;S5RV95GW)&iK%5}nT?~@XxPI@>+thrag<%O$LwjFCTcps@}h!@_MQRlUtQa@sE2Ir28XiFaI8U&O6{5+aQ)=lz0;Y$y--$9YR}KWqifJ$d zFvvy)mGWvi*@ePmNkKs4FZ&zNL=ho1S)~ju#`sH2=>_^%K>rE^B^j{j+WKs21Z4)c zE{JF(av#9@SNQ`oEh}3JDKu?iBn^qB;%UJb^F_m97Ab~y707ZSLTA?d&{lPS5yt&t zsIuBVz`j6;-WxVk1Du zbp@6Ht0;UUl6eV;Fs40MwEa7$va+fYXt&6rq7E7nJ3;p&oD_iEXjFWGRpCT)9U!g? z9E_vb;dZ)8OPp?ZiPPaIeMHZ^2tlIv!nIK(xX z=OEgejS1uP%42x8kza{&+U7SbkzQWf=64eIB-I)vjE{^qB`>3e>P4X-9{N0tkTy{U z3X_V0El>s=n|x=qLn?%4|7|g6WF-RFe zZG`#qFiZrs1f!<6O1U8CXq_-9U5r7~nWkJIeL0vYg6B-av)iW(c-BTqwNoTGsT;Nt zdB?}L!s^%v1Syi~DDp-y9Y$IJhD?%C5KRQJz0AOmt@%>S7EYca#m4;P9Uk+etFvKf z2?8mvNl)GYCOzrm(?kZa&S=()@DgMN1K_pxd?uEV56p;d``7hpzyFPG7pIi<-E6wHTlCJ{_+Hz&3t(19 zTgBt<5wC8%P!2qvk*%W_T>!`KD7`(~-30|tt)BMzqd#@mUp(%(Qdia0x1i+giSPSr zPR`z3zwFuRJO1EiAKr33xO~oraN)wvcUyxyqaU_jy&e2ySLgDso&XGo1QpDuirjO=^VRZ@P?t>)VQO1$#?CG53b$odH&;(ymR=r z6%X&2?fxP9@+P0cojFq1wjH{g0w0a(K7XKcUF+BNGvDu;zI3po0}NaG!P#XUw`vY{ zU0EA<-uv4hYnxo&`fUKT~m{;>o$2dp(-^epFSWOIZ24-&v|U}!+S<9bkty#C8oTp$^Iz5b ygU*|G&PQU``kO9YJ+k{j$L*v0{u}pOS^L}WMSt$7=$q8sKIZkiTW^d?F30~qK+=l< literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-2216958009875447676-1-b68e784a-6b9c-4894-827d-c56978599ecd.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-2216958009875447676-1-b68e784a-6b9c-4894-827d-c56978599ecd.avro new file mode 100644 index 0000000000000000000000000000000000000000..619989a8a3283c08923351d067a48609d3202a8e GIT binary patch literal 5072 zcmbVPYitx%6o!WIhzPboiw(icg2a#(cend2pg{_Ofbv)hBn6hcvv<1#voq_=?A8Su zgb+}{lt(~OqNo8SJVJtsm;fTQJOolu3@9iMDIlSiD5*pd@67DnJF}0r%g>#?bG~!G zbIy0ZnH-qk`Kdw}7lq*x6GjoZ+_K0bC!)wwZV?e9h(Sw-inS~aMd=jD&j_(>l^U(^0&pS41kjHeII=nnA<=8BB;!t32c*rYW0NP_2y7%@ zYlUF@aVylGX)E7!qD7%5EmsrI8U@O+ObpnJO-~qTHz;8sYg}WobOPE9s}qnlsV z@)dc|rnJh!$2iV{WhXhcP!wST|BZ(|g0+r3PZdYeY4zwmZPG+V2O!H!cu1ANdfb2@ za0v4FfhLO6#TZtr5P*C_ldrHclz4bJ4v;)S9CEQ#(eWBOkb3PIU{ zjSC_cjXnmj`d041w%R$hB^Eqw5iAYKrQ)e27xTp;5r`E-xe9_zn9|wieXK@xUw{(+ z2$G?)4@lo+V{$Cdh>57WRq=a3e-8vWAbQa&yG=a38X3t^op`lJiZ#> zDjb&Z{JgQNiOUnKoT~RT%0w}lWKt=6VOl)7(k1#?IF}-l{IVht-eH~z~ zhuZBnm)GgBTD=~()8TNt+^+s=>ZKSHALwfHS_fe2BTz1lBT>S;J+Ew_zK}|_J(%Yj zz%mH0&FZA{d228kZfsa$jI#PQPo&n@YBD@Yf0FKvlFmoOtCAJ*MlHvo5SjWMoRBh7 z8j39)g?OMWI41qh>WL_Xoj?$+F}R^DAVx!z*&Pu8o0|YoU(|uHx%eas<|JTx&8b7m z0%{`6mxE)Xs4jw<*(-Gfxr5$`fLhB4XeQUx5lCMLTolD~q~MwTQx-g9qon&O8l2XR z_=ue26JKGpY!rePNw*X^Bj^?*BY=%gl2#B!1&P1R!jGx>(%cqF-bRZ}_{lju;YXEc z&CyZ>T3(%=oB?!tGR3!286Y~NcrVIJ2yzmUOQOdd_SN`U%DN>F47Q9Yx8dVdQ&BJJ zTxyauM@G?2TCKaK_RLZyhejFoWPnw@8AUbSnV7WpU^93$ii($2@}u2s$FhR5Jhm%} zQBc3oJL1bI7~M~13zjojb}wdfTQsgxFKolKq}%XZG*{XMDfWs6O(}<&nKpeEfw5Em zQ!G_B7?T9}9@|=Bo`&GbQaq`_Fb%QW>}7Vh!|8Q7+-{e{ZufR_Zv56UINChEX$Jc5 z<@{;edfw`K{@kHeeMa}~)^hRL6?-oIdSyw==8KmvKA1Y|;@|$|5AJQOIu&B#(oczH z%ih>fzqET{KnlaPCtqpY^6`c5me%)JwxzyucySElqdv@*F%3X+WJCgrp>HTeGN4_qe`Q-hoBPVv2 z?6}vmZqv0Uv5{dzpEO^1f5Go3x*UVdla3zur+u0cop6dW7!!|TiY^j;WDELl}_C(m|FWF~eQ zv^sC#LhH1?^Blv5mQ{skFa3b));Q0)WYNvjXX+ldKJ2tG|6!G*?#H1g$8_C3=7~-F zx?h^rqe1M_xYu_6=@oA=EDH`_M`n?68uPp?rqE9xMgmz)f~#hL()D8mkUT5kSwH{R$A(o449|C^U%;T$i~Tx zrX%t=qai(QO9Q+2)tx9noFgZyfoyDPMwa|en<|MjY^o4zrU4$`xfPT&oCA*}fP>DW z_GoEoAS%n1?)um|GO}WH-4tSvm>BASCJ1b8F9MnYQe}*6FWqWjMF?Dg7S?D!NxOj5 zBtd40YX)XnC&-^*YaA49lm^%YB}>D#`G$nBs%c>4ykBKO4EC`5rwz;uz&QjwY6pP4 zF%2>72Fk`X(otRJN}m8>qo<3=P*Kt-=2OVxdHX0yR8Il-6e#78;R6LFXakr|BYJbE zs4h!NRKUt>grAo1hRqUb65t*7U`Zv!o7O<}3Q$feN!x%0J7;2FLHrl1o}g91If&gm zG(nNWbttq`?4)WIgDN#hc>$!5RvCE51dim05E4D3*_d>?GayZ)4pmP!2`rNDwnBRL zxN)avn&n$hv}Km+a`o}-Qb3P2Vn7j_o-r_MP{u%DT(MXl0kekn2ndWS){9L(l?Qi9 zcS}Z5*Cir5)v1M+1Q zm>iCTd;5ErJE`{&OmbjBzr2z#pNF_-99cG5?eoH`y?xZa76n^LpLLLDHX5F7e7j1ZkD7e%Q@VHPL=$0y%` zmWV@y7zC3V!3+g}h<>KAIbr}x9|PdFsD-fY*dz`XWnf;-xj_m5^%0h6!!U8wUWVG( zDz$>#qIF`Ra+rZOGEJ>OCR#9294}IX=eJJ*cw(c}+bJHL*NxeTw&OEfAv!h=!HeWM zinb9vhtUu~;gjSRgo+@umjV3vny=1mvE&tAY{pO9;Tb=UI=hCJBk=Ni^t27&(bFit z%w>S-47FaImkiWxCYMZ)bz-Z@v6^km9tf_CDYx)#>Z)iz>s;=UTtmjueOm3ArP;hv zBZI~njWvLEtQkl3-I;l`*%G^RrQJyv~^j2&c2jlz6@4(sy z3$|iDv&G}mdJz}qCf&qx@mzTq)VM1iw5A+>W}4b81{31^*Ep&m7@q_rZrE87p4O1b zqLx(jOR;b?6pi*qBXX=S5>^zsuWwH0KgVYVzu9}G^ZkQMC*NLjVZk5s&mDVr$LI3$ zL;sE5{?^OaSDZIe@lU3veqZ*rdHvYfi)Zg1?^u$`st=_GPs-slKg?CGEZ(wk;gN}h zk;QMApIt1>oG|`=Vz_(%&@^0CS+$~P(YEZ-MU~Di@eXC;($Ue)gI_!tKQglOmg&^l z)7A%9r#D=EeSOESpVwWONySFHrq*9NpV|N2OSr3R^5ad1^&Oe9EmyxC`?2HN>dmJn zwm)z}drvceI{y6`^lS2~&2x@yOje#eaNy$UPha?O?&{|{_8xn!^Uihhnv?$?zW2+8 zmwy?tUs-zJu5If#CG2yzKA$@!y&1l?{m;rx-CK?1U9Vog|DE6d`e@GDXJh*On-A1+ F@joZigB$<= literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-243132775900853210-1-0c03abfe-66a8-43ab-a765-a9e8072b59ed.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-243132775900853210-1-0c03abfe-66a8-43ab-a765-a9e8072b59ed.avro new file mode 100644 index 0000000000000000000000000000000000000000..151ec4194d68cbbe4dfe21440da196da973bd96e GIT binary patch literal 4966 zcmbVPYitx%6fTbvq@-5CrlDwV*9HjLmYIFETfjirDoZM;i;;(Qy*qoiJ9Tzuotb4_ zu>Rtq5G80NCTb-m5y1p#{Gk{j0V8U7$ScI4!4L}x(MAm+il}&JX6N3S-JR_&N&j@_ z&iT&$&N<)tc15gzTtg#VCrj-LmqaPJ(Y3*)=F-U3<&qI6$#Iw4CBl^Ue+r6x0x4=V z!3zjKPVh*GE3U4!8;VY(p_;@e@_`gp=EXkze_Y~loF0jTDlduP>8HXV%d1K7Xawl! zBx)NUcXjz0X24w>TR>u;HM$^KVy~1`C;%-%pz!Mv;6)J2sYo%?%?D;Jfzwb{c`cu) zl15wae%LKNP zZ?r;u?6@JlXUgSUOtdUzi*mK`Y*3&bE5!h7ZF<7Mm_Z2xj&ZHUG6@(ntVw`lRBOG+ z`{MAzw*tDDm(J0w8&Ubx_C@icZkdffP&+ZNHPt#Oxgl6pAG< z<0t_$g6itha{vyYK0p_~Z zYJi}Yc1o2XSLmDwDB~rdr94w5kdX>J6vguu;Mv`i1D>@}iro|qPU}WoMAh+$ ztFRh23PFox8j7kBOoLGpfTfdU6hzZN;w~Ndu{B?j*CNTYwAh57s>2h0ba6KPEJdK@ zHR-7uz@(>Ce1^&Z(HYHoQC<>IP>5U-Jr=O5#>WblEqP!uWkk6xKTQo44U*2K7Rm5q z6y2uPrdJxvDwXnQl+n%-u(~s&sJ1&3i#8T)35P~e@v_SQWVf3!M^IkGW<|3J`tS2f zTp0zU`^j#=ss?j*Vm7Zu<7)N7Cd^2>jN_uY(k>`4S2SorIqb}oby)<)L-{W-R8BB9 z35eXavBEsH;>pq+sl_hM>R`RBFTi+%ey=YWV3{D}s|ok_w_iT8=-O}m#dr67_1v|| zJ8Ms!J-qx&rlIEf+53h*y7y>v(|rC?tG7*&Din5xz0_m*H1k;_3c$n4Ry^+ zW(OwC-MHwBn)@eAz<<+Xi$1$@piVi|eEReA;r~?cTQ^V4Ts>ve+-?2f`!zd{H~)fq zTOzCWZcSb6>=}+N*f)L7$%FgenOOVeU?czS$ebA$!f!U7ZJav%lY3duuG1Hex!Z3n z-u#36vDzExICpv64lsHX>>7L}^81F~DJ_$J%yxH#!pe~;yJ0R6m^?gsee~$5k${;yBn zIWX(H+w-I?E4FW*_4pmICnAAFNnp52^xMaKQJiw#OG^yTfr%BRpqh~apx6ZzbR0E{ z@tJaM6jeJoHZ}{nN_}*e*TtT|OEe4Q1cAuRg@ECJr$&OJmTn}lSr<4&3lbye;}%2E z>qf{pafXv{)-LoCloo=p2z@}}LAAh0(fL|*VZA&jLFM}h3;JLq)t4gR{Xhyppi!F( zs9`=@q6Y)>Fdt-LOy=es9fSyHg(2sKZY~`3K!N0Kpu`n91=LfZIsiEXD5$1IK>K`9 zzPVCVl_j%HhZSTHKdr(m=c`eZ0PU~_iz=bNX%v)Q2c@Lq7CAa326)L(5aY!nC*-O~ z0a{Y;&~Vb!^5%w=Lu%hY&cqUoVpGqU`lA?olF|$^p-$r zE)J#-`spw$nR4Zvj6&hirNE-`7u+5+QA9{JR!yoFWBhTN^aAB8pnL`VyZ~r)ZDlrP zf)WGk7eqJ|dIDhOtK@;rwbN?{o6xicku*e>ilY`^%oPp>X`~qPRUqvR5IUQ^52aW4 zDp=zVLW!350q&bbOcrrop(dnkRrDRuz5^bX_cRM>*0+zJ5zu-^v!fSX45^3$xuPT& zkE;aexT3&3phc0XL^96<4pz(0MS1@g6c!d00{IrLsHlZP#HK5L#LN4jHx(71r6t%y zTL*~iepaWGvgTyl?G$CRnVn|J`GS&p7J@|YJ6N)*48(kpma=9;K}4%P#oAljM_K9` zOv%Y$ybx{8%82n<-WSg{@+)Dly!lm2q?DI3_??J7iM2)%<0GI=$p~nn=Auv#4}B6w zNS-Ja#T<)*El>g+gM25nLW3yvwxl}0lxeY;?M|D6qMVK#TXuGiJ;&Z=Y2HK&Q^@~a zTvak~^}R7AsmD8>I+0(MUXb4Aud|njuCiY0d$-H`c};nF3px$zwfCTT!TGfZo6g;y z{qnCp3knvmD;mD2hi7@(vnxKko-%jafJschnO(=H?Htv;@Zk5QN7GAd9nbkYEbVyv z;+~vmHgEajjqx8o2%l}}kh%Lz*7eI}m$&WTcIeaXn+EmG=<#aN4%eyOhlWg=cuD#^ zZ_g%o-I{4PIv3Z>xRW>epS1N`m!*os{yhEddG>5C>DJVf-%Rb3S+cY&eJ%4_@sybr z=C_xHW?Hu|Ib1j5T7Ai}YnSPJkBWzXw}1b>h9i@!^S9^kJ)U`>YqzBbN><$M{?&`V zxsN*cznwC^%oP;cAKlsh%FX(xjtw7FH=$pq%h7Jwv|II)svh2WbZ+N`d3`qYzGHDr z$xgkJ`ATgU&x#J+D%0Mvr|h7gK6&ux`m*_d^!;h$x~q@f4YhlBJ-+z+`eO_GuN?c* zny;5Nu3A@mBBR^lp47mK3)0E^M>d@4)W6YIvT5uurHze^_ru=G2m6k`_vPpj8?0mI zyk*T_Iok81Wlfup>)Kr`UwF8@b9sIFxQ}ut&ObeC#tE+B#_LTx|MgWpKK#k^+4-Y4 z9H^@Z?i+WtDQk;!z@T)ndC17Ewz7v054es58yNcsV|w%4&GSR&biMn-yBlXMdi>_? ZU5?eM&Xo%?y4goH)mAk8*EYKJ{{za={7V1; literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-7131460395349610904-1-321000ac-b08e-430f-9c79-e40d5e4aa42d.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-7131460395349610904-1-321000ac-b08e-430f-9c79-e40d5e4aa42d.avro new file mode 100644 index 0000000000000000000000000000000000000000..929e6598ea9f2d611ee647b86a9ecfa9f9aad704 GIT binary patch literal 4908 zcmbVPYitx%6sApMi6*rr4UvS9%b><+OK0}g-5P?0)`-@MU5rMk>)n~VyQ8x+%gk)I zD=7gZXh=|uqJkzu6aCSk7{p+L!AFS31T+L<@DURM5flXsN_=3vGqZE=o!y=7EkJFK;=@Qq)sW5B(&O%jAA;V0h zWCh{JDH$n}!S$_NQ*j~(%?v(K3S^kNtfujANfU9LAx(s)tf}CEk!7GDn;CFt9N6e2 zYPT$LePN!T=&FpZAT`ZKS2QN}3eAKHuo47@`~(8B3X(+=8E(3zzycFE2X#}n@<|vu zBqlI2NnBP3jyn!TOYM`LYW_i8=CakEbCbHizvA_mf>ilT{Ed|UB0xq?E zfL|m*6LtY%k%Safk-5;zKp1Fs9;qTqsQGLX=`?Q-CGpZJ;G6=53{pLypa2a3B?(zK zw~OkqB!n5Ptc>}o3vWm(;3fg?VHXxxg1u=ORD^-DQ%M*q%o!QY^c2K~aE~z@@gXK88kY?dce*_wHKPuePSyy_lJB&F zG=1CwyJu?UTS>I86)JM|@a#}v9jnHGz&1T$VA`OB0pGZ6v0MVC4eJu%8|oV-F2Wl~V^`0I%zo+v7$E~}iQ#RPx3E4{#e3)pXgl%@lSuWiq!MUX!*yCCwp z+*JVEZ#51qOpH!+ad_IYSQ<)8B|1Sa7SCt15G#gt6$Hf$rL*RJcvjsXN5w=Inb6t? zq;LF~98^WUn6tMkeh;|sfuy1(YeEWa`=n$YuXj8!2C>9a;7$3U9PQ4pr;sY%)elBJ{3(f8^q#Jm(HwWjr`>5MlgU!LYvWW23OqYz$ zrUhiSv0urH*5-FCkzHQi<9AB-q|zFdjE{~trL5zH+J{3SGV~1?A#0)>6rmD@SRfx9 zk9_-DA_@^85JYMiGvoup`kCtHhyVz^1c2J2I>I($lPFl2fVnlN2FVB1Ls+~4!$eUd z1hu+Vst37F>qJ2LAOWprn(Bd!*I}Y4UZ@PuYoC1Z*hZoC;Zsz>=;^#K+EgW(=dQb zPqp|wl>wqNmi3~%q@ZFDxg>h5U|Wrkm2F${fN^C+xtVWMM@3_#bE!vi3>iiDXtisW zrt?bG3>sxLS_Rg&W)#(PXX4SOgRNrGC@Nl7rEhkx8}kK~RqR$QpJ0EW*JH~l7~M}^ z2i7o{zZLVCEgILV7j|Jz(lsm>&6RdRnY*Gv%gW(prl!pzFagScnWOT9@kl`Ys+|?) z=|((RmL+BVQXmit2crTXj)kIpFcgf$A~SaPe|aK4-g#yIvk(6C%Pk{4*X(FHeB`a+ zgLkyIUO4-vyzp}S2OsbG_L0v=!yETyQu~jtGOkSSUz7vwnU-1oe6*!!-HswtGiSai z9vK<_bmr#SC%N0+z57gl)joOq){9M%y>r{bZyY~8w)dW?$uG_>@03;>U8#&X*8e!Z>vyS4>qg6j_) zKe%D%(#z?|k6s>0ezJAdrO9nW#TO4P7~Xm8_D*i&`qLX0KQsK?y5jfu7V;C9&EMCa lAD^*u@x1h@zs|q5yLD8J{?^m_%5_K2_CCEi_If!*{{vBBm+Sxl literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-8352556540874311077-1-39ec1ceb-2dbe-4ab7-ab4e-0d18bed2d2da.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-8352556540874311077-1-39ec1ceb-2dbe-4ab7-ab4e-0d18bed2d2da.avro new file mode 100644 index 0000000000000000000000000000000000000000..070f738c8a08409fb27780970531ae9a96f0d6ed GIT binary patch literal 4800 zcmbVPU1%It6gK-3rL{jcl%j3Dj-^FNnwe}i*(@y@x~Wtm6*m?ngw5TVySo!-XErml z*{&tfmR5>=s7i&jN>GCMASm^(;EM!N`~ji*P@fe1p(4Hriho!6(;E$nx@Fb8Lf{1u?o|39)A^7aNEsh#dWCf^-w*YA$htbnAf?A#fSnu1@pGI%T3} z0huMPn=Z>b1An3;X;g8DhFl9(ZQX6mcSH!QTBb|9_v^c+w@6SxZ3@W; zHSFR+q#V?UL7FmG#svt6ELMoAlB`)N<%rGm4pEY+pF;jAR4o!S1d6KIL0HoWy}4IZ zpCu(BV3l;>rz^Z^s|rnmg2MqUR6@LI1JsxR<)xB!Ok8$~mK!Qa_+s@Fv?^{9yZ#;8 zpvdti6xu0HR<$Zom71iyASk3=L%}hDBY9K^NsjA|3!Ux_NZY7W^^@M7&agvGOAcFHu+Q@{3)H4 z%!*-1AUpM`#kP%W@HY&53ABzqZxly6At!^L4rwCQ0j!%23@Hs9#LZcjL9huAd{LaS zE1*_+gw2{SUqw2UFg%8VSe|GJ8 zg(4}WV8Hm>*&Iw17t(-L$=70r|5zZs$a@QUZ=t+pBMjH}W|I;W4J|&Wpsf0l>=v4?-i=583_mM?)e~Q$yCE{Yb4_MzsF*#+b zcCGAfRd^2t@1dMw<=R3jV*BKE8`e9_jwree(h~)KMVA)OEFd`UDX@jGUN=2oM z-*-e_b>)!Vsk@VAXH@q+HY`frhJBiXP=w8V4?c(vl#ik`qc8^)ffG{iNJqpWMi>NB z8o>)ifQWXcwKrk_$~XhyuBeN!y|^R}mSA8(&AC5{01XkA>A^2?)G>zI+9`E|+@*73 zpmLmnw(?BfKxVq|P#kZx0Wa*HBJjjUX?9aQIIkOX5k1Feu0k|y9D)}qFcdu_1O}re zfWjxqF9_8@<}M@n2{m7n*J8;lyx5GNp2IVKJaP8@EJxtw4e03^AfTsJe3{Dt(;4c# zI4^l@I7}{?9vk4Q;aJ16We)^X#*|z5H1$<H*f3je|X>XpZ$5~aBcXz#F?)aVrM^@{b8Z+jYocZ>AljfQ~Pf0 zGG-SBCeFogT+2*-c`AM3(y_x&J~wyl>eQ>#?|<~n=aUy_=4b!@aQNy6Zx%OSK5$c8 z+5f;%`>q2op8jj&$1ks~Y`s9<-m>TG-9M&Q=dXYB?|t)w8?D!`|MuU+gZHklT=_!2 TvU={1JJz<|b9uS1@f!aFKTrwl|NPy!H`)D{4G zxR+JfOu!8HLLSCs&g5AjWH?KOf(KRzYOoSYBySrf?#L-%oC4th6l|a%%*ufEdZBi6 zy{HCDOs)kh$RU0@gg0Icqb337VG|Zr!g|vvs9_c;J(UVsU_)|1RBQ#YUo3J$tBMj} z72^(VpwKxnC|ak;6&_JdRH;IZ7k~;WRRi;wg(KQ&2?^(LvVuBYACNYq7DrCD5!gz; z(F)%7;}+>X(^kH*L`z~gCRZEJ1_j!&cnmPsrpFAl8x%8;Fs`*&CIRh+H3>)<)mkq+ z`IDN)lU*{zk(dL|TWRM~kDBOXr(CZPP?e2Ut#!(U59^&A64K z$U{~@4-8S9BB@BNe1H|I4fzUaLy3lm=K;R0zOd!EV{Nnn;Joh zfvpQd4TbIlSbr;dV5VkzO(un=Er_HcvQ+YG@WtF}Fvuds(5?bmPk_+b=6z_Zx>v#K ziXc>2Z6DyiNyKEa;E}3B`c_5n0rNdj$%~b3LNeC&@o^Gb?`U?cqKhGQQ6N`TwBiYV zfR5`5ECN=RIX{wl5eP7>J(sopTUb<7Tm-aRbfBUR8WEeO`w@@mg`OU$_&lq?KIS?= zTxaCxI+;A$K|2|j(?#bnxp~fXJ@qVvi4HvIa5DW6^+8t2nhhlx?e^51C(VVFW$eLJ zS3k}J(c1J!oKM{VJlx2zs2*+g8=gq7FKshC5q}cvjUvuRLaUOK&_*plp&*|6B%F{o zQU(eWi-J8+0vwxuC-g)F!hs>M)>zz70uZaAiSLdu0F#FSh%f3u*kpVX0n5c;X3dF1 zN&spj%$og9(g<@Yhu*7n?#`ESbn+F2PntH(&UFE+h&CCmBw*^l4lTc^HO zyQ}}E9giI!$vwPm>Bc51d%>~VGvyPzUi|Xa&V_}?&YrxPz5Z|E8T!qIZ{BTrz21Lg zbL(?CgLUMm}6<6h~+I@F_&cmtxjivIT!ub#UrG9s}%TYd%=hkayOOBN78*~(uzA*B@>Xf>5a-*7_ewYId>rbD*oe?VEHgD+8J$vRiwf@m` z>A#x|_xe7Rx%{g|{l|^%Rl-v%FRa|RbOy|Ox~BKfD~Gr?bbCL`ud08xUT?YZYs;1) z6H?Y+?HkUn|C&o{>D}#xRM*bpgZazYw37ax94p$taLTc>Yv#e+`n=~{HGf{8aqrya zLn9hDG%q_b>7r-(q6IxiKj^6ImHrXEcrp7#dPe5~b6=TRKJ^5@;=sh)JtjW+&zwtV gyxZ&L_qy$Ob-UWK@Wi@VZ=6f1FDt8hJ*E-<2cc;QMF0Q* literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-9221420601809522049-1-41caec64-2cad-46f5-a966-dd75e3635abc.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-9221420601809522049-1-41caec64-2cad-46f5-a966-dd75e3635abc.avro new file mode 100644 index 0000000000000000000000000000000000000000..09ad8a80111b87a5f580454e112992741352fa25 GIT binary patch literal 5192 zcmbVPYj6`)6y_mR28NajouVDQrVOGq&L(Noqykf*MT-Rrt+fIYH_1)2bh8_FH!T6H zFuaQc<70+ez<0%QL>w(vs34$I)WXOcg@I8(9FS4)NfoVkceD5I=Fv9%%;xU-&i&3g z-}!c?dqLl{47yg}9}qo$7@${}mYAe)5Sj{10`%~L*OX)8=z#n?Ky!=_ijvF6un;}= zF_86&rh>UkVon5U$&XG%0*R8qaMkFym-nDJrM!oh7@h-<&MpCUjN}JX%YlN9qna77 zsUVM{>T_aavyiLSM`w9m>+guPQqE2qd!4O5eSLU3nU)Y35*n-uT>Y;!*dc;-jA@L4>m>pQv|#ZNPY-3Y6}20 z!AnbY4zNt{LKenkZpqg{h;U&Davtd7LIF1vNZtlY$|I+MdJ5F}A!h&ub+ia*uNTTU zSBk2#WXaQE1sKFnukcFwI@BaUJFLN?O6YGI1!dPkDXF+bjt+``UNRKKc(KR{xhj&M zmef0RgF@M2P~=V#T^>G^s8W+;UH~ej5C+;Y9Y<8IE+kyUh!W~_Wk9-(S{ymqO<+Cw zYAbkqj$5SkOn3Rl5-sp`F}WIeRwT!>{&NOmGqOA7)WMt`GW4&!fdr$fF!uPa8B*)&ZK~L^PywU@fkj z=UGT|=z%JV6+#H9l@HKdSd}loJd|j7SQg+sK`qUOqD9B4>p%jggf`yEq+veM6DZBa z!Sq5O9bzR@ft-_3C>**JSTz2E%Z(Y38 zF|d9?go43a09M{g9@tXhZ$07Xn7ytzDdMnG3OD&L1n9=_ki{uaI?I-TS$w(eSD07);pRVz35^{MHI*tCAoN9 zH9*G|1?B-QicB?AaK(>!crWw}Ma5@n31(>P z0C7FWX|-B%t(2XzI4EbH)k@_$GnLfM2ooI`Xw9Xv5cL6CGS@;uM7zD8ZIrf<%<3NO z=g49_5UtJXi1TU4#>0&aOUNUye$^8x^`#7kC*n_Hy-~#Z2xwI@0@|nrC=|p~pM(>V zM@mJp#G+silmN$|-w8btfv{o-tTj3}lmJ9;XyUsg3}DH}0K^ydB5X1~iGbx{Fsp|@-Vf)q)!6iFj!79%czg-nuK5LpGWzf8c7q4{Fm7EazmijDb6Iy~k_ zk!RJ>5(H9Sjh>_dG=B_L=9lR~LTnsh!xm zv7mj?@8i~3qhag(H(xEAmG(zj+B-MwZ5{6w&$*T{eg6xEnO|(^ zx_PW__qUb(KdG?)R%xpl_e<4;6Zfuf-8FKN(;L3D`>S!LtimehQwhDSRMh-GnPOgPLY#`56k9 ztZ066$X%ah-~M=JN%O+xS5{H@%sf!+5ALkpHNv~$NXuWR zQvNPpHhkT(0~PMKRROKAhi5r5zdl;{AOOHa=_F@pYA8Uo*(PzyF9qjZ1$vSCus%U9zyV^Q#Tp9)p7y zU%YC*;#|3Q@}_0mCw;$n?B0s|*EheubnSJ|fHCiH&06=edB~?-Uw*aqku$FCyPx^7 zYBqm&`jxIT=3iI8ej7z~$9Tr$!x#!X<+qU+;yLW89 ze7QL_KmX)*Zl0rJ>f7UweB5^Qz8P=ycO6@k(l~5C>!|q7F?-@$S?BvK8arj*m;oO& zEeJ8cUFh6)ruC!LpZ}RMcE%k8C*D)E;N6GWjys;eZSbK11E-bVRx$0)#_N@HTbhK< u8C~tC|Gux_V z$c=GZqo%$JSsA#{uJqQFmqANCC(6l&hZwqw_?v8W5powXbP=%svcZ|p{EC4ykN%bo zPF(6!44gRROEx&NiH8_CvhbU1aA0B=F>qk`|Lg7P^7Z%j?0qxowC>h3LZ`LYu9HVcu{jf6 z|GP=XhFo+Jw7qh{iBmS@f&=5<%jfNW-U&W$^KsARb5<|s1fR2bSabQT+0B^XvnCg9 zE}tGAaPM1ps?5$ zpdqYk95x8!5q63(XuvH+hyrZ{MkvrS1{+P7BN)V-!Hc||OdKfWJskaMAWiaOKb z(}kA11?WI6Oi+9b>T~b^i6%SbULK8Sh8`Uzaiek=Bn;1UE8Ms%W#E=)w z-%Z|p`4HC(yg)9QW50wYcy-D}6_lPc-wK>&AAIEYd|FoLd^mkm3C2gF1*g`j{PUON znhZShc(ol?F%V4!wb86O_AQgO>zO`)Vtv<3QTNc-ZH=%EE#59Fp^$<`8I`OLYVzd{ z6H(h%;Hnz1LUM1dOv!8#KzwP!nl3k;j literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/0e8edb12-fdf0-4ce1-b6ab-c9c2a05d6d57-m0.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/0e8edb12-fdf0-4ce1-b6ab-c9c2a05d6d57-m0.avro new file mode 100644 index 0000000000000000000000000000000000000000..1f1d617f028d0c90fdbfcc904146a541b0f902d6 GIT binary patch literal 7802 zcmcIpZHN_B7&a};J{F6xFss8rrE$0J%NkD2he|{fMxB1bzPKJ!fXlIWzapc=zrG%fil_ zc|V@_^Evxq>4^&#Ugn#lq>A@nFsTF+HKOz@e&E&3Kq)BUn@+2&^cM?C#UgguSNex0 z75YRiC=RaD^YB+e@yO=7#M!M0Bdu`t_pMwF@HsYQ6qb8ihLzIP|r323oDs05>jVHh|6U1^ExRTJ`JV8lkK76hMhT*t`cZKORa-#nikjOJ(K=IHS9s`lS!r)<>Y`9 z0rQ?)pN+j*OrSqJ6+8G)ECf+ZQBbehq!4tFkQP|bqso3NR11PBa1x`(+*(;qzZY=pQl^OCRJT-jHzmG>`fx+HxVi$qm{sy z5}IS!u8Zo5y_B3#d!fb){o_H z0*kVnQn5E}QyAdcfvOLn(u(as)t5F);2LF|1eI2t0aPrX3y?{sT`xnAK(uiq?3B@3 zyfz;@*+}}8!Gb!A0BR})Fj3-s8s+DjCU=zc=9;Fp1t3F;35OgEyFzxRsd6WEurupm z>Ae7!-q%L9=?z7Ymu-4qn_$sVPwN%(3Nh4pdkiZ$H`#Q#dD!ka@r)8+4m(ubaAwfc zv>uvg%iDlX>&Y(ECY@PZ-WGPuU&Q>)1xf3ctf|*(vkMaKW_ml_Op_Z$su-Jqq_@}3 zn7^%nBxgHx5u-{1Ot*xFOj1JQ_`F%=nBGKMiGT|)f$W_rLcjJoX0ZH=Q z@unI$vtgU_U_duZ7?W@ul+cv}`@S^-d!CCNGESf8d)Fl+?~w}eC>1IjjhrLyXp}_K z?VbBM0K;|yF?<2hGP2`!ere-Azl^qA9InB14TYr_2MEKAkH`h=s*w%(h{+3U#BQQ2 zhVQc2Aj51utl42s%KI=H9|hOh4jzd^z(ze*6WO`ZmE3I^$zscG=E6Q#@{rp&saz*-Y-7V2{L2fdJFw;gP2i(Y|_ zaEa@Wmy+LYMLxv_|V6FU~&bY6bf4N0R&N&*u)173Tt+J$fwWTDOFT*Gp0OE z&orpoEYG08X6ji3ujOv&kQUK=X+mMA>4gr7l;MRAp_{oE=OvunFRh#VmExAu^hxW< z^hpEkVxLNugV~Kdd(bJAoVRjhDk#hsWuIbwLUKzbqh*(jE|pCC=F23Fo3D!@1oZ%}q-sK5!nl0G#_jA9tt`#Wgy4js(F_S=mE>qbst8Vsj%5i=OySmO znb0PW{FneF7+99p)ROgoF)?AO*c%#1SC)+lWTs2Lgl3VCv1WR$F7vFG4VW(90tN*? zOZbY&2-1Z^!yS<&z^T|j#^$^@rSDv`zx(@L8xO48cci{)?xA^yAKdrA=f!I}cCY*B z%cr;Abg=u|1&deSbKNh0zI;r%`h}Cvcfa@8oW+aYez!7j&BhLI{+0HD6IHyf5y1 zbM?+Me}6OoXXn(iKR&(T^pO5z#SKoramS9kHe5DwuJ-FY_WXsCcAoGy6+RZ@=`B oUDrOe=fuW~7Jq&3Wgi zR|=|P;~L!$|EY>g-e{OEfe%$|c|?tbS&qG`Y!|nm#T%|O97z1)MXt*whU3!np{yFV z%_{Ny60v=E3)UsA;TF%we#3)Lp}D3_ZDsocQ@7PikvPxBb#Kk_OU%aLzy-x_SQZ>% zrWIACLaMl7`N|Ysmn9WsN#Gx`4N|fjwK8$J`u3&ilC-L6k&?G>k)x8x3ife{%H^h| zg6wJ$U_m%TU=7Fi0d{^~36QNg1_XIJ_~;?0iu^T#sHQ2DKSkB5sBo(8c&2YUHmVSd z_{39@b_)E%mg#RrYmrtKrh+b8&j6`oe@$qIG&CW1AU@21dB?y^!TDEa(WdFIp_ygG z@`-vZ!e7YeB~+N)s^iwMCv3;7uO#qG$!mm?vFVv77m-6Cq>Ax35FkqQEMDfV0)^6JD$c&|y`2h~+FoQ51$ zmiLj?MV8Jqs=;OtnSm0DTtakn2Agk@l4+O9TM%*Xdr(-Bf)&Td7E2f8!FDP>+Aar8 zP6orV8a2B_w+pFJp>brf>tfm=OrO*skNJACrK}eU$W(D~U{7{4#IJACDZyhX3 zZ%X-zq)kD94+g49fJ)1E1Jz{GEP-p7aS~KoejiY=d{!Zo469Lr9)W1%M%b>PMZB1e zoopmMRboLsKmfG@1u$0P{5HzYHBIg)kIgkr>k2@I6cY|P7<7g7Oe@Hp)W-IKgQZUZ zSo&la*`_xYK~}culU;&EuXkJxv>< zc{bh!blO;Yp|1LYTC=$ik0wle=cEXv!odFpl3hXKlmXsiIvWkFhEszf z%k>)GCo%E-H5>TwgO{zkdyK}j5gQ6bvA>C;t;S=m(@giZj8ofTSl_jaN9gObNJ#Bq}Eu(UIo&_ z2K6WaaECK#V1)zRTQ)Ttn3Jrhi+PL=6VEV)Za5WU90q4B?32NO73DUF>jAgx(-lrV za+G>NN%!RkO@qKSxf+uDM>?Ox+j~fWr-dZ57mo?>6d~RcXyMS?BL4gk=lJ4it9OX- zur5?KaqzYTjB){tU^X-PwxnoWid;zIsO)?*s4RNYdyFoO3ey5|ptZ1rXZAcSc@a(q zO~~m}>_{;2ERHWUh?gg(fwTMebj-BL_JuwXCZ7GN$YzA;0GPh8Nn~(>E@0#A~@5I;2H-zqFvR9eSZdBBgktL+JMJ#bXI4_e(p;{Yr4l4t>(bQhm|@ zyU3@qX=C;x&lYk*$zv-=rh>wJQT7SeCnUF3GFp1c=u*jaZ@x^@xcR!cgdiVeK9*v> z!VL(XvondAP$TZ1BnL~;!@Hwpdi2RU&UN702p6Emgm*%;LkNsTv;|2M=7}6k@G&8* zbr4%x2Z<=Gi&HfrFJ@deVdHj=idL57W>E+8S4;~&XF6(n%ap1w|-2G#p{&c1|_tf0Lv~hD~&hU(D+6Nb>zkTO><=XVQ yM}32r`=*Q2FXaFHwOTHJ^4$+T%EzaV_4f~4Ir{qBy+?kVIQoA7XMKm7y!;OsL?f;M literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/snap-1455293623190430422-1-c34940c4-6b84-4560-85a3-35e17802d85d.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/snap-1455293623190430422-1-c34940c4-6b84-4560-85a3-35e17802d85d.avro new file mode 100644 index 0000000000000000000000000000000000000000..a968682b674d42b94dc791d6e4aeb73973fcd00d GIT binary patch literal 4789 zcmbVPU1%It6t2;hRx$LUM$y{dPDLL~WM+TTWT9xxrnHb^+l?ZWu)RBTH#<5%>&$G{ zHRMH#4JmoDh!0ZwAZP_2>Vrk9G1XuMi$uYQR?s5qLkmhpMexCMXLjzrv%9lDecahQ z=R5a1=X~dzrwY&AwtWE4I`(6(S|TPsDa}but|TL*wTx{_3}mC zAn>@T6GL;Q%<;LV6J_j`;6y!;>p8kL1HUy}g*cP8iap)7(9_2bp{nkc(19HC(OJ|X zU6V3tS)Lzkj%^TYMvQLQLhLEq!v>-WB3FNjAl*WRnn&C)-Fjd}2wcXFr_+4$Zkecg zKxT>SmdCQr!Jp_*7FArLA*3$Bb4Oor%-STRZGN*fTAjP5!N(9Z|)a0 zU`a^}Sf&pAbcHu*SD{HzcsPWGN{BaYfEpH{{8aL;h0AWq_96v|UaWqCR>dn}FStV+ z6gk<1LOaFHt9B)>Qe%`C1ch{JC_E-`B&URs5NyGNKoqB( z3aC{PVXGF%SBVZK43A+TmM5CUMx{}7MuiUKV1DT6oum+Rx)Uh2T3~9Vh%1ICWoS-O zp%@WTFkt+hd;unk3u(xz6lgKSKN3nW^4~)KTc~I|2*b7g*`x%;1B(lyQZC;Du>V%? zz{>pe{GbHWW`Z=-m&(XIyI8Jbniv#=UWH(_#OZ8%A6ZoQ=SeMZ5)adT!1^YR$q7q! zYGr?`!h0xu4;2i%&=yh=+oz~Iu-;*IMA2oCz9{f3dbD`f41(jn0^10?u08`YZzGFT z>2sIv-?40VB8%uPx~OPE24Z*kenho3qV_}a4eXJDa2){G4=bs3Ix(6YP9&94IhB-C ziNt+=>O+7D2m0ilK)i{)p;_X%u-W^PyTWxe6l}phWv8wZSei3+-?O-vjW*boib@xM z;E4R{$`QL$cPGuxsP20lSd_X0`!oZg2%GsHd=MR|07Yp=VGbw;C!*f5j)+4f7z9%q z!3)KJh<2v6H(~(FJ_f*DQ5Ru*aY-C3&A`H%bAJ>A8X+v#gJ0sP!wj{xQ|bn}OXtKu zc^!!v$-aSr?}N8sfR>FF6Dq^DJUnacpv8S1<^ zFGXy)OfH!o8{n$pSi`bq4+K-jlw0^T4ODcLbuPC^fgj`O5v>lrQYWj_%AavYM_R!8 z&Wxi*?#wJ&C)gGajibV{s(-Ra%~&j`Zh={$Ho^Zs@5YsJFutFn2CQeWcqbO|T0AbT z7nrah={Amw=gPaF!Cdj64dsY3)7E7%m;~p)!BEA)L?j@0%fnDFWAAWT9_dhH5Z@H}BOz+*@ GnEU^9TW6L4 literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/snap-8278841256334127309-1-0e8edb12-fdf0-4ce1-b6ab-c9c2a05d6d57.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/snap-8278841256334127309-1-0e8edb12-fdf0-4ce1-b6ab-c9c2a05d6d57.avro new file mode 100644 index 0000000000000000000000000000000000000000..ef9e9c5fc2f370ef040371b7984987eee8580041 GIT binary patch literal 4870 zcmbVPU5pb|6c$7ib%ToQh7h9H;VWlh<#w;Z0xig)6@3ft6%Tv?o zo%5ahopZkP^@-e$Id^s8QQKPWs0C8QyQQ6yJ60l6TC$01*_zZXnYc*57qO}5iQ{JT zxrRxgTYwYQK#ptc<}iHMEEVExvQ+HqmWiHzY9p%XZUH@-K^{7b z+M{byIw8vw-Sx2zVh)SZ4NHi<*>bUgXoASm#|hF+lpAx26Qo-WtO$Wi*miZAPu3|B zH4Dfraou!T)*1K_Z5%>nhiJ&PP{r2W+I(X|Sk*FJ;=NyGK@9eY|D%JfJaP*J1=OaI zJfLA0cOzv$BL=CaoPW(X8ju!FFs5qfj4 zs6I8F zXo4cg>riN?I9b&yM^$Q&@`9j{_81C|2^`72LP+w0?zqtD-hecXx}kcqNnnwDzZJCE z;|_T}(=6Y5qHU{EmurY;p8|TU5d(_Y^o)U7gE9spYjEh9QCMRHqi(HXehYVc3hHb?kYqINAw074&pS6R8eh-E?3`Y2YAk&aw=GO?cpo z;%2)HYL!RW9P{O?K!*~B$1o7f6OCe{Tq`>LLI-j%FLd}$Qh>Q_Hc)Ifz|=?{mkn1+ z)11OWkrGlcVEpZD4kn5VX~3%FYca!rGLT;6y@kBDP~Nf;hHHDXNePMu78gXhRJs9R z@2&QMm5FT=-4aY&5u~BIR8kY{VwrNWh(R&vRR~rKoX)2AkwtZXo{VLS#Km+Uu)c|6 za)YVbVjy?=B zZy}RZ=yQke-@&1w4MT|DqJ@gaWFWTF^CPOI5p_Nk-@q>E3f2K|y*w35rc%91EYTN_ z!$&eM_b>EP?*U9WaBEDK7X$T0?DmWj+kxFaH-2BRkb3+*I5)OfR|%}m;i~i5vxE&d z7?!e1SHJIxy!y%^!&CJq_1>uJd~8^ix(ypO4WS5|`Zk;p9Vs70sYhWRC;}&>-;th( zL&O*a(;C4IMSzHgrm;I>07{Ypa9`9y*misp2TL%ppyu2mMSz9~%e3K`IBFk5ZS0j= zL2l7IF;F?qKpVNHRvc7+-R?;mxfZG%O7 zv5?#1acRB4hWSZ1@mxGt-UT)GiU+MJN0^zWK8wM`IR7=4Dhehf0ht@NR>0Fzm@Mi^ zMZ=`@CK9p!cwa28^vk_*xi=QOY3;=NZ~y!4;>@DepC8|~ao;!d&fPq9MtS^%yzrLo z_oPdmUFwR@GL`%Ff)2Z>jdygrD$LFhB*McKo-~ZzF z(Tl4motFv^;gd7h3ug!aI`?kRt7{f%1IsT=96UU=skEf<{{BM`_r1O9w!|mP4rHbt z{OE%Z2iEQSb>VZ*?(UrW`TO(NF8zD@#K_?E)zYOuCTCWxee9KI-n_T`!^oZ=Cw)%Ac3NyL$cdmF;(|`{~N5J8SR$KkU$n8vp SparkSession: + return ( + SparkSession.builder.appName("generate_iceberg_dv_fixture") + .master("local[1]") + .config("spark.jars.packages", ICEBERG_PACKAGE) + .config( + "spark.sql.extensions", + "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions", + ) + .config("spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") + .config("spark.sql.catalog.spark_catalog.type", "hadoop") + .config("spark.sql.catalog.spark_catalog.warehouse", str(warehouse)) + .config("spark.ui.enabled", "false") + .getOrCreate() + ) + + +def generate_simple_fixture(spark: SparkSession) -> None: + spark.sql( + f""" + CREATE TABLE default.{SIMPLE_TABLE_NAME} (id BIGINT) + USING iceberg + TBLPROPERTIES ( + 'format-version' = '3', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ) + """ + ) + spark.sql(f"INSERT INTO default.{SIMPLE_TABLE_NAME} SELECT id FROM range(0, 200)") + spark.sql( + f"DELETE FROM default.{SIMPLE_TABLE_NAME} " + f"WHERE id IN ({', '.join(str(x) for x in SIMPLE_DELETED_IDS)})" + ) + + +def generate_complex_fixture(spark: SparkSession) -> None: + spark.sql( + f""" + CREATE TABLE default.{COMPLEX_TABLE_NAME} (id BIGINT, data STRING) + USING iceberg + PARTITIONED BY (bucket(5, id)) + TBLPROPERTIES ( + 'format-version' = '3', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ) + """ + ) + spark.sql( + f"INSERT INTO default.{COMPLEX_TABLE_NAME} " + f"SELECT id, char(id + ascii('a')) FROM range(10, 100)" + ) + spark.sql(f"DELETE FROM default.{COMPLEX_TABLE_NAME} WHERE id < 20") + spark.sql(f"DELETE FROM default.{COMPLEX_TABLE_NAME} WHERE id >= 90") + spark.sql( + f"INSERT INTO default.{COMPLEX_TABLE_NAME} " + f"SELECT id, char(id + ascii('a')) FROM range(100, 200)" + ) + spark.sql(f"DELETE FROM default.{COMPLEX_TABLE_NAME} WHERE id >= 150") + spark.sql(f"ALTER TABLE default.{COMPLEX_TABLE_NAME} ADD COLUMNS (label STRING)") + spark.sql( + f""" + INSERT INTO default.{COMPLEX_TABLE_NAME} + SELECT id, char(id + ascii('a')), 'new' + FROM range(200, 250) + """ + ) + spark.sql( + f"DELETE FROM default.{COMPLEX_TABLE_NAME} " + f"WHERE id IN ({', '.join(str(x) for x in COMPLEX_DELETED_IDS)})" + ) + spark.sql(f"UPDATE default.{COMPLEX_TABLE_NAME} SET label = 'updated' WHERE id = 25") + spark.sql(f"CALL system.rewrite_data_files(table => 'default.{COMPLEX_TABLE_NAME}')") + + +def cleanup_crc_files(table_dir: Path) -> None: + for crc_file in table_dir.rglob("*.crc"): + crc_file.unlink() + + +def generate_fixture(warehouse: Path, *, simple: bool, complex_table: bool) -> None: + spark = build_spark_session(warehouse) + try: + if simple: + generate_simple_fixture(spark) + if complex_table: + generate_complex_fixture(spark) + finally: + spark.stop() + + if simple: + simple_dir = warehouse / "default" / SIMPLE_TABLE_NAME + if not simple_dir.exists(): + raise RuntimeError(f"Expected table directory at {simple_dir}") + cleanup_crc_files(simple_dir) + + if complex_table: + complex_dir = warehouse / "default" / COMPLEX_TABLE_NAME + if not complex_dir.exists(): + raise RuntimeError(f"Expected table directory at {complex_dir}") + cleanup_crc_files(complex_dir) + + +def main() -> None: + args = sys.argv[1:] + complex_only = False + if args and args[0] == "--complex-only": + complex_only = True + args = args[1:] + + warehouse = Path(args[0]).resolve() if args else DEFAULT_WAREHOUSE.resolve() + if warehouse.exists(): + shutil.rmtree(warehouse) + warehouse.mkdir(parents=True) + + generate_fixture(warehouse, simple=not complex_only, complex_table=True) + + if complex_only: + print(f"Wrote complex Iceberg v3 deletion vector fixture to {warehouse / 'default' / COMPLEX_TABLE_NAME}") + else: + print(f"Wrote Iceberg v3 deletion vector fixtures to {warehouse / 'default'}") + + +if __name__ == "__main__": + main() diff --git a/tests/queries/0_stateless/data_puffin/blob_overlaps_footer.puffin b/tests/queries/0_stateless/data_puffin/blob_overlaps_footer.puffin new file mode 100644 index 0000000000000000000000000000000000000000..99774a3ed01b1738d6e28303714b1fcf7be12fdb GIT binary patch literal 298 zcmbV|Jr2S!423yL<(be{LLJyxn7gA6Y2(mHAqk0Fsj7NY4uey{1@PqW$$tK6Z`b

2@5GD)`6%WZjs<9gR_&D?Q*tt-(UUNWTST2Dd`;*E z6mEL<7=NWfp zU*n+D6Lm3n1N|m`*8LKCeAo3*7y(~E47tcu|q^# hEQ{%Cd}^vQrLH$xb|KjqA6v>dJ6hWli+YhB zPvMgfB35frDaS;a9hKMtGg-6yE9?ApPhC{ay7g2iomnjko`uRnF1K|$8yB>9 Se3c|@a2)sJpZr-+{`?1*olp1x literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/compressed_footer.puffin b/tests/queries/0_stateless/data_puffin/compressed_footer.puffin new file mode 100644 index 0000000000000000000000000000000000000000..ec4fe3ac3ab376ff30b74a7d94b26fd64ebbefb7 GIT binary patch literal 497 zcmXw0&1w`u5UxbUYc4`SSckr-u$`INnc3+bFUdvlpv1F;rGKikO=f4>o}PrTGuUEueaA0R$3F1OoK2Z^ z*qX=ib27`zx)jB{tj?xv=%gQ3edNxthUXN(wVh6k{H*jqbbWKB6`Eq3Q| z#hOY?nGe0k5FOI&4GU;7JcEyh8yO|Hu6+-`9;I>ZQ{6g!vFgVGmxGt#V(CX~UAJ7I zyIQ(Zqz0`V+n7$k5B(T4rmrALvpt&4GoDw4sBuy7Y>ovlg;HE9QFD}~vS?JMb3nEWTx2~jqcLScwr!k||R y?-OQ+kf*`o__Isj6SB}wZW87z2Z8-T$d#dvNEAy3S3Ejd*Sc>aO*nJ_ literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/compression_codec_number.puffin b/tests/queries/0_stateless/data_puffin/compression_codec_number.puffin new file mode 100644 index 0000000000000000000000000000000000000000..8f826a7477fbd13d4907b89bb0b3dfaf0a8b39fc GIT binary patch literal 238 zcmbV{Jr2S!423yLl^G9_KtgOR%-vCk{5T|{O+xJ=p{jS`sJI9&fT#C8`$^C3`J^|0 zPb=wJ*}hm($L2CCepVEvOf@bdc*BNn$f5~2A{%&UwQW>L*!xeW6QXP?#ln!hgh8(s w-Y3isAy0$F@n@G12w7++H$CPnM}hr8$d#dvNR&zjS3Ejd=%rg;-|yD0Z)>$S=Kufz literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/dense_range_100k.puffin b/tests/queries/0_stateless/data_puffin/dense_range_100k.puffin new file mode 100644 index 0000000000000000000000000000000000000000..8030beafbe5ad71ccb05a76fcde89dfc9bbc2794 GIT binary patch literal 294 zcmXw#L2d#u3`H~IBFH-=(qW`-psQ}WW?!KyGjSkNnxq+nghsRNHF~D1C*T(Bs67I& z{&T*6oNV7-8>Q6Z_it@07LR&aRcfZx?N+IuZ#fX9SUbFaYZSjeNq4p7ixkp!!)A)}oS(>}FOO{NW4XH!V8q@Gyvg(SOmrdS_M@O0)R QXeI-U<8Hj%zd|1S52mV4e5Je4AdX9ibl$d1^6ojV>{jf39a@loP~2Fhsg|qU;Q`w z{vLb#^;s*WW;@o1&0CwLQvWtjlTN|_ literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/dv_envelope_crc_mismatch.puffin b/tests/queries/0_stateless/data_puffin/dv_envelope_crc_mismatch.puffin new file mode 100644 index 0000000000000000000000000000000000000000..941d40183c97470a9f92aa669e266562c56a2c33 GIT binary patch literal 284 zcmXw#F>V4e5JgR-^f>|=QDT;jXhIPwO=@~X;a!h|CC4`20ST@4EIAXRgu^fcVXFVW zzQ4y_eSg(Tsmba5VMi>Uvz#mS;E{S`^F_U?>E-Wp#rYnZkeUvb<8=>o%*YBX1|*-N zK443!=tB+DF*uLG-jf@qE}GnJ6KQYxh_1!L-5z6$I_`fPEKfT**C8=2I1@1b&{jYw zmz1%PN2%?hpu-{!R$D_xJ0I@*R6-S*CSa9KAw8FwvE&O$c56+sKAPa^$W8DL!>}2y J{~sXlyaADaPapsQ literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/dv_envelope_length_mismatch.puffin b/tests/queries/0_stateless/data_puffin/dv_envelope_length_mismatch.puffin new file mode 100644 index 0000000000000000000000000000000000000000..eefdd6fe794f592ed48f886053e825b58c0045fd GIT binary patch literal 284 zcmXw#F>V4e5Je3VeU5-eN{6#7q6r;MYWj%6yB-Hij_r5{B(&PIzCZwi=)%3FmI%Z@A76Xz` zQ6I3SRP>>S*%X||VDHH-Qx{F{c8RpNd_>n`;m;mpi#qPV8!S&dIoBaEEjbe~e$h@q zD3_G6kVmQQp`gPe4OUx2Mmrzw`&2>|nI>SBO(8v(d1J{JlI+%+Vtq8h(~+BC0mHBz KuKyn(@4NvE>QAHq literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/dv_nonzero_snapshot_id.puffin b/tests/queries/0_stateless/data_puffin/dv_nonzero_snapshot_id.puffin new file mode 100644 index 0000000000000000000000000000000000000000..3a186d83bf473bc8f4dbbfff223f6c981674b03a GIT binary patch literal 297 zcmbV|Jr2S!423yL<(be{0tvCPFn32ClE#5ZAqk0Fsj7NXE`w8X0zCP9vY&st+qK>O z9UXTspj=QHF4cVjnj>-s8Xb}iUQ9SpNMdSXUxBgc^j6t3WiMlX1`<|tOIs9tkLU*! zetaoRtcoHBV=@xck-?#V(39bGW{N39Ok}aRE|Ab*Dh^s`Nm3Z=wsq=4HA&^Ljxv#G eN*p-lfmBu-WYX44XK7|9XkcCY_49j7Ir9sWDM}~+ literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/dv_with_compression_codec.puffin b/tests/queries/0_stateless/data_puffin/dv_with_compression_codec.puffin new file mode 100644 index 0000000000000000000000000000000000000000..8a2ccd88e81c3debe12e74a4ef206015db3be968 GIT binary patch literal 326 zcmbV|v2Fq}42JtC$TR725=d3IE}hvCLoWGKB84O*28o{Ty*px~_66$6-yi$8o{H#jcM+(ufjC`2UxtULLiHqlFq8CAmN zitJ$W^*o{d$t8pi7TQWpLjK~D!0t_`wo=hJLTzVi2x#iyjkjz~i0Qe{pfs?Z5{7CU vn~lb+q%x3oH)orV*`~;MaiR?^DwcC?=v{x}iigvu^u49!xcvNHO>gi3qW@Eu literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/fields_element_out_of_int32_range.puffin b/tests/queries/0_stateless/data_puffin/fields_element_out_of_int32_range.puffin new file mode 100644 index 0000000000000000000000000000000000000000..613e15ebba46723d474fa4c9084c4081eabcca06 GIT binary patch literal 311 zcmbVII}XAy4CN@5XF?kUl(Mlfv!f1a<3Oa4gw!CRst4v4I0!C)r}zAM&-U@Km$Sd) z$kQWqE_4Iziqi=6jK~NyIwTvscwkQd$N;W5E z3_+*i5=l72GD4OS-tHKB6ybjDF#pEU+5w4?VvfgfrVE2+%_!xVFq6Y#H9|&%ixi#mEqn`~fGGJHb`R39 z)y!yicC?%O+d&9%@%eS*76gv7UJ3Eb7vh}66LBh*-|rtA@=wylY?@qmkCViG0YcJ2 z1PnQ;9azj&?FPB*qz@3??0h2Dk}hqVQ3XdUYG4W8Mo1P^8ea|6qny2OfW)rJ6Cpk0 zmI<>`bAgHhIChd1JXB6_%4yWfhdB4?nf0hmgudvCYQ=L&kPaeuv%0#FbPTwsBzYwd h2QK+9-&W8_;bu17#b93^^4Oq5XhxPW05|zg_WA7kcCB`Q zqvq)aTI<^ZF4=tnngenL3N4cHPE6R5k7BA|pMf@L)z(?%=Z1yB$uRVugi$;tswClS zMAxBk~jU_U}i3I(D1C zqv!4gMjyulhw8ilJri;U20fCCK|OGynAM|$Z3Wh$H=oLZsRW(M97#FDEvgdYN_%VE;I}7nNp4kGdUFF1u|O9;$f80q?L94vr!||kRClYNoP{6 dM9nFWB-w7#*|?y+qbEB-3+pHN rfaeT9K>GQikyvW^8m1|O7!C{${zhhlm_kG%i}huxwXg4O^Sa;zu+ck^ literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/footer_root_array.puffin b/tests/queries/0_stateless/data_puffin/footer_root_array.puffin new file mode 100644 index 0000000000000000000000000000000000000000..bfb8f877841bd1cbea882315d9749eec748d691a GIT binary patch literal 87 fcmWG=b2MZi8w3D#M;q!W80jb&$8tiA0SN*Cl$Heu literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/generate_puffin_fixtures.py b/tests/queries/0_stateless/data_puffin/generate_puffin_fixtures.py new file mode 100644 index 000000000000..b9000a523cf7 --- /dev/null +++ b/tests/queries/0_stateless/data_puffin/generate_puffin_fixtures.py @@ -0,0 +1,745 @@ +#!/usr/bin/env python3 +"""Generate synthetic puffin test fixtures for ClickHouse stateless tests.""" + +from __future__ import annotations + +import json +import struct +import subprocess +import sys +import zlib +from pathlib import Path + +try: + import lz4.frame + import xxhash +except ImportError: + subprocess.check_call([sys.executable, "-m", "pip", "install", "lz4", "xxhash", "-q"]) + import lz4.frame + import xxhash + +try: + import pyroaring +except ImportError: + subprocess.check_call([sys.executable, "-m", "pip", "install", "pyroaring", "-q"]) + import pyroaring + +OUTPUT_DIR = Path(__file__).parent +PUFFIN_MAGIC = b"PFA1" +DELETION_VECTOR_MAGIC = bytes([0xD1, 0xD3, 0x39, 0x64]) +INFLATED_CONTENT_SIZE = 0x40000000 +# Must stay in sync with PUFFIN_FOOTER_MAX_PAYLOAD_SIZE in PuffinBlockInputFormat.cpp. +FOOTER_MAX_PAYLOAD_SIZE = 16 * 1024 * 1024 +FOOTER_LZ4_MAX_RATIO = 255 +INVALID_KEY = 0x7FFFFFFF +LARGE_KEY = 1_000_000 +SPARSE_SUB_POSITION = 42 +BLOB_PLACEHOLDER = b"\x00" * 58 +DEFAULT_REFERENCED_DATA_FILE = "/data/table/part-00000.parquet" + + +def default_dv_properties(cardinality: str = "0") -> dict[str, str]: + return { + "referenced-data-file": DEFAULT_REFERENCED_DATA_FILE, + "cardinality": cardinality, + } + + +def wrap_deletion_vector_blob(vector: bytes) -> bytes: + combined_length = len(DELETION_VECTOR_MAGIC) + len(vector) + crc_input = DELETION_VECTOR_MAGIC + vector + crc = zlib.crc32(crc_input) & 0xFFFFFFFF + return struct.pack(">I", combined_length) + crc_input + struct.pack(">I", crc) + + +def footer_json_for_blob(blob: bytes, properties: dict[str, str] | None = None) -> bytes: + payload = { + "blobs": [ + { + "type": "deletion-vector-v1", + "fields": [], + "snapshot-id": -1, + "sequence-number": -1, + "offset": 4, + "length": len(blob), + "properties": properties if properties is not None else default_dv_properties(), + } + ] + } + return json.dumps(payload, separators=(", ", ": ")).encode("utf-8") + + +def build_puffin_file( + blob: bytes, + footer_json: bytes, + *, + compressed: bool = False, + lz4_declare_content_size: bool = True, +) -> bytes: + return build_puffin_file_from_blobs( + [blob], + footer_json, + compressed=compressed, + lz4_declare_content_size=lz4_declare_content_size, + ) + + +def build_puffin_file_from_blobs( + blobs: list[bytes], + footer_json: bytes, + *, + compressed: bool = False, + lz4_declare_content_size: bool = True, +) -> bytes: + if compressed: + # Puffin requires a single LZ4 frame; Content Size must be present for valid files. + footer_payload = lz4.frame.compress(footer_json, store_size=lz4_declare_content_size) + flags = bytes([0x01, 0x00, 0x00, 0x00]) + else: + footer_payload = footer_json + flags = b"\x00\x00\x00\x00" + + footer_length = struct.pack(" tuple[bytes, bytes]: + blob_end = puffin.index(PUFFIN_MAGIC, 4) + blob = puffin[4:blob_end] + footer_len = struct.unpack(" int: + return (xxhash.xxh32(descriptor, seed=0).intdigest() >> 8) & 0xFF + + +def set_lz4_content_size(compressed: bytes, content_size: int) -> bytes: + """Replace the Content Size field in an LZ4 frame that already declares it. + + FLG bit 0x08 is Content Size; the previous implementation wrongly used 0x10 + (Block Checksum), which produced frames that fail decompression. + """ + data = bytearray(compressed) + if len(data) < 15: + raise ValueError("LZ4 frame is too short to contain a Content Size field") + if (data[4] & 0x08) == 0: + raise ValueError("LZ4 frame does not declare Content Size") + struct.pack_into(" bytes: + return set_lz4_content_size(compressed, fake_size) + + +def write_fixture(name: str, content: bytes) -> None: + path = OUTPUT_DIR / name + path.write_bytes(content) + print(f"Wrote {path} ({len(content)} bytes)") + + +def generate_compressed_footer(source: Path) -> None: + puffin = source.read_bytes() + blob, footer_json = extract_blob_and_footer_json(puffin) + write_fixture("compressed_footer.puffin", build_puffin_file(blob, footer_json, compressed=True)) + + +def generate_invalid_blob_bounds() -> None: + cases = { + "overflow_offset_length.puffin": (9223372036854775797, 20), + "negative_offset.puffin": (-1, 10), + "length_exceeds_file.puffin": (4, 10_000), + # Offset/length stay within the total file size, but the blob would extend past the + # end of the blob region (i.e. into the footer payload), which must also be rejected. + "blob_overlaps_footer.puffin": (4, len(BLOB_PLACEHOLDER) + 8), + } + for name, (offset, length) in cases.items(): + footer_json = footer_json_for_blob(BLOB_PLACEHOLDER) + payload = json.loads(footer_json.decode("utf-8")) + payload["blobs"][0]["offset"] = offset + payload["blobs"][0]["length"] = length + write_fixture(name, build_puffin_file(BLOB_PLACEHOLDER, json.dumps(payload, separators=(", ", ": ")).encode("utf-8"))) + + +def generate_invalid_roaring_bitmap() -> None: + vector = struct.pack(" None: + vector = struct.pack(" None: + """Corrupt deletion-vector-v1 framing so extractDeletionVectorPayload cannot silently regress.""" + bitmap = pyroaring.BitMap([2, 5]) + vector = struct.pack("I", len(DELETION_VECTOR_MAGIC) + len(vector) + 1) + write_fixture( + "dv_envelope_length_mismatch.puffin", + build_puffin_file(bytes(length_mismatch), footer_json_for_blob(bytes(length_mismatch), properties)), + ) + + bad_magic = bytearray(base) + bad_magic[4:8] = b"XXXX" + write_fixture( + "dv_envelope_bad_magic.puffin", + build_puffin_file(bytes(bad_magic), footer_json_for_blob(bytes(bad_magic), properties)), + ) + + crc_mismatch = bytearray(base) + crc_mismatch[-1] ^= 0xFF + write_fixture( + "dv_envelope_crc_mismatch.puffin", + build_puffin_file(bytes(crc_mismatch), footer_json_for_blob(bytes(crc_mismatch), properties)), + ) + + +def generate_inflated_lz4_content_size(source: Path) -> None: + puffin = source.read_bytes() + blob, footer_json = extract_blob_and_footer_json(puffin) + footer_payload = set_lz4_content_size(lz4.frame.compress(footer_json, store_size=True), INFLATED_CONTENT_SIZE) + footer_length = struct.pack(" None: + """contentSize passes size*255 but exceeds the absolute decompressed-footer cap. + + Pads the compressed footer payload so the ratio guard would allow the forged size; + the parser must reject on the absolute ceiling before allocating. + """ + puffin = source.read_bytes() + blob, footer_json = extract_blob_and_footer_json(puffin) + forged_content_size = FOOTER_MAX_PAYLOAD_SIZE + 1 + min_payload_for_ratio = (forged_content_size + FOOTER_LZ4_MAX_RATIO - 1) // FOOTER_LZ4_MAX_RATIO + footer_payload = bytearray( + set_lz4_content_size(lz4.frame.compress(footer_json, store_size=True), forged_content_size) + ) + if len(footer_payload) < min_payload_for_ratio: + footer_payload.extend(b"\x00" * (min_payload_for_ratio - len(footer_payload))) + footer_length = struct.pack(" None: + footer_json = footer_json_for_blob(BLOB_PLACEHOLDER) + write_fixture( + "missing_lz4_content_size.puffin", + build_puffin_file( + BLOB_PLACEHOLDER, + footer_json, + compressed=True, + lz4_declare_content_size=False, + ), + ) + + +def generate_lz4_trailing_bytes() -> None: + """Valid single LZ4 frame plus trailing garbage; FooterPayloadSize includes the junk.""" + footer_json = footer_json_for_blob(BLOB_PLACEHOLDER) + footer_payload = lz4.frame.compress(footer_json, store_size=True) + b"GARBAGE" + footer_length = struct.pack(" None: + """Truncated LZ4 frame: header parses with content size, body is incomplete (must not hang).""" + footer_json = footer_json_for_blob(BLOB_PLACEHOLDER) + full_frame = lz4.frame.compress(footer_json, store_size=True) + # Keep enough bytes for LZ4F_getFrameInfo to succeed, but truncate the body. + footer_payload = full_frame[:19] + footer_length = struct.pack(" None: + # One missing blob field covers requireBlobMetadataField; keep distinct blobs / + # properties / DV-property cases that hit different branches. + footer_json = footer_json_for_blob(BLOB_PLACEHOLDER) + payload = json.loads(footer_json.decode("utf-8")) + del payload["blobs"][0]["type"] + write_fixture( + "missing_type.puffin", + build_puffin_file( + BLOB_PLACEHOLDER, + json.dumps(payload, separators=(", ", ": ")).encode("utf-8"), + ), + ) + + footer_field_cases = { + "missing_blobs.puffin": {}, + "null_blobs.puffin": {"blobs": None}, + "null_blob_entry.puffin": {"blobs": [None]}, + "invalid_blob_entry.puffin": {"blobs": ["not-an-object"]}, + } + for name, footer_payload in footer_field_cases.items(): + write_fixture( + name, + build_puffin_file( + BLOB_PLACEHOLDER, + json.dumps(footer_payload, separators=(", ", ": ")).encode("utf-8"), + ), + ) + + dv_property_cases = { + "missing_properties.puffin": None, + "null_properties.puffin": "null", + "missing_referenced_data_file.puffin": {"cardinality": "0"}, + "missing_cardinality.puffin": {"referenced-data-file": DEFAULT_REFERENCED_DATA_FILE}, + # Array covers the non-object branch; string would be the same check. + "invalid_properties_array.puffin": [], + } + for name, properties in dv_property_cases.items(): + case_payload = json.loads(footer_json.decode("utf-8")) + if properties is None: + del case_payload["blobs"][0]["properties"] + elif properties == "null": + case_payload["blobs"][0]["properties"] = None + else: + case_payload["blobs"][0]["properties"] = properties + write_fixture( + name, + build_puffin_file( + BLOB_PLACEHOLDER, + json.dumps(case_payload, separators=(", ", ": ")).encode("utf-8"), + ), + ) + + # Present compression-codec (including null) must be rejected for DV; one fixture is enough. + case_payload = json.loads(footer_json.decode("utf-8")) + case_payload["blobs"][0]["compression-codec"] = "lz4" + write_fixture( + "dv_with_compression_codec.puffin", + build_puffin_file( + BLOB_PLACEHOLDER, + json.dumps(case_payload, separators=(", ", ": ")).encode("utf-8"), + ), + ) + + +def generate_invalid_property_value_types() -> None: + """Property maps must have string values; non-strings must fail with BAD_ARGUMENTS.""" + footer_json = footer_json_for_blob(BLOB_PLACEHOLDER) + base = json.loads(footer_json.decode("utf-8")) + # One extra-key non-string covers the shared !isString check; also require a wrong-typed + # required DV key so that path stays covered. + cases = { + "invalid_property_number.puffin": {**default_dv_properties(), "ndv": 5}, + "invalid_property_cardinality_number.puffin": { + "referenced-data-file": DEFAULT_REFERENCED_DATA_FILE, + "cardinality": 5, + }, + } + for name, properties in cases.items(): + case_payload = json.loads(json.dumps(base)) + case_payload["blobs"][0]["properties"] = properties + write_fixture( + name, + build_puffin_file( + BLOB_PLACEHOLDER, + json.dumps(case_payload, separators=(", ", ": ")).encode("utf-8"), + ), + ) + + +def generate_invalid_integer_fields() -> None: + """BlobMetadata integer fields must be JSON integers, not floats, strings, or booleans. + + Poco reports JSON booleans as integers (`std::numeric_limits::is_integer`), so + boolean cases must be rejected explicitly in the reader. Keep one scalar and one fields[] + example per JSON type / range branch rather than one file per field name. + """ + footer_json = footer_json_for_blob(BLOB_PLACEHOLDER) + base = json.loads(footer_json.decode("utf-8")) + cases = { + "float_offset.puffin": ("offset", 5.1), + "float_fields_element.puffin": ("fields", [1.9]), + "string_offset.puffin": ("offset", "4"), + "bool_offset.puffin": ("offset", True), + "bool_fields_element.puffin": ("fields", [True]), + "fields_element_out_of_int32_range.puffin": ("fields", [2**40]), + # Poco stores integers that fail signed Int64 parse as UInt64 (2**63 wraps in tryParse64). + "offset_out_of_int64_range.puffin": ("offset", 2**64 - 1), + "fields_element_out_of_int64_range.puffin": ("fields", [2**64 - 1]), + } + for name, (field, value) in cases.items(): + case_payload = json.loads(json.dumps(base)) + case_payload["blobs"][0][field] = value + write_fixture( + name, + build_puffin_file( + BLOB_PLACEHOLDER, + json.dumps(case_payload, separators=(", ", ": ")).encode("utf-8"), + ), + ) + + +def generate_invalid_string_fields() -> None: + """BlobMetadata type / compression-codec must be JSON strings.""" + footer_json = footer_json_for_blob(BLOB_PLACEHOLDER) + base = json.loads(footer_json.decode("utf-8")) + + case_payload = json.loads(json.dumps(base)) + case_payload["blobs"][0]["type"] = 123 + write_fixture( + "type_number.puffin", + build_puffin_file( + BLOB_PLACEHOLDER, + json.dumps(case_payload, separators=(", ", ": ")).encode("utf-8"), + ), + ) + + # Non-DV blob so DV omit-codec logic does not hide the type error. + # Number covers !isString; null covers present-null for the optional codec field. + theta_cases = { + "compression_codec_number.puffin": 1, + "compression_codec_null.puffin": None, + } + for name, codec_value in theta_cases.items(): + case_payload = { + "blobs": [ + { + "type": "apache-datasketches-theta-v1", + "fields": [], + "snapshot-id": -1, + "sequence-number": -1, + "offset": 4, + "length": len(BLOB_PLACEHOLDER), + "compression-codec": codec_value, + } + ] + } + write_fixture( + name, + build_puffin_file( + BLOB_PLACEHOLDER, + json.dumps(case_payload, separators=(", ", ": ")).encode("utf-8"), + ), + ) + + +def generate_cardinality_mismatch_large_bitmap() -> None: + bitmap = pyroaring.BitMap([2, 5, 7, 100, 65536]) + vector = struct.pack(" None: + """RLE-dense roaring: tiny on-disk blob, 100k consecutive positions — must remain readable.""" + bitmap = pyroaring.BitMap() + bitmap.add_range(0, 100_000) + vector = struct.pack(" None: + """Declared cardinality above the absolute materialization ceiling must be rejected early.""" + bitmap = pyroaring.BitMap([0]) + vector = struct.pack(" None: + """cardinality must parse as UInt64; invalid strings must fail with BAD_ARGUMENTS.""" + bitmap = pyroaring.BitMap([2, 5]) + vector = struct.pack(" None: + """Iceberg requires deletion-vector-v1 snapshot-id and sequence-number to be -1.""" + footer_json = footer_json_for_blob(BLOB_PLACEHOLDER) + base = json.loads(footer_json.decode("utf-8")) + # Shared check rejects either field; one nonzero fixture is enough. + case_payload = json.loads(json.dumps(base)) + case_payload["blobs"][0]["snapshot-id"] = 1 + write_fixture( + "dv_nonzero_snapshot_id.puffin", + build_puffin_file( + BLOB_PLACEHOLDER, + json.dumps(case_payload, separators=(", ", ": ")).encode("utf-8"), + ), + ) + + +def generate_sparse_large_key() -> None: + bitmap = pyroaring.BitMap() + bitmap.add(SPARSE_SUB_POSITION) + vector = struct.pack(" None: + theta_blob = b"\x00" * 16 + bitmap = pyroaring.BitMap([2, 5]) + vector = struct.pack(" None: + theta_blob = b"\x00" * 16 + bitmap = pyroaring.BitMap([2, 5]) + vector = struct.pack(" None: + flags = b"\x00\x00\x00\x00" + footer_length = struct.pack(" None: + """FileMetadata must be a JSON object. + + Poco's RFC 4627 parser accepts array as a top-level value, so this exercises the + Object::Ptr guard. String/number roots fail earlier inside the parser itself. + """ + write_raw_footer_fixture("footer_root_array.puffin", b"[1, 2, 3]") + + +def generate_missing_footer_leading_magic() -> None: + """Footer must start with its own Magic; header Magic must not double as footer-open Magic. + + Layout Magic | FooterPayload | Size | Flags | Magic is invalid even when blobs is empty. + """ + footer_payload = b'{"blobs": []}' + flags = b"\x00\x00\x00\x00" + footer_length = struct.pack(" None: + """Optional FileMetadata.properties must be a JSON object with string values.""" + bitmap = pyroaring.BitMap([2, 5]) + vector = struct.pack(" None: + """Malformed JSON / oversize integers must fail with BAD_ARGUMENTS, not STD_EXCEPTION.""" + write_raw_footer_fixture("malformed_footer_json.puffin", b"{") + # Larger than UInt64::max; Poco NumberParser throws SyntaxException. + write_raw_footer_fixture( + "footer_integer_overflow.puffin", + b'{"blobs": [{"type": "apache-datasketches-theta-v1", "fields": [], ' + b'"snapshot-id": 99999999999999999999999999999999, "sequence-number": 1, ' + b'"offset": 4, "length": 4, "properties": {}}]}', + ) + + +def main() -> None: + spark_fixture = OUTPUT_DIR / "spark_deletion_vector.puffin" + if not spark_fixture.exists(): + raise SystemExit(f"Missing {spark_fixture}; run generate_spark_puffin.py first") + + generate_compressed_footer(spark_fixture) + generate_invalid_blob_bounds() + generate_invalid_roaring_bitmap() + generate_invalid_bitmap_key() + generate_invalid_dv_envelope() + generate_inflated_lz4_content_size(spark_fixture) + generate_lz4_content_size_within_ratio_over_absolute_cap(spark_fixture) + generate_missing_lz4_content_size() + generate_lz4_trailing_bytes() + generate_incomplete_lz4_footer() + generate_missing_required_fields() + generate_invalid_property_value_types() + generate_invalid_integer_fields() + generate_invalid_string_fields() + generate_invalid_footer_root() + generate_missing_footer_leading_magic() + generate_invalid_file_metadata_properties() + generate_unparseable_footer_json() + generate_mixed_blob_types() + generate_invalid_non_dv_properties() + + generate_cardinality_mismatch_large_bitmap() + generate_dense_range_100k() + generate_cardinality_exceeds_materialization_limit() + generate_invalid_cardinality_strings() + generate_invalid_dv_snapshot_sequence() + generate_sparse_large_key() + + +if __name__ == "__main__": + main() diff --git a/tests/queries/0_stateless/data_puffin/generate_spark_puffin.py b/tests/queries/0_stateless/data_puffin/generate_spark_puffin.py new file mode 100644 index 000000000000..375b15b9b9be --- /dev/null +++ b/tests/queries/0_stateless/data_puffin/generate_spark_puffin.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Generate a Puffin deletion-vector file using Apache Spark + Iceberg v3. + +Requirements: + - Java 11+ + - pyspark + - Network access on first run (downloads iceberg-spark-runtime) + +Usage: + python3 generate_spark_puffin.py + python3 generate_puffin_fixtures.py +""" + +from __future__ import annotations + +import shutil +import tempfile +from pathlib import Path + +try: + from pyspark.sql import SparkSession +except ImportError as exc: # pragma: no cover - helper script + raise SystemExit("pyspark is required") from exc + +OUTPUT = Path(__file__).with_name("spark_deletion_vector.puffin") +ICEBERG_PACKAGE = "org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.9.0" +DELETED_IDS = [2, 5, 7, 100, 65536] + + +def find_puffin_file(warehouse: Path) -> Path: + puffin_files = sorted(warehouse.rglob("*-deletes.puffin")) + if len(puffin_files) != 1: + raise RuntimeError(f"Expected exactly one puffin file, found {len(puffin_files)}: {puffin_files}") + return puffin_files[0] + + +def build_spark_session(warehouse: Path) -> SparkSession: + return ( + SparkSession.builder.appName("generate_spark_puffin") + .master("local[1]") + .config("spark.jars.packages", ICEBERG_PACKAGE) + .config( + "spark.sql.extensions", + "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions", + ) + .config("spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") + .config("spark.sql.catalog.spark_catalog.type", "hadoop") + .config("spark.sql.catalog.spark_catalog.warehouse", str(warehouse)) + .config("spark.ui.enabled", "false") + .getOrCreate() + ) + + +def generate_puffin(warehouse: Path) -> Path: + spark = build_spark_session(warehouse) + try: + spark.sql( + """ + CREATE TABLE default.spark_puffin_source (id BIGINT) + USING iceberg + TBLPROPERTIES ( + 'format-version' = '3', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ) + """ + ) + spark.sql("INSERT INTO default.spark_puffin_source SELECT id FROM range(0, 70000)") + spark.sql( + "DELETE FROM default.spark_puffin_source " + f"WHERE id IN ({', '.join(str(x) for x in DELETED_IDS)})" + ) + finally: + spark.stop() + + return find_puffin_file(warehouse) + + +def main() -> None: + with tempfile.TemporaryDirectory(prefix="clickhouse_spark_puffin_") as tmp: + warehouse = Path(tmp) + puffin_path = generate_puffin(warehouse) + shutil.copyfile(puffin_path, OUTPUT) + print(f"Wrote {OUTPUT} ({OUTPUT.stat().st_size} bytes) from {puffin_path}") + + +if __name__ == "__main__": + main() diff --git a/tests/queries/0_stateless/data_puffin/incomplete_lz4_footer.puffin b/tests/queries/0_stateless/data_puffin/incomplete_lz4_footer.puffin new file mode 100644 index 0000000000000000000000000000000000000000..2e45030e78b6c239b68b369578203b3940004773 GIT binary patch literal 97 lcmWG=b2MZi8w3D#vncsWWH{V`nqhYd$Pfl%Mj!@B0|3!~2220| literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/inflated_lz4_content_size.puffin b/tests/queries/0_stateless/data_puffin/inflated_lz4_content_size.puffin new file mode 100644 index 0000000000000000000000000000000000000000..27f2545a8ff92134ebcf03d0efd1700135822ce6 GIT binary patch literal 497 zcmXw0&5G1O5UyPnuX_;|gfaBR6{a`IB$@QY%kD++psZ(EhW@GEX*QFzozBWK>>DWf z2!b!7kD$KD;7#ygMX;z3sz2(dtG1v+56d!skT=GYu@fyy!FT$X0XNR`i|eAj(y-aIh!-< zur*KN*KCoObt#HvSzXTA&`CdR`pBJO4bO?dcO!Nhc{gqqhDWe}+*>#8V<(f{q!e6SG>!|)^L-{kST<@*k5AL|yr#@`*#H6c4N=#g9u>Vgf$ ztS%Ky$6y_L^Xl9&wLz!$5J@@1PgEr(Je@H#sKWVJV}6&TwJj3Ul6yRcJslVpE2flV z!b}b&wu6iovm_X0G-+j>f6Y@0m85HrWzw0{Tr^|LACk=0=xkij-qDqlU=Dr1>+iBC HOZI#Mr$i;?Cg$npCuOB3mng;7%0Nv9sRIC)Aq>g@ literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/invalid_cardinality_negative.puffin b/tests/queries/0_stateless/data_puffin/invalid_cardinality_negative.puffin new file mode 100644 index 0000000000000000000000000000000000000000..7b7e3bb2dbee6e40aa143dc01d5ebe170a835568 GIT binary patch literal 285 zcmXw#F-`+95JerN^c*1?rOdEdiH3AE)aj!X-u0MRGPdKLM2c2>7S6&E5GCR+%mgyk ze_!9s zLr!?6o^!=>3#$eAK+#>3d!ZrOG8{5vaB>7&wdi=F)Dx!DTSxU~XYU7Mp-0I?O0RrS zP%1SSsvJmbN2uhfYJ)>(*`g2ex=*9ju#a70(&Vxv~kaf4-Ru__ufiKbo*upgJ Lr=R~H(1-p4S-nrt literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/invalid_cardinality_non_numeric.puffin b/tests/queries/0_stateless/data_puffin/invalid_cardinality_non_numeric.puffin new file mode 100644 index 0000000000000000000000000000000000000000..4e784be4e8e9e5d732f9a53e1284507155f6a880 GIT binary patch literal 295 zcmXw#F^`+Q!#(s?mP1LvhlI$~Q2rZ~^J~fcXb+onvBwCWkLpb1( zfhpyfAfp40?PUfFnG=jM8nv>{zxtF+CF;gQnRG_A;Q1m+7b16Sbv7<&@9;`VGEgr) O))V4e5JgR-^f>|=rNmh_qRDn?Qqv;}?|KZD9NX~@NNBZZ$yqo8qT~)aLuN=Q znCidL_xHD#AMdqN>iKkjvlAA#Y354(@kBlG@lA5HLd;3Rm3 Rag>sG`K$H#^ZzCC@?ZaFRipp_ literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/invalid_file_property_number.puffin b/tests/queries/0_stateless/data_puffin/invalid_file_property_number.puffin new file mode 100644 index 0000000000000000000000000000000000000000..90fe9010006cd7b1d51fa8cbf1ea7ea12433f33d GIT binary patch literal 330 zcmZ8dF-`+95Ok1esCYuOqKvs5f`W83)aen0eO@OP#x}mqNzv)v!drL(qU1B|9;9Kb znbGX*Xt$3Kty1dh^XuL%7@TIaR_d28)Fq2&>Rhe9-#@nOAHl|Kd)O?WMxbLsM&K|a zg`D(&9p$Qr7FJ8}0i&Cp&&>K{O25yfgX0z5iiB?mOdaZUc(qtha`wJQVs^`(h-pWA zA;L<{1uF%V*bypvtdiigGi0<6aqiOyYslD$O)-`9TIQM|A0+A4nd(9^G0>5d;0C5i dO8(35DjMQ&JDc*0Y$iAG#@#es{Jl)R@&o!5T?7CC literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/invalid_non_dv_properties_array.puffin b/tests/queries/0_stateless/data_puffin/invalid_non_dv_properties_array.puffin new file mode 100644 index 0000000000000000000000000000000000000000..0e570e10a1a84e58a28ce156255b050845bb01ec GIT binary patch literal 445 zcmbVJK~BRk5Ok5a^MuIa7?Y%`km!vI=iX5cPP`@-u5IjXr7Fr>cneQJ9Qh3E0HUb3 z?rCP^wPt2F5BCi_o$L3HJGD5{*EM6`e8H|rzhIZF{Cs`e(Ei9f>$}9)#pB5F5P%Su zLG}X(B~hklK*CAj04T-2;bpn&;Cc{lx9uSQ%3kXP zNNh!KEVw5eh%~pvLi7P58kqlWr?U9-qrd_h@5CN@^zp>SnI@L?v5Qsq^v}h}BWQ>@ xhnmX}+Ss|EDpymX>?~A4Mih0P8-f%Yb49(3$~b8a9w;GS@@ZfrQvtn7gA6O>?1> m$MI&6e=<2&z(PCm1jH*30{eqd%9N1V<8yXtm*@AJwc87}?=c(z literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/invalid_property_cardinality_number.puffin b/tests/queries/0_stateless/data_puffin/invalid_property_cardinality_number.puffin new file mode 100644 index 0000000000000000000000000000000000000000..e9dedbbd75cc9d5729c886f021b91d0df33eafed GIT binary patch literal 296 zcmbV|Jr2S!423yLm6=c~frQvtn7gA6N#j7IkOarAR8_qwr@^V<0(kQGWIzA3*Gsec zJ6i6Zh4xWrajtH&peX@o!hi>IF~|{miYUiMY%5_Mc(Yb^OgX5OACaUp+#;JI!E1of zgABt{1ODV_Z3bdGaE*s>rw7CF#FQ{5KypxQXMtb=i-VCyk&@Q=b)ULWO-g&HlR}dF c5<512%q26#xJL literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/invalid_property_number.puffin b/tests/queries/0_stateless/data_puffin/invalid_property_number.puffin new file mode 100644 index 0000000000000000000000000000000000000000..e61998faf0952043ced97e9608d45b6c7967fcba GIT binary patch literal 308 zcmbV|Jr2S!423yLm6_010tvCPFn32C(#D~YLJ|_UQdRZ3oCCXp3*gD$ll}bD+^+TR z?`Wud5uFQN7MJ3_2%IBOCNwxe8@!yL$B^Vyi+v%C1*f;lmMD7_^AJ#4O)au!30@=k z9;6?i9i*QejLCpRN2+n)A2<-4&O|YV2nj8)?IIEwNbI1cR;Z-0Zri71Dp7R~%BT{` jisL{j4Mg6pS4ms1oW+@(M8%rd6VF-Kc71<;#20@6z6ngn literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/invalid_roaring_bitmap.puffin b/tests/queries/0_stateless/data_puffin/invalid_roaring_bitmap.puffin new file mode 100644 index 0000000000000000000000000000000000000000..fb818b09610cdd80b069febeea7fa7a3eef8b13c GIT binary patch literal 269 zcmXw!F%H5o3`NT!*pX*KDPn<*g@K73F{F(HkwOwuw-Ty)1r|<*Sh)!UI0*jqpY`)& zZ`Z3L%d+A9u}}jxf4y1R>vr92InY6AeObeDwCjK_0oj2NVH8i{Gn8H?^fM~2ygDMTMIlEGBlK}3x) yH7KDZNuiDX?vp3VNtPD#AR~!s(u6I4NNKl9MrE9|hA!L$Q|SAB|NLE??)(4{$Wsjf literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/length_exceeds_file.puffin b/tests/queries/0_stateless/data_puffin/length_exceeds_file.puffin new file mode 100644 index 0000000000000000000000000000000000000000..f06f02e694d161b64fb69f73991fb43410ebd220 GIT binary patch literal 301 zcmbVIF%H5o4CE=5XF^+viH(K1JL-@$4viF&khqnqs&C~hI2C*VPrj4ye0F!cwuir~ zWAzHk1(o4a-B+L`B6pzCA=%)?j01%v<`#|>7>iEtlRcyCWz1tBVKqxcTPS=@=m!*j zdMZpmIvA6Y7@c{>p?}cGpl!IAh$%!&WU)A}kkDW%CR%7oQW)#@NOeIqN#(GPGLb0q d=EN=Uq!MqCNn01KDc>!rOEJSg=nWl1u>bVIXAOeU35`UVO< zf{&t)puR})rg(5H_zIrB!wmC>-^_33GbhJKky7g5>-VJ{`N6-k3YFTqo!n7>*LQsz z`?jO*sr$Tts@Ji&854ME%%1h~h<@Oy}|JbfWt@ZTn5*OlkGXM1Dcu4pQf0>S3En_fXwD zYKo#4r&sYrS5lw3jb6=?iEg{5mClvob6@FB3hAD!PuAGfrKTvW59;URz-`+gugd&< z(+s_=`!?;)*X>XgWxZZX`EFgtNnoW&hsp(es_mPh%O!ZF#z8ppxlL%I*)&i2_a-=FXA@BDl7w6kTMR;9c2Br|<|efn8_%sWY) zSd(dWb9mII`9{Vsa`7)c8xM8gN?v*>JARrs+imH_3!9}y7EcqANal$R!+4QNmS?kM zaWJ3Rg-v`-qi6nn|5FqM5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** z5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL jKmY**5I_I{1Q0*~0R#|00D=D_urnH|kxzAUd=&iw*h-E} literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/lz4_trailing_bytes.puffin b/tests/queries/0_stateless/data_puffin/lz4_trailing_bytes.puffin new file mode 100644 index 0000000000000000000000000000000000000000..18763d94df2ddc8b099b3656c89ae19b9c65b4e2 GIT binary patch literal 318 zcmbV{u}TCn5QgVKEkzK~@=Phly#)oa5cfc@5)K;+xgxunxPh3&?A*bzEcggMfQ@e? z)|NiN`UcK|PvF47_cQ!6^UrUJ9R6K%c5jH=gZk?0Z^z{)!_Q$WN*BsTT#O%DLFIT-(0Y0yoya7NLtMfneW0kJ}=Q%DE;!O#Iz6nUNELX z;`u2%9QsG9S)84-iYY`)WbqziD-s$^Q+U@>D^k)}_X<1NHhg5HN{3lgiR5GiFDFcV khTfpxs#Ma}D`#mvJUtOzx9Em+v*K=A%&vRJ5%-Y&7lf-%2mk;8 literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/malformed_footer_json.puffin b/tests/queries/0_stateless/data_puffin/malformed_footer_json.puffin new file mode 100644 index 0000000000000000000000000000000000000000..cace309ac9f5e07f40de2281079337af295457e6 GIT binary patch literal 79 WcmWG=b2MZi8w3D#S2IGjgE#|7GXi5s7Bl;eN zA0G*~Vm_o!v7E9Y15*kdUgBDtn6vn!BpSn;@QaP-nOeFRt Rb}V@y!LoGA`}>7*(+3ytL{I<# literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/missing_footer_leading_magic.puffin b/tests/queries/0_stateless/data_puffin/missing_footer_leading_magic.puffin new file mode 100644 index 0000000000000000000000000000000000000000..59370e33b82e30747de0beda825b7229341da0df GIT binary patch literal 29 ecmWG=b2O}0O3KMkDps;mh>or0Wq^PHpacMOq6U5d literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/missing_lz4_content_size.puffin b/tests/queries/0_stateless/data_puffin/missing_lz4_content_size.puffin new file mode 100644 index 0000000000000000000000000000000000000000..ab820d3be7ca5fffe582910b2e99481379e80871 GIT binary patch literal 303 zcmbVIK}rKb5bS_@6d{NnV;{wFv!EcJ)Ppx6HxCkUcBjV-j5FhGClHne@dZA>i$4;7 zAozgw2ljp;4Fy#+)z#gz+v^fGu9>*E@b2*O>g`*G{2jGe`B*i0IeBQ2*NKFoAy4dL zP%ElAX0_4ToOmF}I literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/missing_properties.puffin b/tests/queries/0_stateless/data_puffin/missing_properties.puffin new file mode 100644 index 0000000000000000000000000000000000000000..e1308a29a6ceaace9340597e12a5bb58d9bc655c GIT binary patch literal 204 zcmWG=b2MZi8w3D#S1Tpu@I^D7COOH7_|;H?K4|DYXbDoS&9foLU0p anE=(~q~@iUWB@s)7PYapbx^m1Tmk@4b3>?@7uO@UHlA21|7t~ug^Kmvp+3T2xz{(nlsB)9=9N{}C yKR#NdpB(x=gM}*bIQTmc0>_h3Od%q%#dda~PA6l%c9v&hI?%c{>+Ad9*7gmJW;YZ7 literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/missing_type.puffin b/tests/queries/0_stateless/data_puffin/missing_type.puffin new file mode 100644 index 0000000000000000000000000000000000000000..d3a741b6e0adcfbb34fcb8ce999cb7f43330f1b6 GIT binary patch literal 268 zcmbV{F%H5o3`IFg<(be{0tvCPFn32ClE$HtLJ|^pKvk~B*>DhCfIsA+TD>XaO*EM6`e8z4_zhc*{{Cs`e(Ei9< y9BL{<^x8}XRk@l9Wow}FGN7pQ+!CbNm@DdJP})kP@kn|2f~T}iyPvD%;=ceQXnfxQ literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/negative_offset.puffin b/tests/queries/0_stateless/data_puffin/negative_offset.puffin new file mode 100644 index 0000000000000000000000000000000000000000..94a3818915795910ee158e3d1e0919871504d530 GIT binary patch literal 299 zcmbVIF%E+;4D3^o=cd5gtxIS2iXkCRiIgU3VvrExsr^eGR6bBozLW2Kc6YzEr@yOX z^$N-bmEl@FR-h#!cc9TB+2F;D1BE2!7S0tIi%##8J)`Vp%wr&7HA_TWD11-o2NZsK zDNF|)jLAp`a_FBlGHM%^#1tYXvRHgqNN6w>3oW!HDU5Y{pt_)%q;gnCnMgFHF5L1) ZD&Yp1wDromjq-eEg;Vc{hQE~~6z)T!A%@D`qcIPw|R0Yp)6 z>}h7?wPt3w_je6DpX;~xwOX9$o0_pNzF=3RU$9G7e!RSHX@BIc^=;yt;$h@?3_ys> zAiExfk|@&?AmJpi2bAK_@Ur0AfK~HOvm=_E45{}hjG}GPWC@-Ja2*IY>|2OGve!BR z5?j$53+@qnBF!za5Pg7%2J*jduy!hoUq=copz%%|phF+eeEieKvOe{(%C7$S7^E)y>hlbCD{&?{4@eRBXR~B9g+=RJg}pX#6!Yj4#uL>pUE?$-mBPO0|~2{B4jS%eL&x# z@WZXe^eYErdL%}2Q&sDxs>){7l-s(NYgseRp})|T`}KxVOd(<-i+R%s2@R$!s)bgh tgt6{pvB;<()jBMqN+il0OHSDXWxJh9+Ir65L^IHezxE9vk&`SHTgRZ z%pRb1q3vNiJq|$6h@62&hh&2n59}x;@u*-n1!K|a&*X_w?^WzCfrQme5mheX?T)@f z;qTWL)2|$i>5&+1>bhxUT{f%rav|kXZX~lD`ZHa)U6+hv3K0`o%!>v{XfS0>Ewmyf sjCCJ}MMedw)?pD&1) || true + echo "$err" | grep -oF "$needle" || true + if [[ -n "$extra" ]]; then + echo "$err" | grep -oF "$extra" || true + fi + ;; + raw_puffin) + echo "--- $path_or_label ---" + err=$($CLICKHOUSE_LOCAL -q "$extra" 2>&1) || true + echo "$err" | grep -oF "$needle" || true + ;; + meta) + echo "--- $(basename "$path_or_label") ---" + $CLICKHOUSE_LOCAL -q "SELECT blob_type FROM file('$path_or_label', PuffinMetadata)" 2>&1 \ + | grep -oF "$needle" || true + ;; + meta_re) + echo "--- $(basename "$path_or_label") ---" + $CLICKHOUSE_LOCAL -q "SELECT blob_type FROM file('$path_or_label', PuffinMetadata)" 2>&1 \ + | grep -oE "$needle" || true + ;; + raw_meta) + echo "--- $path_or_label ---" + err=$($CLICKHOUSE_LOCAL -q "$extra" 2>&1) || true + echo "$err" | grep -oF "$needle" || true + ;; + *) + echo "unknown puffin error case kind: $kind" >&2 + return 1 + ;; + esac + } > "$out" +} + +launch() +{ + run_case "$@" & + pids+=($!) + while [[ ${#pids[@]} -ge $PARALLEL ]]; do + wait_one + done +} + +finish_puffin_errors() +{ + while [[ ${#pids[@]} -gt 0 ]]; do + wait_one + done + + local i + for ((i = 0; i < id; i++)); do + cat "$TMP/$i.out" + done +} diff --git a/tests/queries/0_stateless/data_puffin/spark_deletion_vector.puffin b/tests/queries/0_stateless/data_puffin/spark_deletion_vector.puffin new file mode 100644 index 0000000000000000000000000000000000000000..4a0d3e8a7e101cb50ab448311bc2300566c52fb8 GIT binary patch literal 499 zcmY*WOK#jS3{_GDy+W3TRxL29t+71Tc$ZCp?viy74D_KgLOhlt%PE3Ej?;tm2yHJE zDxXCN5g@5Y@{zo{yx@e8_22(ndxFM?3=JWtug(`z;^(Lp>M1!RUx`KQ`cM7u3bT(? zk3r9LxH;1N6oC%Z!U)-ii9LeJA+bkJSJZhJZGo-1=!I-IqPtzuc~WuShs?aC1LGKg zhXp1BY+4Q)(pxY%H$$d_SkV!tyS%4ECs#D45Fup`SoTO0I7lVcGOMy;&X0vmoe!}# zqc`{au*|TVqe}O?xH#vh-4^_>ofopU;M6ka`qS?mmSmtV)#|1QYZ&iZ*~7MF%?4T~ zq}EJn*)vc=JFvPj+#;-?`44P`(`209sAT=5MxURMQ%9elarn>tO#+pXye^1~sLUR! zp9XLVchxe@5sb$j+UlDLhl9^mZ?$q-v=V^BM%tzky#{S`C;RoLv%Qu06#tIRe!t(g G9P9=9F_a_# literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/sparse_large_key.puffin b/tests/queries/0_stateless/data_puffin/sparse_large_key.puffin new file mode 100644 index 0000000000000000000000000000000000000000..e927acc5628db3d2b7040a5896c06dbcf0984096 GIT binary patch literal 282 zcmXw!F-`+95JiU*kRwD>qL|GhnxsI1f;v4);a!gtOU5?dNu+4CheWj8gO-_N!_)X>qkgA1Jyxc@P?sEyF3Z2~M7vRxSEEQ|ig2^Jho(W@qmQVxecrL`p|KDLe_4 znhRA9q_rbf@>I3K8E4rVAL6`EqtvkNA~nTUHfxF literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/string_offset.puffin b/tests/queries/0_stateless/data_puffin/string_offset.puffin new file mode 100644 index 0000000000000000000000000000000000000000..704412f86fa0341c0b9df08ff234f348b22a16f3 GIT binary patch literal 300 zcmbV|!3x4K42FG_(X(|MD2O*No_CiWx;1r$+SW81jImesS!{;BfH~z4$@eE|Zr6JE zcQko=5uFQN5|`q>2%010OlWXGHh4KfPXXmri+v%C1*f;pHml@SOhX`Pbsmv5m*6#m z??L+U*#Z8_!I%WZ+#_Qg_y-MHp3YfC3=xnlFnAFN29OzAYDG#K>$Z(bRwb##NrLYlP08jp&?PuBD<=n3R zj*g~hp?%avoT}?Aa7sW~yWOoTVI6q$DfC1os8sF|r8CqaTbAHCLKr}X@u4CA Date: Tue, 28 Jul 2026 09:49:45 +0200 Subject: [PATCH 2/5] Fix count-from-files cache guard polarity for `_headers`. Restore `!headers_requested` after a merge-conflict resolution inverted the condition, which disabled the cache for ordinary counts and allowed wrong headers on cache hits. Co-authored-by: Cursor --- src/Storages/ObjectStorage/StorageObjectStorageSource.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index c79da9e5100e..c10ff9a4ebe6 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -774,7 +774,7 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade const bool effective_need_only_count = need_only_count; #endif - const bool can_use_count_cache = effective_need_only_count && headers_requested + const bool can_use_count_cache = effective_need_only_count && !headers_requested && context_->getSettingsRef()[Setting::use_cache_for_count_from_files] && canUseCountFromFilesCache(object_info); From ece3851f45538c667d49df43ffe7ad14af46581a Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 28 Jul 2026 09:56:14 +0200 Subject: [PATCH 3/5] Add required sidebar frontmatter to Puffin format docs. Include `sidebar_label`, `sidebar_position`, and input/output flags so the new pages match AGENTS.md and peer format docs. Co-authored-by: Cursor --- docs/en/interfaces/formats/Puffin/Puffin.md | 4 ++++ docs/en/interfaces/formats/Puffin/PuffinMetadata.md | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/docs/en/interfaces/formats/Puffin/Puffin.md b/docs/en/interfaces/formats/Puffin/Puffin.md index 8532fcff0080..7f9bf4b78ad9 100644 --- a/docs/en/interfaces/formats/Puffin/Puffin.md +++ b/docs/en/interfaces/formats/Puffin/Puffin.md @@ -1,6 +1,10 @@ --- description: 'Documentation for the Puffin format' +input_format: true +output_format: false keywords: ['Puffin'] +sidebar_label: 'Puffin' +sidebar_position: 1 slug: /interfaces/formats/Puffin title: 'Puffin' doc_type: 'reference' diff --git a/docs/en/interfaces/formats/Puffin/PuffinMetadata.md b/docs/en/interfaces/formats/Puffin/PuffinMetadata.md index dd716aeda62a..240d217869b6 100644 --- a/docs/en/interfaces/formats/Puffin/PuffinMetadata.md +++ b/docs/en/interfaces/formats/Puffin/PuffinMetadata.md @@ -1,6 +1,10 @@ --- description: 'Documentation for the PuffinMetadata format' +input_format: true +output_format: false keywords: ['PuffinMetadata'] +sidebar_label: 'PuffinMetadata' +sidebar_position: 2 slug: /interfaces/formats/PuffinMetadata title: 'PuffinMetadata' doc_type: 'reference' From 830790235d069c80a8ae75d747e73833973f829a Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 28 Jul 2026 10:08:28 +0200 Subject: [PATCH 4/5] Guard IcebergDeletionVector.cpp with USE_AVRO. Without the guard, Fast test builds that disable Avro fail to compile because IcebergPathFromMetadata is only visible when Avro is enabled. Co-authored-by: Cursor --- .../DataLakes/Iceberg/IcebergDeletionVector.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.cpp index 100e399a682e..4497a0aed812 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.cpp @@ -1,3 +1,7 @@ +#include "config.h" + +#if USE_AVRO + #include #include @@ -6,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -159,3 +164,5 @@ DataLakeObjectMetadata::ExcludedRowsPtr loadDeletionVector( } } + +#endif From 4b30a8f3e85e7a25c723ba1fbd58b912bce5fff8 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 28 Jul 2026 11:22:31 +0200 Subject: [PATCH 5/5] Fix settings history --- src/Core/SettingsChangesHistory.cpp | 69 +---------------------------- 1 file changed, 2 insertions(+), 67 deletions(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 70ad87748008..f2819a95bf77 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -39,73 +39,8 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() /// controls new feature and it's 'true' by default, use 'false' as previous_value). /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) /// Note: please check if the key already exists to prevent duplicate entries. - addSettingsChanges(settings_changes_history, "26.8", - { - {"unique_key_probe_implementation", "auto", "auto", "New setting: selects the UNIQUE KEY probe implementation (currently only the simple baseline exists)"}, - {"allow_lossy_numeric_supertype", false, false, "New setting that lets if/multiIf/coalesce/ifNull/array/map resolve all-numeric branches with no lossless common type (e.g. Decimal + Float64) to a numeric supertype (Float64, with possible precision loss), so the result can be aggregated. Independent of use_variant_as_common_type: with it off such branches previously raised NO_COMMON_TYPE, with it on they became a Variant; either way they now resolve to Float64."}, - {"analyzer_compatibility_apply_final_to_all_joined_tables", false, false, "New setting on master (default false = the fixed behavior). The behavior flip itself is recorded under 26.6, and the introduction for backports to older release branches (with default true) under 26.4."}, - {"join_runtime_filter_min_probe_rows", 0, 1000, "New setting to control minimum probe side size for installing JOIN runtime filters. It wasn't limited before, so previous value is 0 meaning always install."}, - {"query_plan_short_circuit_constant_false_join", false, true, "New setting to short-circuit a JOIN with a constant-false ON condition so the non-contributing side is not read. previous_value=false so `compatibility` with versions before 26.8 restores the pre-existing behavior (no short-circuit)."}, - }); - addSettingsChanges(settings_changes_history, "26.7", - { - {"analyzer_compatibility_allow_non_aggregate_in_having", false, false, "New compatibility setting. When enabled, the analyzer mimics the legacy `HAVING`-to-`WHERE` rewrite for non-aggregate AND-conjuncts instead of raising `NOT_AN_AGGREGATE`."}, - {"dictionary_lazy_load", "auto", "auto", "New setting overriding the server setting `dictionaries_lazy_load` for an individual dictionary."}, - {"discard_query_data", false, false, "New setting to skip sending query result rows to the client over the native TCP protocol."}, - {"optimize_trivial_count_with_sparsity_filter", false, false, "New (experimental) setting to serve `SELECT count() FROM t WHERE ` from per-column `num_defaults` / `num_rows` recorded in `serialization.json` when `` partitions rows into defaults vs non-defaults."}, - {"merge_tree_generic_exclusion_search_max_steps", 0, 0, "New setting to limit the number of steps of the generic exclusion search over the primary key index."}, - {"use_streaming_marks_compression", false, false, "New setting to compress marks into in-memory representation one block at a time (streaming) instead of materializing the full plain marks array, reducing peak memory during marks loading for compact parts with many substreams."}, - {"s3_validate_etag_on_read", false, true, "New setting to detect concurrent in-place overwrites of S3/GCS objects during a read by validating the GET response ETag against the listed one. previous_value=false so `compatibility` with versions before 26.7 restores the pre-existing behavior (no validation)."}, - {"ai_function_text_default_credentials", "", "", "New setting"}, - {"ai_function_embedding_default_credentials", "", "", "New setting"}, - {"dead_blobs_to_delay_insert", 0, 0, "New setting to override the `MergeTree` setting with the same name per query."}, - {"dead_blobs_to_throw_insert", 0, 0, "New setting to override the `MergeTree` setting with the same name per query."}, - {"input_format_csv_missing_nullable_as_empty_string", false, false, "New setting to read a missing value of `Nullable(String)` from CSV as an empty string instead of NULL."}, - {"use_legacy_to_time", true, false, "Use the new `toTime` function (converting values to the `Time` data type) by default instead of the legacy `toTime` (which is still available as `toTimeWithFixedDate`)."}, - {"reserve_memory", 0, 0, "New setting to reserve memory for specific workload before starting a query."}, - {"parallel_replicas_plan_based", false, false, "New setting"}, - {"use_paimon_metadata_files_cache", false, false, "New setting to enable in-memory caching of parsed Paimon metadata files (manifest lists and manifests). For persistent Paimon table engines it must be enabled before metadata initialization; table functions evaluate it per query. Avoids repeated downloads and deserialization of metadata files from object storage on subsequent queries."}, - {"optimize_or_like_chain", false, true, "Enable by default: optimize OR chains of LIKE/ILIKE/match into multiSearchAny (pure-substring patterns) or multiMatchAny (other patterns, when Hyperscan/Vectorscan is permitted); when neither fast path applies the original OR chain is kept unchanged."}, - {"optimize_or_like_chain_min_patterns", 0, 10, "New setting controlling the minimum number of non-pure-substring LIKE/ILIKE/match branches (sharing the same LHS expression) required for optimize_or_like_chain to rewrite a chain into multiMatchAny. Shorter chains are kept as-is because the multiMatchAny (Hyperscan) rewrite only becomes faster than short-circuit OR evaluation from about nine branches."}, - {"optimize_or_like_chain_min_substrings", 0, 4, "New setting controlling the minimum number of pure-substring (%needle%) LIKE/ILIKE branches (sharing the same LHS expression) required for optimize_or_like_chain to rewrite a chain into multiSearchAny."}, - {"input_format_arrow_use_native_reader", false, true, "New setting to use the native ClickHouse reader for the Arrow and ArrowStream formats instead of the Apache Arrow library."}, - {"input_format_orc_use_fast_decoder", true, true, "Obsolete setting, the native ClickHouse ORC decoder is now always used (the Apache Arrow-based ORC reader has been removed)."}, - {"output_format_arrow_use_native_writer", false, true, "New setting to use the native ClickHouse writer for the Arrow and ArrowStream formats instead of the Apache Arrow library."}, - {"allow_minmax_index_for_json", true, false, "Forbid creating minmax skip index on JSON columns by default because the index serialization cannot handle heterogeneous Field values"}, - {"s3_allow_server_credentials_in_user_queries", true, false, "New setting to block S3 access from user SQL from resolving the server's own ambient credentials (environment/IMDS/IRSA/instance-profile/AWS-config-file/role_arn-STS/GCP-OAuth-metadata). The previous behavior (allowed) is restored with compatibility settings."}, - {"query_plan_merge_expression_into_join", false, true, "New setting. Allow to merge Expression step into JOIN step during join reordering optimization."}, - {"skip_unavailable_shards_mode", "unavailable_or_table_missing", "unavailable_or_table_missing", "New setting to control which exceptions from a remote shard are ignored when `skip_unavailable_shards` is enabled. The default matches the historical behavior: a shard whose table is missing is treated as unavailable."}, - {"use_text_index_tokens_cache", false, true, "Enabled the text index tokens cache globally."}, - {"use_text_index_header_cache", false, true, "Enabled the text index header cache globally."}, - {"optimize_aggregation_in_order_limit", false, true, "New setting to push the `LIMIT` into aggregation-in-order for early termination when the `ORDER BY` is a prefix of the `GROUP BY` sort description."}, - {"explain_query_plan_default", "legacy", "pretty", "From 26.7, `EXPLAIN PLAN` defaults to `actions=1, compact=1, pretty=1`. Set this to `legacy` to restore the pre-26.7 output."}, - {"format_geojson_validate_geometry", true, true, "New setting that controls whether the GeoJSON format enforces RFC 7946 geometry validity (minimum points per line and ring, ring closure, non-empty multi-geometries) when reading and writing"}, - {"use_partition_minmax_for_primary_key_pruning", false, true, "New setting to use the part's partition minmax to prune more granules during primary key analysis for `MergeTree` tables, when a primary key column is also an input column of the partition key."}, - {"allow_delta_lake_writes", false, false, "Added an alias for setting `allow_experimental_delta_lake_writes`, which was moved to Beta."}, - {"allow_experimental_delta_lake_writes", false, false, "Delta Lake writes were moved to Beta."}, - {"optimize_redundant_comparisons", false, true, "New setting to detect conflicting and redundant comparison conditions on the same expression within AND chains."}, - {"mysql_datatypes_support_level", "decimal,datetime64,date2Date32", "decimal,datetime64,date2Date32,geometry", "Map MySQL's concrete spatial types (LINESTRING, POLYGON, MULTILINESTRING, MULTIPOLYGON, MULTIPOINT) and the generic GEOMETRY type to the corresponding ClickHouse geometric types by default. The generic GEOMETRY column maps to the umbrella Geometry type; reading a value whose subtype has no ClickHouse counterpart (GEOMETRYCOLLECTION) throws at read time."}, - {"snappy_mode", "basic", "basic", "New setting to control the wire format used for snappy compression in generic file/URL I/O. The default `basic` preserves backward-compatible Hadoop snappy block format reads; HTTP `Content-Encoding: snappy` always uses the framing format independently of this setting."}, - {"compile_regular_expressions", false, true, "New setting to enable JIT compilation of simple regular expressions in functions like `match` and `extract`."}, - {"min_count_to_compile_regular_expression", 3, 3, "New setting controlling how many times a regular expression must be used before it is JIT-compiled."}, - {"allow_aggregate_partitions_independently", false, true, "Enable independent per-partition aggregation by default when the partition key suits the GROUP BY key. The existing runtime heuristics in `ReadFromMergeTree::requestOutputEachPartitionThroughSeparatePortForAggregation` already skip the optimization when the partition layout is unfavorable (too few partitions, too many partitions, or significantly skewed partition sizes), so enabling the setting is safe in the cases where it would otherwise be a no-op."}, - {"text_index_lazy_intersection_density_threshold", 0.2, 0.2, "Renamed from `text_index_density_threshold` (kept as an alias); selects the posting list intersection algorithm in lazy posting list apply mode."}, - {"allow_experimental_text_index_lazy_apply", false, true, "Lazy posting list apply mode for the text index is no longer experimental; the setting is now obsolete and has no effect (lazy mode is selected via `text_index_posting_list_apply_mode = 'lazy'`)."}, - {"allow_experimental_url_wildcard_from_index_pages", false, false, "New setting to enable expanding wildcards in the `url` table function by listing HTTP index pages."}, - {"url_wildcard_max_directories_to_read", 100000, 100000, "New setting to limit the number of directories read when expanding wildcards in the `url` table function."}, - {"allow_experimental_eval_table_function", false, false, "New setting to enable the experimental table function `eval`."}, - {"output_format_csv_header_serialize_tuple_into_separate_columns", false, true, "New setting. When output_format_csv_serialize_tuple_into_separate_columns is enabled, the CSVWithNames/CSVWithNamesAndTypes header now flattens Tuple columns into their leaf fields so the header width matches the data. Set to false to restore the previous single-name header."}, - {"enable_join_runtime_filters_index_analysis", false, false, "New setting to enable join filtering using dynamic index analysis"}, - {"vector_search_use_quantized_codes", false, false, "New setting to opt into the two-stage approximate vector-search optimization over a Quantize(...) column codec; queries stay exact by default."}, - {"reader_executor_use_long_connections", false, false, "New experimental ReaderExecutor setting (off by default): reuse a held source connection across sequential windows."}, - {"reader_executor_min_bytes_for_seek", 2097152, 2097152, "New experimental ReaderExecutor setting: forward-gap bound for bridging on a held source connection."}, - {"reader_executor_max_tail_for_drain", 1048576, 1048576, "New experimental ReaderExecutor setting: drain bound for completing a dropped long connection."}, - {"precise_float_parsing", false, true, "Use the precise (closest-representable) float parsing algorithm by default, now that it is faster than the previous fast algorithm. Set to false to restore the pre-26.7 fast-but-less-accurate parsing in conversion functions."}, - {"optimize_and_compare_chain_max_hash_work", 0, 5'000'000, "New setting that bounds the work of the `optimize_and_compare_chain` optimization (measured in query-tree nodes hashed) so it cannot dominate analysis of queries with very many or very large `AND`-chains of comparisons. The previous value `0` (unlimited) reproduces the pre-26.7 behavior where the optimization was uncapped, so `compatibility` set to an earlier version keeps deriving transitive predicates without a budget. Set to `0` to disable the budget."}, - {"iceberg_manifest_min_count_to_compact", 30, 30, "New setting to control manifest compaction for Iceberg tables."}, - {"show_remote_databases_in_system_tables", true, true, "New setting to control whether `MySQL` and `PostgreSQL` databases are shown in `system.tables`, `system.columns` and `system.completions`."}, - {"use_constant_folding_in_index_analysis", false, false, "New setting to fold partition-level constants into the filter predicate per part during MergeTree index analysis, improving pruning for filters whose branches depend on partition values."}, - {"join_runtime_filter_size_from_hash_table_stats", false, true, "Use hash table size statistics collected from previous executions to size the JOIN runtime filter. When disabled, fall back to the fixed `join_runtime_bloom_filter_bytes`."}, + addSettingsChanges(settings_changes_history, "26.6.1.20001.altinityantalya", + { {"use_puffin_files_cache", false, true, "Enables cache of parsed Puffin file content such as deletion vectors."}, });