From 5331fef650533a710700448e667020deb3bac73b Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Mon, 10 Aug 2026 17:26:46 +0200 Subject: [PATCH 01/13] add tests, update doc, core logic Signed-off-by: Konstantin Morozov --- docs/en/antalya/part_export.md | 15 +- docs/en/antalya/partition_export.md | 15 +- src/Core/Settings.cpp | 5 +- src/Core/SettingsEnums.h | 1 + src/Storages/MergeTree/ExportPartTask.cpp | 20 +-- .../MergeTree/ExportPartitionUtils.cpp | 123 ++++++++++++--- src/Storages/MergeTree/ExportPartitionUtils.h | 14 +- .../tests/gtest_export_partition_ordering.cpp | 110 +++++++++++++ .../test.py | 144 +++++++++++++++--- .../test.py | 55 ++++++- .../test.py | 61 +++++--- .../test.py | 71 ++++++++- 12 files changed, 542 insertions(+), 92 deletions(-) diff --git a/docs/en/antalya/part_export.md b/docs/en/antalya/part_export.md index 5cf8e2aa8b0e..bb8b7645c0a7 100644 --- a/docs/en/antalya/part_export.md +++ b/docs/en/antalya/part_export.md @@ -140,11 +140,20 @@ In case a table function is used as the destination, the schema can be omitted a - **Type**: `MergeTreePartExportSchemaMismatchMode` - **Default**: `strict` -- **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` allows a column-count mismatch between the source `MergeTree` table and the destination table. Columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Possible values: - - `strict` - the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. +- **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` allows the source `MergeTree` table to have more columns than the destination table and how destination columns are matched in that case. Possible values: + - `strict` - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`, and the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - `ignore_extra_source_columns_by_position` - the source may have more columns than the destination. The extra trailing source columns (by position) are dropped and not exported. The destination having more columns than the source is still rejected in this mode. + - `ignore_extra_source_columns_by_name` - the source may have more columns than the destination. Every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be reordered and additional source columns may occur in any position. Unmatched source columns are not exported. If a destination column is absent from the source, including when it was renamed, the export throws `THERE_IS_NO_COLUMN`; there is no positional fallback. Use `ignore_extra_source_columns_by_position` when renamed columns should be matched by position. - The extra trailing source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. + Extra source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. Type conversion and `export_merge_tree_part_allow_lossy_cast` are applied after columns are matched. + + Error behavior: + + - In `strict` mode, a different number of source and destination columns throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. + - In `ignore_extra_source_columns_by_position` mode, the destination having more columns than the source throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. + - In `ignore_extra_source_columns_by_name` mode, a destination column which is absent from the source throws `THERE_IS_NO_COLUMN`. + - Renaming a destination column in `ignore_extra_source_columns_by_name` mode throws `THERE_IS_NO_COLUMN`; the mode does not fall back to positional matching. + - After columns have been matched successfully, a cast rejected by the export type-safety check throws `INCOMPATIBLE_COLUMNS`. ## Examples diff --git a/docs/en/antalya/partition_export.md b/docs/en/antalya/partition_export.md index 316f7591b3a8..486ee7551b35 100644 --- a/docs/en/antalya/partition_export.md +++ b/docs/en/antalya/partition_export.md @@ -139,11 +139,20 @@ Notes: - **Type**: `MergeTreePartExportSchemaMismatchMode` - **Default**: `strict` -- **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` allows a column-count mismatch between the source `MergeTree` table and the destination table. Columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Possible values: - - `strict` - the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. +- **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` allows the source `MergeTree` table to have more columns than the destination table and how destination columns are matched in that case. Possible values: + - `strict` - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`, and the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - `ignore_extra_source_columns_by_position` - the source may have more columns than the destination. The extra trailing source columns (by position) are dropped and not exported. The destination having more columns than the source is still rejected in this mode. + - `ignore_extra_source_columns_by_name` - the source may have more columns than the destination. Every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be reordered and additional source columns may occur in any position. Unmatched source columns are not exported. If a destination column is absent from the source, including when it was renamed, the export throws `THERE_IS_NO_COLUMN`; there is no positional fallback. Use `ignore_extra_source_columns_by_position` when renamed columns should be matched by position. - The extra trailing source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. + Extra source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. Type conversion and `export_merge_tree_part_allow_lossy_cast` are applied after columns are matched. + + Error behavior: + + - In `strict` mode, a different number of source and destination columns throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. + - In `ignore_extra_source_columns_by_position` mode, the destination having more columns than the source throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. + - In `ignore_extra_source_columns_by_name` mode, a destination column which is absent from the source throws `THERE_IS_NO_COLUMN`. + - Renaming a destination column in `ignore_extra_source_columns_by_name` mode throws `THERE_IS_NO_COLUMN`; the mode does not fall back to positional matching. + - After columns have been matched successfully, a cast rejected by the export type-safety check throws `INCOMPATIBLE_COLUMNS`. ## Examples diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 4109ce49ac12..52ce73c6bf24 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -7628,10 +7628,11 @@ Allow `EXPORT PART`/`EXPORT PARTITION` to apply lossy (non-value-preserving) cas When exporting to Apache Iceberg, the partition value written to the metadata is derived from the source partition columns by casting them to the destination partition-field types and applying the destination partition transform — the same computation the exported data files use, so the metadata stays consistent with the data. A lossy cast on a partition column remains semantically truncating: both the data files and the metadata contain the truncated value, and such casts require this setting to be enabled. )", 0) \ DECLARE(MergeTreePartExportSchemaMismatchMode, export_merge_tree_part_schema_mismatch_mode, MergeTreePartExportSchemaMismatchMode::strict, R"( -Controls whether `EXPORT PART`/`EXPORT PARTITION` allows a column-count mismatch between the source `MergeTree` table and the destination table. Columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. +Controls whether `EXPORT PART`/`EXPORT PARTITION` allows the source `MergeTree` table to have more columns than the destination table and how destination columns are matched. Possible values: -- `strict` (default) - the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. +- `strict` (default) - columns are matched positionally and the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - `ignore_extra_source_columns_by_position` - the source may have more columns than the destination. The extra trailing source columns (by position) are dropped and not exported. The destination having more columns than the source is still rejected in this mode. +- `ignore_extra_source_columns_by_name` - every destination column is matched to a source column with the same exact, case-sensitive name. Destination columns may be reordered and additional source columns may occur in any position. A destination column absent from the source throws `THERE_IS_NO_COLUMN`; there is no positional fallback. )", 0) \ \ /* ####################################################### */ \ diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index 0c82d40345cd..f5c48509e0b0 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -578,6 +578,7 @@ enum class MergeTreePartExportSchemaMismatchMode : uint8_t { strict, ignore_extra_source_columns_by_position, + ignore_extra_source_columns_by_name, }; DECLARE_SETTING_ENUM(MergeTreePartExportSchemaMismatchMode) diff --git a/src/Storages/MergeTree/ExportPartTask.cpp b/src/Storages/MergeTree/ExportPartTask.cpp index 5ef537bbef1c..8aa3511afad4 100644 --- a/src/Storages/MergeTree/ExportPartTask.cpp +++ b/src/Storages/MergeTree/ExportPartTask.cpp @@ -113,15 +113,6 @@ namespace } } - /// Mirrors `InterpreterInsertQuery::addInsertToSelectPipeline`: positional match, - /// destination header = `getSampleBlockNonMaterialized()`, all type bridging is done - /// by the CAST inside `makeConvertingActions`. No pre-validation, no per-column - /// lossy/non-lossy classification — restrictions are exactly what INSERT SELECT enforces. - /// - /// Exception: when `export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_position'` - /// and the source has more columns than the destination, the extra trailing source - /// columns (by position) are dropped by a preliminary projection step before the - /// positional convert, so `makeConvertingActions` always sees equal-sized inputs. void addExportConvertingActions( QueryPlan & plan_for_part, const IStorage & destination_storage, @@ -131,9 +122,10 @@ namespace = destination_storage.getInMemoryMetadataPtr()->getSampleBlockNonMaterialized(); const auto & destination_columns = destination_header.getColumnsWithTypeAndName(); + const auto schema_mismatch_mode = + local_context->getSettingsRef()[Setting::export_merge_tree_part_schema_mismatch_mode].value; const bool ignore_extra_source_columns_by_position = - local_context->getSettingsRef()[Setting::export_merge_tree_part_schema_mismatch_mode] - == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position; + schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position; auto source_columns = plan_for_part.getCurrentHeader()->getColumnsWithTypeAndName(); @@ -169,7 +161,9 @@ namespace auto dag = ActionsDAG::makeConvertingActions( source_columns, destination_columns, - ActionsDAG::MatchColumnsMode::Position, + schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name + ? ActionsDAG::MatchColumnsMode::Name + : ActionsDAG::MatchColumnsMode::Position, local_context); auto expression_step = std::make_unique( @@ -353,8 +347,6 @@ bool ExportPartTask::executeStep() /// This is a hack that materializes the columns before the export so they can be exported to tables that have matching columns materializeSpecialColumns(plan_for_part.getCurrentHeader(), metadata_snapshot, local_context, plan_for_part); - /// Align the pipeline header with the destination's non-materialized sample block, - /// using the same `makeConvertingActions(Position)` call INSERT SELECT performs. addExportConvertingActions(plan_for_part, *destination_storage, local_context); QueryPlanOptimizationSettings optimization_settings(local_context); diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index d852a16a9bc3..27bf0be39e55 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -68,6 +68,7 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ILLEGAL_COLUMN; extern const int NUMBER_OF_COLUMNS_DOESNT_MATCH; + extern const int THERE_IS_NO_COLUMN; extern const int INCOMPATIBLE_COLUMNS; extern const int NO_SUCH_COLUMN_IN_TABLE; extern const int FILE_ALREADY_EXISTS; @@ -125,6 +126,7 @@ namespace ExportPartitionUtils ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, ErrorCodes::ILLEGAL_COLUMN, ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, + ErrorCodes::THERE_IS_NO_COLUMN, ErrorCodes::INCOMPATIBLE_COLUMNS, ErrorCodes::NO_SUCH_COLUMN_IN_TABLE, ErrorCodes::NOT_IMPLEMENTED, @@ -914,6 +916,24 @@ namespace return true; } + void verifyExportColumnCastIsSafe( + const ColumnWithTypeAndName & source_column, + const ColumnWithTypeAndName & destination_column, + const StorageID & destination_storage_id) + { + if (canBeSafelyCast(source_column.type, destination_column.type)) + return; + + throw Exception(ErrorCodes::INCOMPATIBLE_COLUMNS, + "Cannot export to {}: column '{}' requires a lossy cast from {} to {}, " + "which may change values. Set `export_merge_tree_part_allow_lossy_cast = 1` " + "to allow lossy casts during export.", + destination_storage_id.getFullTableName(), + destination_column.name, + source_column.type->getName(), + destination_column.type->getName()); + } + void verifyPartitionKeyColumn( const ColumnWithTypeAndName & source_column, const ColumnWithTypeAndName & destination_column, @@ -944,6 +964,46 @@ namespace destination_column.type->getName()); } + void verifyExportColumnCastsAreSafe( + const ColumnsWithTypeAndName & source_columns, + const ColumnsWithTypeAndName & destination_columns, + MergeTreePartExportSchemaMismatchMode schema_mismatch_mode, + const StorageID & destination_storage_id) + { + if (schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name) + { + std::unordered_map source_columns_by_name; + source_columns_by_name.reserve(source_columns.size()); + for (const auto & source_column : source_columns) + source_columns_by_name.emplace(source_column.name, &source_column); + + for (const auto & destination_column : destination_columns) + { + const auto source_it = source_columns_by_name.find(destination_column.name); + if (source_it == source_columns_by_name.end()) + throw Exception( + ErrorCodes::THERE_IS_NO_COLUMN, + "Cannot find column `{}` in source stream", + destination_column.name); + + verifyExportColumnCastIsSafe(*source_it->second, destination_column, destination_storage_id); + } + return; + } + + if (source_columns.size() < destination_columns.size() + || (schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::strict + && source_columns.size() != destination_columns.size())) + throw Exception( + ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, + "Number of columns doesn't match (source: {} and result: {})", + source_columns.size(), + destination_columns.size()); + + for (size_t i = 0; i < destination_columns.size(); ++i) + verifyExportColumnCastIsSafe(source_columns[i], destination_columns[i], destination_storage_id); + } + void verifyExportSchemaCastable( const StorageMetadataPtr & source_metadata, const StorageMetadataPtr & destination_metadata, @@ -966,9 +1026,10 @@ namespace /// the trimming `ExportPartTask::addExportConvertingActions` applies to the real data. /// The reverse (destination has more columns than source) is always rejected below by /// `makeConvertingActions`, in both modes. + const auto schema_mismatch_mode = + context->getSettingsRef()[Setting::export_merge_tree_part_schema_mismatch_mode].value; const bool ignore_extra_source_columns_by_position = - context->getSettingsRef()[Setting::export_merge_tree_part_schema_mismatch_mode] - == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position; + schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position; if (ignore_extra_source_columns_by_position && source_columns.size() > destination_columns.size()) { @@ -984,7 +1045,9 @@ namespace (void) ActionsDAG::makeConvertingActions( source_columns, destination_columns, - ActionsDAG::MatchColumnsMode::Position, + schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name + ? ActionsDAG::MatchColumnsMode::Name + : ActionsDAG::MatchColumnsMode::Position, context); const auto & source_columns_description = source_metadata->getColumns(); @@ -1001,29 +1064,47 @@ namespace const bool allow_lossy_cast = context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; - const size_t num_columns = std::min(source_columns.size(), destination_columns.size()); - for (size_t i = 0; i < num_columns; ++i) + /// Partition-key columns must keep a stable identity across the conversion, regardless of + /// `allow_lossy_cast`, so that `PARTITION BY` keeps selecting the same values after export. + /// `ignore_extra_source_columns_by_name` matches columns by name (see `makeConvertingActions` + /// above), so look the source column up by name instead of assuming the same position. + if (schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name) { - const auto & source_column = source_columns[i]; - const auto & destination_column = destination_columns[i]; + std::unordered_map source_positions_by_name; + source_positions_by_name.reserve(source_columns.size()); + for (size_t i = 0; i < source_columns.size(); ++i) + source_positions_by_name.emplace(source_columns[i].name, i); - if (partition_key_owner_columns.contains(source_column.name)) - verifyPartitionKeyColumn(source_column, destination_column, i, destination_storage_id); + for (const auto & destination_column : destination_columns) + { + if (!partition_key_owner_columns.contains(destination_column.name)) + continue; - /// Lossy casts may silently change values, so reject them unless the user opts in. - if (allow_lossy_cast) - continue; + const auto source_it = source_positions_by_name.find(destination_column.name); + if (source_it == source_positions_by_name.end()) + continue; /// Reported as THERE_IS_NO_COLUMN by verifyExportColumnCastsAreSafe below. - if (!canBeSafelyCast(source_column.type, destination_column.type)) - throw Exception(ErrorCodes::INCOMPATIBLE_COLUMNS, - "Cannot export to {}: column '{}' requires a lossy cast from {} to {}, " - "which may change values. Set `export_merge_tree_part_allow_lossy_cast = 1` " - "to allow lossy casts during export.", - destination_storage_id.getFullTableName(), - destination_column.name, - source_column.type->getName(), - destination_column.type->getName()); + verifyPartitionKeyColumn( + source_columns[source_it->second], destination_column, source_it->second, destination_storage_id); + } } + else + { + const size_t num_columns = std::min(source_columns.size(), destination_columns.size()); + for (size_t i = 0; i < num_columns; ++i) + if (partition_key_owner_columns.contains(source_columns[i].name)) + verifyPartitionKeyColumn(source_columns[i], destination_columns[i], i, destination_storage_id); + } + + /// Lossy casts may silently change values, so reject them unless the user opts in. + if (allow_lossy_cast) + return; + + verifyExportColumnCastsAreSafe( + source_columns, + destination_columns, + schema_mismatch_mode, + destination_storage_id); } } diff --git a/src/Storages/MergeTree/ExportPartitionUtils.h b/src/Storages/MergeTree/ExportPartitionUtils.h index 7605bd43ac4a..492bb178b3e2 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.h +++ b/src/Storages/MergeTree/ExportPartitionUtils.h @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include #include "Storages/IStorage.h" @@ -87,15 +89,23 @@ namespace ExportPartitionUtils const StorageMetadataPtr & source_metadata, const StorageMetadataPtr & destination_metadata); + void verifyExportColumnCastsAreSafe( + const ColumnsWithTypeAndName & source_columns, + const ColumnsWithTypeAndName & destination_columns, + MergeTreePartExportSchemaMismatchMode schema_mismatch_mode, + const StorageID & destination_storage_id); + /// Validates that source columns can be exported into the destination with the - /// same positional CAST matching as `INSERT INTO dest SELECT * FROM src`. Lossy - /// casts are rejected unless `export_merge_tree_part_allow_lossy_cast` is set. + /// configured positional or name-based CAST matching. Lossy casts are rejected + /// unless `export_merge_tree_part_allow_lossy_cast` is set. /// /// By default the source and destination must have the same number of columns. /// If `export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_position'`, a /// source with more columns than the destination is allowed: the extra trailing /// source columns (by position) are excluded from the comparison here, matching /// what `ExportPartTask::addExportConvertingActions` drops from the actual data. + /// If the mode is `ignore_extra_source_columns_by_name`, destination columns are + /// matched to source columns by their exact names and may appear in a different order. /// /// Throws BAD_ARGUMENTS on any violation. void verifyExportSchemaCastable( diff --git a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp index df20755ba590..8186db87d944 100644 --- a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp +++ b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp @@ -3,7 +3,10 @@ #include #include #include +#include #include +#include +#include #include #include #include @@ -11,6 +14,12 @@ namespace DB { +namespace ErrorCodes +{ + extern const int INCOMPATIBLE_COLUMNS; + extern const int THERE_IS_NO_COLUMN; +} + namespace Setting { extern const SettingsMergeTreePartExportSchemaMismatchMode export_merge_tree_part_schema_mismatch_mode; @@ -18,6 +27,27 @@ namespace Setting namespace { + template + ColumnWithTypeAndName makeColumn(const String & name) + { + auto type = std::make_shared(); + return {type->createColumn(), type, name}; + } + + template + void expectExceptionCode(Function && function, int expected_code) + { + try + { + function(); + FAIL() << "Expected exception code " << expected_code; + } + catch (const Exception & exception) + { + EXPECT_EQ(exception.code(), expected_code) << exception.message(); + } + } + ExportReplicatedMergeTreePartitionManifest makeValidManifest() { ExportReplicatedMergeTreePartitionManifest manifest; @@ -171,4 +201,84 @@ TEST_F(ExportPartitionManifestBackCompatTest, SchemaMismatchModeAppliedToWorkerC } } +TEST(ExportColumnCastsTest, UsesSelectedMatchingMode) +{ + const ColumnsWithTypeAndName source_columns = { + makeColumn("id"), + makeColumn("year"), + makeColumn("payload"), + }; + const ColumnsWithTypeAndName destination_columns = { + makeColumn("payload"), + makeColumn("year"), + makeColumn("id"), + }; + const StorageID destination_storage_id{"test", "destination"}; + + expectExceptionCode( + [&] + { + ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, + destination_columns, + MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position, + destination_storage_id); + }, + ErrorCodes::INCOMPATIBLE_COLUMNS); + + EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, + destination_columns, + MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name, + destination_storage_id)); +} + +TEST(ExportColumnCastsTest, RejectsLossyCastAfterMatchingByName) +{ + const ColumnsWithTypeAndName source_columns = { + makeColumn("id"), + makeColumn("year"), + makeColumn("extra"), + }; + const ColumnsWithTypeAndName destination_columns = { + makeColumn("id"), + makeColumn("year"), + }; + + expectExceptionCode( + [&] + { + ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, + destination_columns, + MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name, + StorageID{"test", "destination"}); + }, + ErrorCodes::INCOMPATIBLE_COLUMNS); +} + +TEST(ExportColumnCastsTest, RejectsMissingDestinationColumnAfterMatchingByName) +{ + const ColumnsWithTypeAndName source_columns = { + makeColumn("id"), + makeColumn("year"), + makeColumn("extra"), + }; + const ColumnsWithTypeAndName destination_columns = { + makeColumn("renamed_id"), + makeColumn("year"), + }; + + expectExceptionCode( + [&] + { + ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, + destination_columns, + MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name, + StorageID{"test", "destination"}); + }, + ErrorCodes::THERE_IS_NO_COLUMN); +} + } diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index 6cc022eb593f..da27618d142b 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -37,6 +37,12 @@ ) +EXTRA_SOURCE_COLUMN_MODES = [ + pytest.param("ignore_extra_source_columns_by_position", id="by-position"), + pytest.param("ignore_extra_source_columns_by_name", id="by-name"), +] + + # --------------------------------------------------------------------------- # Cluster fixture # --------------------------------------------------------------------------- @@ -738,13 +744,8 @@ def test_export_part_column_count_mismatch_source_fewer_is_rejected(cluster): node.query(f"DROP TABLE IF EXISTS {iceberg}") -def test_export_part_source_more_columns_allowed_with_ignore_extra_setting(cluster): - """ - Source has 3 columns (id, year, extra), destination has 2 (id, year). - With `export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_position'`, - the export must succeed: the trailing `extra` source column is dropped - (matched positionally) and only `id`/`year` land in the destination. - """ +@pytest.mark.parametrize("mismatch_mode", EXTRA_SOURCE_COLUMN_MODES) +def test_export_part_source_more_columns_allowed_with_ignore_extra_setting(cluster, mismatch_mode): node = cluster.instances["node1"] sfx = unique_suffix() mt = f"mt_ignore_extra_{sfx}" @@ -758,7 +759,7 @@ def test_export_part_source_more_columns_allowed_with_ignore_extra_setting(clust export_part( node=node, table=mt, part=part_2020, dest=iceberg, - extra_settings="export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_position'", + extra_settings=f"export_merge_tree_part_schema_mismatch_mode = '{mismatch_mode}'", ) wait_for_export_part(node=node, table=mt, part=part_2020) @@ -774,13 +775,16 @@ def test_export_part_source_more_columns_allowed_with_ignore_extra_setting(clust node.query(f"DROP TABLE IF EXISTS {iceberg}") -def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_ignore_extra_setting(cluster): - """ - `ignore_extra_source_columns_by_position` only relaxes the source-has-more-columns - direction. Source has 2 columns (id, year), destination has 3 (id, year, extra): - the destination cannot be filled from the source, so this must still be - rejected synchronously even with the relaxed setting. - """ +@pytest.mark.parametrize( + "mismatch_mode,expected_error", + [ + pytest.param("ignore_extra_source_columns_by_position", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), + pytest.param("ignore_extra_source_columns_by_name", "THERE_IS_NO_COLUMN", id="by-name"), + ], +) +def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_ignore_extra_setting( + cluster, mismatch_mode, expected_error +): node = cluster.instances["node1"] sfx = unique_suffix() mt = f"mt_ignore_extra_fewer_{sfx}" @@ -796,12 +800,9 @@ def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_igno f"ALTER TABLE {mt} EXPORT PART '{part_2020}' TO TABLE {iceberg} " f"SETTINGS allow_experimental_export_merge_tree_part = 1, " f"allow_experimental_insert_into_iceberg = 1, " - f"export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_position'" - ) - assert "NUMBER_OF_COLUMNS_DOESNT_MATCH" in error, ( - f"Expected NUMBER_OF_COLUMNS_DOESNT_MATCH for source Date: Mon, 17 Aug 2026 14:01:21 +0200 Subject: [PATCH 02/13] remove comments --- src/Storages/MergeTree/ExportPartitionUtils.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 27bf0be39e55..8dad6c706e08 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -1064,10 +1064,6 @@ namespace const bool allow_lossy_cast = context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; - /// Partition-key columns must keep a stable identity across the conversion, regardless of - /// `allow_lossy_cast`, so that `PARTITION BY` keeps selecting the same values after export. - /// `ignore_extra_source_columns_by_name` matches columns by name (see `makeConvertingActions` - /// above), so look the source column up by name instead of assuming the same position. if (schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name) { std::unordered_map source_positions_by_name; @@ -1082,7 +1078,7 @@ namespace const auto source_it = source_positions_by_name.find(destination_column.name); if (source_it == source_positions_by_name.end()) - continue; /// Reported as THERE_IS_NO_COLUMN by verifyExportColumnCastsAreSafe below. + continue; verifyPartitionKeyColumn( source_columns[source_it->second], destination_column, source_it->second, destination_storage_id); From 28f263e183c23e3a8a00628cf54ee2d206231980 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Tue, 18 Aug 2026 10:59:46 +0200 Subject: [PATCH 03/13] update text Signed-off-by: Konstantin Morozov --- src/Storages/MergeTree/ExportPartitionUtils.cpp | 16 ++++++++++------ src/Storages/StorageReplicatedMergeTree.cpp | 1 - 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 8dad6c706e08..36451885ff7e 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -938,19 +938,21 @@ namespace const ColumnWithTypeAndName & source_column, const ColumnWithTypeAndName & destination_column, size_t position, - const StorageID & destination_storage_id) + const StorageID & destination_storage_id, + bool match_by_name) { if (source_column.name != destination_column.name) throw Exception( ErrorCodes::BAD_ARGUMENTS, "Cannot export to {}: partition key column '{}' is at position {} in the source " "table, but the destination's column at that position is named '{}'. EXPORT " - "PART/PARTITION matches columns by position, so partition key columns must be " - "declared at the same position in both tables.", + "PART/PARTITION {} so partition key columns must be declared {} in both tables.", destination_storage_id.getFullTableName(), source_column.name, position, - destination_column.name); + destination_column.name, + match_by_name ? "matches columns by name" : "matches columns by position", + match_by_name ? "with the same name" : "at the same position"); if (!haveSameTupleElementLayout(source_column.type, destination_column.type)) throw Exception( @@ -1081,7 +1083,8 @@ namespace continue; verifyPartitionKeyColumn( - source_columns[source_it->second], destination_column, source_it->second, destination_storage_id); + source_columns[source_it->second], destination_column, source_it->second, destination_storage_id, + /*match_by_name=*/ true); } } else @@ -1089,7 +1092,8 @@ namespace const size_t num_columns = std::min(source_columns.size(), destination_columns.size()); for (size_t i = 0; i < num_columns; ++i) if (partition_key_owner_columns.contains(source_columns[i].name)) - verifyPartitionKeyColumn(source_columns[i], destination_columns[i], i, destination_storage_id); + verifyPartitionKeyColumn(source_columns[i], destination_columns[i], i, destination_storage_id, + /*match_by_name=*/ false); } /// Lossy casts may silently change values, so reject them unless the user opts in. diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index af6259574d4a..a1257264ef22 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -8413,7 +8413,6 @@ void StorageReplicatedMergeTree::exportPartitionToTable(const PartitionCommand & auto src_snapshot = getInMemoryMetadataPtr(); auto destination_snapshot = dest_storage->getInMemoryMetadataPtr(); - /// Positional CAST matching, like `INSERT INTO dest SELECT * FROM src`. ExportPartitionUtils::verifyExportSchemaCastable( src_snapshot, destination_snapshot, dest_storage->getStorageID(), query_context); From e9636335988fee24d6ffde453b6c53f91de9c2ac Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Tue, 18 Aug 2026 13:15:12 +0200 Subject: [PATCH 04/13] by_name only for extra columns Signed-off-by: Konstantin Morozov --- docs/en/antalya/part_export.md | 6 +-- docs/en/antalya/partition_export.md | 6 +-- src/Core/Settings.cpp | 2 +- src/Storages/MergeTree/ExportPartTask.cpp | 9 +++- .../MergeTree/ExportPartitionUtils.cpp | 16 +++++-- src/Storages/MergeTree/ExportPartitionUtils.h | 6 ++- .../tests/gtest_export_partition_ordering.cpp | 45 +++++++++++++++++++ .../test.py | 2 +- .../test.py | 2 +- .../test.py | 2 +- 10 files changed, 78 insertions(+), 18 deletions(-) diff --git a/docs/en/antalya/part_export.md b/docs/en/antalya/part_export.md index bb8b7645c0a7..59f93cd1e0b8 100644 --- a/docs/en/antalya/part_export.md +++ b/docs/en/antalya/part_export.md @@ -143,7 +143,7 @@ In case a table function is used as the destination, the schema can be omitted a - **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` allows the source `MergeTree` table to have more columns than the destination table and how destination columns are matched in that case. Possible values: - `strict` - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`, and the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - `ignore_extra_source_columns_by_position` - the source may have more columns than the destination. The extra trailing source columns (by position) are dropped and not exported. The destination having more columns than the source is still rejected in this mode. - - `ignore_extra_source_columns_by_name` - the source may have more columns than the destination. Every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be reordered and additional source columns may occur in any position. Unmatched source columns are not exported. If a destination column is absent from the source, including when it was renamed, the export throws `THERE_IS_NO_COLUMN`; there is no positional fallback. Use `ignore_extra_source_columns_by_position` when renamed columns should be matched by position. + - `ignore_extra_source_columns_by_name` - only takes effect when the source has more columns than the destination. In that case, every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be reordered and the extra source columns may occur in any position. Unmatched source columns are not exported. If a destination column is absent from the source, including when it was renamed, the export throws `THERE_IS_NO_COLUMN`; there is no positional fallback. When the source does **not** have more columns than the destination, this mode behaves exactly like `strict` - columns are matched positionally, and a column-count mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. Use `ignore_extra_source_columns_by_position` when renamed columns should be matched by position. Extra source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. Type conversion and `export_merge_tree_part_allow_lossy_cast` are applied after columns are matched. @@ -151,8 +151,8 @@ In case a table function is used as the destination, the schema can be omitted a - In `strict` mode, a different number of source and destination columns throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - In `ignore_extra_source_columns_by_position` mode, the destination having more columns than the source throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - - In `ignore_extra_source_columns_by_name` mode, a destination column which is absent from the source throws `THERE_IS_NO_COLUMN`. - - Renaming a destination column in `ignore_extra_source_columns_by_name` mode throws `THERE_IS_NO_COLUMN`; the mode does not fall back to positional matching. + - In `ignore_extra_source_columns_by_name` mode, when the source has more columns than the destination, a destination column absent from the source (including a renamed one) throws `THERE_IS_NO_COLUMN`; the mode does not fall back to positional matching in that case. + - In `ignore_extra_source_columns_by_name` mode, when the source does not have more columns than the destination, a column-count mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`, same as `strict`. - After columns have been matched successfully, a cast rejected by the export type-safety check throws `INCOMPATIBLE_COLUMNS`. diff --git a/docs/en/antalya/partition_export.md b/docs/en/antalya/partition_export.md index 486ee7551b35..4548a32117c7 100644 --- a/docs/en/antalya/partition_export.md +++ b/docs/en/antalya/partition_export.md @@ -142,7 +142,7 @@ Notes: - **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` allows the source `MergeTree` table to have more columns than the destination table and how destination columns are matched in that case. Possible values: - `strict` - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`, and the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - `ignore_extra_source_columns_by_position` - the source may have more columns than the destination. The extra trailing source columns (by position) are dropped and not exported. The destination having more columns than the source is still rejected in this mode. - - `ignore_extra_source_columns_by_name` - the source may have more columns than the destination. Every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be reordered and additional source columns may occur in any position. Unmatched source columns are not exported. If a destination column is absent from the source, including when it was renamed, the export throws `THERE_IS_NO_COLUMN`; there is no positional fallback. Use `ignore_extra_source_columns_by_position` when renamed columns should be matched by position. + - `ignore_extra_source_columns_by_name` - only takes effect when the source has more columns than the destination. In that case, every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be reordered and the extra source columns may occur in any position. Unmatched source columns are not exported. If a destination column is absent from the source, including when it was renamed, the export throws `THERE_IS_NO_COLUMN`; there is no positional fallback. When the source does **not** have more columns than the destination, this mode behaves exactly like `strict` - columns are matched positionally, and a column-count mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. Use `ignore_extra_source_columns_by_position` when renamed columns should be matched by position. Extra source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. Type conversion and `export_merge_tree_part_allow_lossy_cast` are applied after columns are matched. @@ -150,8 +150,8 @@ Notes: - In `strict` mode, a different number of source and destination columns throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - In `ignore_extra_source_columns_by_position` mode, the destination having more columns than the source throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - - In `ignore_extra_source_columns_by_name` mode, a destination column which is absent from the source throws `THERE_IS_NO_COLUMN`. - - Renaming a destination column in `ignore_extra_source_columns_by_name` mode throws `THERE_IS_NO_COLUMN`; the mode does not fall back to positional matching. + - In `ignore_extra_source_columns_by_name` mode, when the source has more columns than the destination, a destination column absent from the source (including a renamed one) throws `THERE_IS_NO_COLUMN`; the mode does not fall back to positional matching in that case. + - In `ignore_extra_source_columns_by_name` mode, when the source does not have more columns than the destination, a column-count mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`, same as `strict`. - After columns have been matched successfully, a cast rejected by the export type-safety check throws `INCOMPATIBLE_COLUMNS`. ## Examples diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 52ce73c6bf24..c9f364eeaf51 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -7632,7 +7632,7 @@ Controls whether `EXPORT PART`/`EXPORT PARTITION` allows the source `MergeTree` Possible values: - `strict` (default) - columns are matched positionally and the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - `ignore_extra_source_columns_by_position` - the source may have more columns than the destination. The extra trailing source columns (by position) are dropped and not exported. The destination having more columns than the source is still rejected in this mode. -- `ignore_extra_source_columns_by_name` - every destination column is matched to a source column with the same exact, case-sensitive name. Destination columns may be reordered and additional source columns may occur in any position. A destination column absent from the source throws `THERE_IS_NO_COLUMN`; there is no positional fallback. +- `ignore_extra_source_columns_by_name` - the source may have more columns than the destination. If it does, every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be reordered and the extra source columns may occur in any position; a destination column absent from the source throws `THERE_IS_NO_COLUMN`, with no positional fallback. When the source does not have more columns than the destination, this mode behaves exactly like `strict`: columns are matched positionally, and a mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. )", 0) \ \ /* ####################################################### */ \ diff --git a/src/Storages/MergeTree/ExportPartTask.cpp b/src/Storages/MergeTree/ExportPartTask.cpp index 8aa3511afad4..8f33e48646cb 100644 --- a/src/Storages/MergeTree/ExportPartTask.cpp +++ b/src/Storages/MergeTree/ExportPartTask.cpp @@ -128,8 +128,9 @@ namespace schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position; auto source_columns = plan_for_part.getCurrentHeader()->getColumnsWithTypeAndName(); + const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); - if (ignore_extra_source_columns_by_position && source_columns.size() > destination_columns.size()) + if (ignore_extra_source_columns_by_position && src_has_extra_columns) { LOG_DEBUG(getLogger("ExportPartTask"), "Source has {} columns while destination has {} columns, " @@ -158,10 +159,14 @@ namespace source_columns = plan_for_part.getCurrentHeader()->getColumnsWithTypeAndName(); } + const bool ignore_extra_source_columns_by_name = + schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name + && src_has_extra_columns; + auto dag = ActionsDAG::makeConvertingActions( source_columns, destination_columns, - schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name + ignore_extra_source_columns_by_name ? ActionsDAG::MatchColumnsMode::Name : ActionsDAG::MatchColumnsMode::Position, local_context); diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 36451885ff7e..e482a6babbb8 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -972,7 +972,10 @@ namespace MergeTreePartExportSchemaMismatchMode schema_mismatch_mode, const StorageID & destination_storage_id) { - if (schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name) + const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); + + if (schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name + && src_has_extra_columns) { std::unordered_map source_columns_by_name; source_columns_by_name.reserve(source_columns.size()); @@ -1030,10 +1033,11 @@ namespace /// `makeConvertingActions`, in both modes. const auto schema_mismatch_mode = context->getSettingsRef()[Setting::export_merge_tree_part_schema_mismatch_mode].value; + const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); const bool ignore_extra_source_columns_by_position = schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position; - if (ignore_extra_source_columns_by_position && source_columns.size() > destination_columns.size()) + if (ignore_extra_source_columns_by_position && src_has_extra_columns) { LOG_DEBUG(getLogger("ExportPartitionUtils"), "Source has {} columns while destination has {} columns, " @@ -1044,10 +1048,14 @@ namespace source_columns.resize(destination_columns.size()); } + const bool ignore_extra_source_columns_by_name = + schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name + && src_has_extra_columns; + (void) ActionsDAG::makeConvertingActions( source_columns, destination_columns, - schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name + ignore_extra_source_columns_by_name ? ActionsDAG::MatchColumnsMode::Name : ActionsDAG::MatchColumnsMode::Position, context); @@ -1066,7 +1074,7 @@ namespace const bool allow_lossy_cast = context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; - if (schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name) + if (ignore_extra_source_columns_by_name) { std::unordered_map source_positions_by_name; source_positions_by_name.reserve(source_columns.size()); diff --git a/src/Storages/MergeTree/ExportPartitionUtils.h b/src/Storages/MergeTree/ExportPartitionUtils.h index 492bb178b3e2..5c164d43de09 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.h +++ b/src/Storages/MergeTree/ExportPartitionUtils.h @@ -104,8 +104,10 @@ namespace ExportPartitionUtils /// source with more columns than the destination is allowed: the extra trailing /// source columns (by position) are excluded from the comparison here, matching /// what `ExportPartTask::addExportConvertingActions` drops from the actual data. - /// If the mode is `ignore_extra_source_columns_by_name`, destination columns are - /// matched to source columns by their exact names and may appear in a different order. + /// If the mode is `ignore_extra_source_columns_by_name` and the source has more columns + /// than the destination, destination columns are matched to source columns by their exact + /// names and may appear in a different order. With an equal (or smaller) column count this + /// mode has no extra source columns to ignore and behaves exactly like `strict`. /// /// Throws BAD_ARGUMENTS on any violation. void verifyExportSchemaCastable( diff --git a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp index 8186db87d944..0dde781e4871 100644 --- a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp +++ b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp @@ -207,6 +207,7 @@ TEST(ExportColumnCastsTest, UsesSelectedMatchingMode) makeColumn("id"), makeColumn("year"), makeColumn("payload"), + makeColumn("extra"), }; const ColumnsWithTypeAndName destination_columns = { makeColumn("payload"), @@ -233,6 +234,50 @@ TEST(ExportColumnCastsTest, UsesSelectedMatchingMode) destination_storage_id)); } +TEST(ExportColumnCastsTest, MatchingByNameNoOpWithoutExtraSourceColumns) +{ + const ColumnsWithTypeAndName source_columns = { + makeColumn("id"), + makeColumn("year"), + makeColumn("payload"), + }; + const ColumnsWithTypeAndName reordered_destination_columns = { + makeColumn("payload"), + makeColumn("year"), + makeColumn("id"), + }; + const StorageID destination_storage_id{"test", "destination"}; + + for (const auto mode : + {MergeTreePartExportSchemaMismatchMode::strict, + MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position, + MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name}) + { + expectExceptionCode( + [&] + { + ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, reordered_destination_columns, mode, destination_storage_id); + }, + ErrorCodes::INCOMPATIBLE_COLUMNS); + } + + const ColumnsWithTypeAndName same_order_destination_columns = { + makeColumn("id"), + makeColumn("year"), + makeColumn("payload"), + }; + + for (const auto mode : + {MergeTreePartExportSchemaMismatchMode::strict, + MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position, + MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name}) + { + EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, same_order_destination_columns, mode, destination_storage_id)); + } +} + TEST(ExportColumnCastsTest, RejectsLossyCastAfterMatchingByName) { const ColumnsWithTypeAndName source_columns = { diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index da27618d142b..ca8cd1301542 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -779,7 +779,7 @@ def test_export_part_source_more_columns_allowed_with_ignore_extra_setting(clust "mismatch_mode,expected_error", [ pytest.param("ignore_extra_source_columns_by_position", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), - pytest.param("ignore_extra_source_columns_by_name", "THERE_IS_NO_COLUMN", id="by-name"), + pytest.param("ignore_extra_source_columns_by_name", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-name"), ], ) def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_ignore_extra_setting( diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index 4256359f9dd1..d732b6d91f03 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -989,7 +989,7 @@ def test_export_part_column_count_mismatch_source_fewer_is_rejected(cluster): "mismatch_mode,expected_error", [ pytest.param("ignore_extra_source_columns_by_position", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), - pytest.param("ignore_extra_source_columns_by_name", "THERE_IS_NO_COLUMN", id="by-name"), + pytest.param("ignore_extra_source_columns_by_name", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-name"), ], ) def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_ignore_extra_setting( diff --git a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py b/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py index 96d19c3472ce..69f018ec01d5 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py @@ -1649,7 +1649,7 @@ def test_export_partition_source_more_columns_allowed_with_ignore_extra_setting( "mismatch_mode,expected_error", [ pytest.param("ignore_extra_source_columns_by_position", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), - pytest.param("ignore_extra_source_columns_by_name", "THERE_IS_NO_COLUMN", id="by-name"), + pytest.param("ignore_extra_source_columns_by_name", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-name"), ], ) def test_export_partition_column_count_mismatch_source_fewer_still_rejected_with_ignore_extra_setting( From 98fdc517adc59b5f58b2dc0b7ba02cccee7fea6d Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Tue, 18 Aug 2026 13:19:43 +0200 Subject: [PATCH 05/13] update tests Signed-off-by: Konstantin Morozov --- .../test.py | 43 +++++++++++++------ 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index ca8cd1301542..e5691a3e1a11 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -1082,27 +1082,16 @@ def test_export_part_reordered_subset_requires_matching_by_name(cluster): node.query(f"DROP TABLE IF EXISTS {iceberg}") -@pytest.mark.parametrize( - "source_columns,values", - [ - pytest.param("id Int32, year Int32", "(1, 2020), (2, 2020)", id="same-column-count"), - pytest.param( - "id Int32, year Int32, extra String", - "(1, 2020, 'first'), (2, 2020, 'second')", - id="extra-source-column", - ), - ], -) -def test_export_part_match_by_name_requires_every_destination_column(cluster, source_columns, values): +def test_export_part_match_by_name_requires_every_destination_column(cluster): node = cluster.instances["node1"] sfx = unique_suffix() mt = f"mt_match_by_name_missing_{sfx}" iceberg = f"iceberg_match_by_name_missing_{sfx}" - make_mt(node, mt, source_columns, "year") + make_mt(node, mt, "id Int32, year Int32, extra String", "year") make_iceberg_s3(node, iceberg, "renamed_id Int32, year Int32", "year") - node.query(f"INSERT INTO {mt} VALUES {values}") + node.query(f"INSERT INTO {mt} VALUES (1, 2020, 'first'), (2, 2020, 'second')") part_2020 = get_part(node, mt, "2020") error = node.query_and_get_error( @@ -1121,6 +1110,32 @@ def test_export_part_match_by_name_requires_every_destination_column(cluster, so node.query(f"DROP TABLE IF EXISTS {iceberg}") +def test_export_part_match_by_name_falls_back_to_positional_without_extra_source_columns(cluster): + # No extra source column here, so `ignore_extra_source_columns_by_name` behaves like `strict`. + node = cluster.instances["node1"] + sfx = unique_suffix() + mt = f"mt_match_by_name_fallback_{sfx}" + iceberg = f"iceberg_match_by_name_fallback_{sfx}" + + make_mt(node, mt, "id Int32, year Int32", "year") + make_iceberg_s3(node, iceberg, "renamed_id Int32, year Int32", "year") + + node.query(f"INSERT INTO {mt} VALUES (1, 2020), (2, 2020)") + part_2020 = get_part(node, mt, "2020") + + export_part( + node, mt, part_2020, iceberg, + extra_settings="export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_name'", + ) + wait_for_export_part(node, mt, part_2020) + + result = node.query(f"SELECT renamed_id, year FROM {iceberg} ORDER BY renamed_id").strip() + assert result == "1\t2020\n2\t2020", f"Unexpected data:\n{result}" + + node.query(f"DROP TABLE IF EXISTS {mt} SYNC") + node.query(f"DROP TABLE IF EXISTS {iceberg}") + + def test_export_part_with_castable_widening(cluster): """ Source column is Int32, destination expects Int64. makeConvertingActions From a6bf40864d9fad4779312752fcf208dd302c1e46 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Fri, 21 Aug 2026 18:47:42 +0200 Subject: [PATCH 06/13] change settings Signed-off-by: Konstantin Morozov --- docs/en/antalya/part_export.md | 34 ++-- docs/en/antalya/partition_export.md | 34 ++-- src/Core/Settings.cpp | 18 +- src/Core/Settings.h | 2 +- src/Core/SettingsChangesHistory.cpp | 3 +- src/Core/SettingsEnums.cpp | 2 +- src/Core/SettingsEnums.h | 9 +- ...portReplicatedMergeTreePartitionManifest.h | 28 +-- src/Storages/MergeTree/ExportPartTask.cpp | 20 +-- .../MergeTree/ExportPartitionUtils.cpp | 75 ++++---- src/Storages/MergeTree/ExportPartitionUtils.h | 22 +-- .../tests/gtest_export_partition_ordering.cpp | 164 ++++++++++++++---- src/Storages/StorageReplicatedMergeTree.cpp | 6 +- .../test.py | 100 ++++++++--- .../test.py | 95 ++++++++-- .../test.py | 73 ++++---- .../test.py | 54 +++++- 17 files changed, 521 insertions(+), 218 deletions(-) diff --git a/docs/en/antalya/part_export.md b/docs/en/antalya/part_export.md index 59f93cd1e0b8..413e5101956c 100644 --- a/docs/en/antalya/part_export.md +++ b/docs/en/antalya/part_export.md @@ -49,13 +49,13 @@ SETTINGS allow_experimental_export_merge_tree_part = 1 Source and destination tables must support positional schema conversion. The following differences between the two schemas are allowed: -- **Column names** may differ between source and destination for non-partition-key columns - columns are matched by position, similar to `INSERT INTO dest SELECT * FROM src`, not by name. +- **Column names** may differ between source and destination for non-partition-key columns when `export_merge_tree_part_schema_match_mode = 'match_by_position'` (the default) - columns are matched by position, similar to `INSERT INTO dest SELECT * FROM src`, not by name. Set `export_merge_tree_part_schema_match_mode = 'match_by_name'` to match columns by their exact, case-sensitive name instead, allowing destination columns to be declared in a different order than the source. - **Column types** may differ, as long as the source type is safely castable to the destination type. Set `export_merge_tree_part_allow_lossy_cast = 1` to also permit lossy casts. - **`Tuple` element names** may differ if either the source or destination declares the tuple without named elements: an unnamed `Tuple` (e.g. `Tuple(Int32, Int32)`) is matched against the destination by element position and type only, not by name. For example, exporting from `t Tuple(Int32, Int32)` to `t Tuple(x Int32, y Int32)` is allowed as long as element types match positionally. The following requirements apply to the source and destination: -1. **Column count** - source and destination must have the same number of columns by default. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. Set `export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_position'` to allow a source table with extra trailing columns; the destination having more columns than the source is still rejected in this mode. +1. **Column count** - by default (`ignore_extra_source_columns = false`) every source column must have a corresponding destination column: with `export_merge_tree_part_schema_match_mode = 'match_by_position'` (the default) the source and destination must have the same number of columns; with `'match_by_name'` they must have the same set of column names. A mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. Set `ignore_extra_source_columns = 1` to allow a source table with columns that have no corresponding destination column; such columns are dropped and not exported. The destination having a column absent from the source is always rejected, regardless of this setting. 2. **`PARTITION BY` expressions** - the whole part must land in a single destination partition. Identical expressions always satisfy this; otherwise the destination expression has to be computable from the values the source partition key pins, or be proven single-valued over the part's min/max range. The same requirement applies to the partition fields and transforms of an Apache Iceberg destination. See [Source partition key compatibility](/docs/en/antalya/partition_export.md#source-partition-key-compatibility). 3. **The position of every column backing the partition key** - it is not enough for the `PARTITION BY` expressions to be textually identical: every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. If such a column contains a named `Tuple`, its element names must also be declared in the same order (an unnamed `Tuple` on either side is exempt from this, per the allowance above). This comparison is recursive through nested tuples and through container types such as `Array` and `Map`. @@ -136,23 +136,31 @@ In case a table function is used as the destination, the schema can be omitted a **Warning:** A lossy cast on a partition column remains semantically truncating. For example, if a table is partitioned by an `Int64` column and some partition values do not fit into a destination `Int32` partition column, both the data files and the Iceberg metadata will contain the truncated `Int32` value (they agree with each other, but the original `Int64` value is lost). Such casts require `export_merge_tree_part_allow_lossy_cast = 1`. -### `export_merge_tree_part_schema_mismatch_mode` (Optional) +### `export_merge_tree_part_schema_match_mode` (Optional) -- **Type**: `MergeTreePartExportSchemaMismatchMode` -- **Default**: `strict` -- **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` allows the source `MergeTree` table to have more columns than the destination table and how destination columns are matched in that case. Possible values: - - `strict` - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`, and the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - - `ignore_extra_source_columns_by_position` - the source may have more columns than the destination. The extra trailing source columns (by position) are dropped and not exported. The destination having more columns than the source is still rejected in this mode. - - `ignore_extra_source_columns_by_name` - only takes effect when the source has more columns than the destination. In that case, every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be reordered and the extra source columns may occur in any position. Unmatched source columns are not exported. If a destination column is absent from the source, including when it was renamed, the export throws `THERE_IS_NO_COLUMN`; there is no positional fallback. When the source does **not** have more columns than the destination, this mode behaves exactly like `strict` - columns are matched positionally, and a column-count mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. Use `ignore_extra_source_columns_by_position` when renamed columns should be matched by position. +- **Type**: `MergeTreePartExportSchemaMatchMode` +- **Default**: `match_by_position` +- **Description**: Controls how `EXPORT PART`/`EXPORT PARTITION` matches source `MergeTree` columns to destination columns. Possible values: + - `match_by_position` (default) - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Column names are not otherwise considered. + - `match_by_name` - every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be declared in a different order than the source. A destination column absent from the source, including when it was renamed, throws `THERE_IS_NO_COLUMN`; there is no positional fallback. + + See `ignore_extra_source_columns` below for how a source column without a corresponding destination column is handled in each mode. + +### `ignore_extra_source_columns` (Optional) + +- **Type**: `Bool` +- **Default**: `false` +- **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` tolerates source `MergeTree` columns that have no corresponding destination column. + - `false` (default) - such a source column is rejected: the source and destination must match exactly. With `export_merge_tree_part_schema_match_mode = 'match_by_position'` this means the same number of columns; with `'match_by_name'` this means the same set of column names, so a source table with columns absent from the destination is rejected even if the matched columns would otherwise be compatible. A mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. + - `true` - a source column without a corresponding destination column is dropped and not exported, instead of throwing. In `match_by_position` mode, this allows a source with extra trailing columns (the destination having more columns than the source is still always rejected). In `match_by_name` mode, this allows source columns whose name has no destination counterpart, which may occur in any position. Extra source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. Type conversion and `export_merge_tree_part_allow_lossy_cast` are applied after columns are matched. Error behavior: - - In `strict` mode, a different number of source and destination columns throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - - In `ignore_extra_source_columns_by_position` mode, the destination having more columns than the source throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - - In `ignore_extra_source_columns_by_name` mode, when the source has more columns than the destination, a destination column absent from the source (including a renamed one) throws `THERE_IS_NO_COLUMN`; the mode does not fall back to positional matching in that case. - - In `ignore_extra_source_columns_by_name` mode, when the source does not have more columns than the destination, a column-count mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`, same as `strict`. + - With `ignore_extra_source_columns = false` (default), a source column without a corresponding destination column throws `NUMBER_OF_COLUMNS_DOESNT_MATCH` - in `match_by_position` mode this means any column-count mismatch, in `match_by_name` mode this means the source and destination column-name sets differ. + - In `match_by_position` mode, the destination having more columns than the source always throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`, regardless of `ignore_extra_source_columns`. + - In `match_by_name` mode, a destination column absent from the source (including a renamed one) always throws `THERE_IS_NO_COLUMN`, regardless of `ignore_extra_source_columns`; there is no positional fallback. - After columns have been matched successfully, a cast rejected by the export type-safety check throws `INCOMPATIBLE_COLUMNS`. diff --git a/docs/en/antalya/partition_export.md b/docs/en/antalya/partition_export.md index 4548a32117c7..9e03e62fe2a0 100644 --- a/docs/en/antalya/partition_export.md +++ b/docs/en/antalya/partition_export.md @@ -52,9 +52,9 @@ TO TABLE [destination_database.]destination_table ## Requirements -`EXPORT PARTITION` exports each part via the same mechanism as [`EXPORT PART`](/docs/en/antalya/part_export.md#requirements), so the source and destination tables must satisfy the same compatibility requirements. Column names may differ (columns are matched by position, not by name), and column types may differ as long as they are safely castable (or `export_merge_tree_part_allow_lossy_cast = 1` is set). Beyond that, the following requirements apply: +`EXPORT PARTITION` exports each part via the same mechanism as [`EXPORT PART`](/docs/en/antalya/part_export.md#requirements), so the source and destination tables must satisfy the same compatibility requirements. Columns are matched by position by default, or by their exact, case-sensitive name if `export_merge_tree_part_schema_match_mode = 'match_by_name'` is set, and column types may differ as long as they are safely castable (or `export_merge_tree_part_allow_lossy_cast = 1` is set). Beyond that, the following requirements apply: -1. **Column count** - source and destination must have the same number of columns by default. Set `export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_position'` to allow a source table with extra trailing columns; the destination having more columns than the source is still rejected in this mode. +1. **Column count** - by default (`ignore_extra_source_columns = false`) every source column must have a corresponding destination column: with `export_merge_tree_part_schema_match_mode = 'match_by_position'` (the default) the source and destination must have the same number of columns; with `'match_by_name'` they must have the same set of column names. Set `ignore_extra_source_columns = 1` to allow a source table with columns that have no corresponding destination column; the destination having a column absent from the source is still always rejected. 2. **`PARTITION BY` expressions** - the whole source partition must land in a single destination partition. Identical expressions always satisfy this; otherwise the destination expression has to be computable from the values the source partition key pins, or be proven single-valued over the partition's min/max range. The same requirement applies to the partition fields and transforms of an Apache Iceberg destination. See [Source partition key compatibility](#source-partition-key-compatibility). 3. **Partition key column positions and layouts** - every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. Named `Tuple` elements within such a column must also be declared in the same order, including tuples nested inside `Array` or `Map`. This applies even if both tables' `PARTITION BY` expressions are textually identical. See [`EXPORT PART` requirements](/docs/en/antalya/part_export.md#requirements) for a worked example and the corresponding exception message. @@ -135,23 +135,31 @@ Notes: **Warning:** A lossy cast on a partition column remains semantically truncating. For example, if a table is partitioned by an `Int64` column and some partition values do not fit into a destination `Int32` partition column, both the data files and the Iceberg metadata will contain the truncated `Int32` value (they agree with each other, but the original `Int64` value is lost). Such casts require `export_merge_tree_part_allow_lossy_cast = 1`. -### `export_merge_tree_part_schema_mismatch_mode` (Optional) +### `export_merge_tree_part_schema_match_mode` (Optional) -- **Type**: `MergeTreePartExportSchemaMismatchMode` -- **Default**: `strict` -- **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` allows the source `MergeTree` table to have more columns than the destination table and how destination columns are matched in that case. Possible values: - - `strict` - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`, and the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - - `ignore_extra_source_columns_by_position` - the source may have more columns than the destination. The extra trailing source columns (by position) are dropped and not exported. The destination having more columns than the source is still rejected in this mode. - - `ignore_extra_source_columns_by_name` - only takes effect when the source has more columns than the destination. In that case, every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be reordered and the extra source columns may occur in any position. Unmatched source columns are not exported. If a destination column is absent from the source, including when it was renamed, the export throws `THERE_IS_NO_COLUMN`; there is no positional fallback. When the source does **not** have more columns than the destination, this mode behaves exactly like `strict` - columns are matched positionally, and a column-count mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. Use `ignore_extra_source_columns_by_position` when renamed columns should be matched by position. +- **Type**: `MergeTreePartExportSchemaMatchMode` +- **Default**: `match_by_position` +- **Description**: Controls how `EXPORT PART`/`EXPORT PARTITION` matches source `MergeTree` columns to destination columns. Possible values: + - `match_by_position` (default) - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Column names are not otherwise considered. + - `match_by_name` - every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be declared in a different order than the source. A destination column absent from the source, including when it was renamed, throws `THERE_IS_NO_COLUMN`; there is no positional fallback. + + See `ignore_extra_source_columns` below for how a source column without a corresponding destination column is handled in each mode. + +### `ignore_extra_source_columns` (Optional) + +- **Type**: `Bool` +- **Default**: `false` +- **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` tolerates source `MergeTree` columns that have no corresponding destination column. + - `false` (default) - such a source column is rejected: the source and destination must match exactly. With `export_merge_tree_part_schema_match_mode = 'match_by_position'` this means the same number of columns; with `'match_by_name'` this means the same set of column names, so a source table with columns absent from the destination is rejected even if the matched columns would otherwise be compatible. A mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. + - `true` - a source column without a corresponding destination column is dropped and not exported, instead of throwing. In `match_by_position` mode, this allows a source with extra trailing columns (the destination having more columns than the source is still always rejected). In `match_by_name` mode, this allows source columns whose name has no destination counterpart, which may occur in any position. Extra source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. Type conversion and `export_merge_tree_part_allow_lossy_cast` are applied after columns are matched. Error behavior: - - In `strict` mode, a different number of source and destination columns throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - - In `ignore_extra_source_columns_by_position` mode, the destination having more columns than the source throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - - In `ignore_extra_source_columns_by_name` mode, when the source has more columns than the destination, a destination column absent from the source (including a renamed one) throws `THERE_IS_NO_COLUMN`; the mode does not fall back to positional matching in that case. - - In `ignore_extra_source_columns_by_name` mode, when the source does not have more columns than the destination, a column-count mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`, same as `strict`. + - With `ignore_extra_source_columns = false` (default), a source column without a corresponding destination column throws `NUMBER_OF_COLUMNS_DOESNT_MATCH` - in `match_by_position` mode this means any column-count mismatch, in `match_by_name` mode this means the source and destination column-name sets differ. + - In `match_by_position` mode, the destination having more columns than the source always throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`, regardless of `ignore_extra_source_columns`. + - In `match_by_name` mode, a destination column absent from the source (including a renamed one) always throws `THERE_IS_NO_COLUMN`, regardless of `ignore_extra_source_columns`; there is no positional fallback. - After columns have been matched successfully, a cast rejected by the export type-safety check throws `INCOMPATIBLE_COLUMNS`. ## Examples diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index c9f364eeaf51..45eb39385754 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -7627,12 +7627,20 @@ Allow `EXPORT PART`/`EXPORT PARTITION` to apply lossy (non-value-preserving) cas When exporting to Apache Iceberg, the partition value written to the metadata is derived from the source partition columns by casting them to the destination partition-field types and applying the destination partition transform — the same computation the exported data files use, so the metadata stays consistent with the data. A lossy cast on a partition column remains semantically truncating: both the data files and the metadata contain the truncated value, and such casts require this setting to be enabled. )", 0) \ - DECLARE(MergeTreePartExportSchemaMismatchMode, export_merge_tree_part_schema_mismatch_mode, MergeTreePartExportSchemaMismatchMode::strict, R"( -Controls whether `EXPORT PART`/`EXPORT PARTITION` allows the source `MergeTree` table to have more columns than the destination table and how destination columns are matched. + DECLARE(MergeTreePartExportSchemaMatchMode, export_merge_tree_part_schema_match_mode, MergeTreePartExportSchemaMatchMode::match_by_position, R"( +Controls how `EXPORT PART`/`EXPORT PARTITION` matches source `MergeTree` columns to destination columns. Possible values: -- `strict` (default) - columns are matched positionally and the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. -- `ignore_extra_source_columns_by_position` - the source may have more columns than the destination. The extra trailing source columns (by position) are dropped and not exported. The destination having more columns than the source is still rejected in this mode. -- `ignore_extra_source_columns_by_name` - the source may have more columns than the destination. If it does, every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be reordered and the extra source columns may occur in any position; a destination column absent from the source throws `THERE_IS_NO_COLUMN`, with no positional fallback. When the source does not have more columns than the destination, this mode behaves exactly like `strict`: columns are matched positionally, and a mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. +- `match_by_position` (default) - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Column names are not otherwise considered. +- `match_by_name` - every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be declared in a different order than the source. A destination column absent from the source throws `THERE_IS_NO_COLUMN`, with no positional fallback. + +See also `ignore_extra_source_columns`, which controls whether a source column without a corresponding destination column is dropped or rejected. +)", 0) \ + DECLARE(Bool, ignore_extra_source_columns, false, R"( +Controls whether `EXPORT PART`/`EXPORT PARTITION` tolerates source `MergeTree` columns that have no corresponding destination column. +- `false` (default) - such a source column is rejected: the source and destination must match exactly (in `export_merge_tree_part_schema_match_mode = 'match_by_position'`, this means the same number of columns; in `'match_by_name'`, the same set of column names). A mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. +- `true` - a source column without a corresponding destination column is dropped and not exported, instead of throwing. In `match_by_position` mode, this allows a source with extra trailing columns (the destination having more columns than the source is still always rejected). In `match_by_name` mode, this allows source columns whose name has no destination counterpart. + +Extra source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. )", 0) \ \ /* ####################################################### */ \ diff --git a/src/Core/Settings.h b/src/Core/Settings.h index a5fef6160201..a166435fcd95 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -85,7 +85,7 @@ class WriteBuffer; M(CLASS_NAME, Map) \ M(CLASS_NAME, MaxThreads) \ M(CLASS_NAME, MergeTreePartExportFileAlreadyExistsPolicy) \ - M(CLASS_NAME, MergeTreePartExportSchemaMismatchMode) \ + M(CLASS_NAME, MergeTreePartExportSchemaMatchMode) \ M(CLASS_NAME, Milliseconds) \ M(CLASS_NAME, MsgPackUUIDRepresentation) \ M(CLASS_NAME, MySQLDataTypesSupport) \ diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 60b9114149a6..580d09fc8703 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -44,7 +44,8 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"object_storage_cluster_join_mode", "allow", "allow", "New setting"}, {"export_merge_tree_partition_task_timeout_seconds", "3600", "86400", "Increase default value to make it more realistic"}, {"export_merge_tree_part_allow_lossy_cast", false, false, "New setting to gate lossy casts in EXPORT PART/PARTITION behind explicit acknowledgment"}, - {"export_merge_tree_part_schema_mismatch_mode", "strict", "strict", "New setting to allow EXPORT PART/EXPORT PARTITION when the source table has more columns than the destination"}, + {"export_merge_tree_part_schema_match_mode", "match_by_position", "match_by_position", "New setting to control how EXPORT PART/EXPORT PARTITION matches source columns to destination columns"}, + {"ignore_extra_source_columns", false, false, "New setting to allow EXPORT PART/EXPORT PARTITION when the source table has columns absent from the destination"}, {"export_merge_tree_partition_retry_initial_backoff_seconds", 5, 5, "New setting for exponential back-off between failed part export retries in an export partition task"}, {"export_merge_tree_partition_retry_max_backoff_seconds", 300, 300, "New setting capping the exponential back-off between failed part export retries in an export partition task"}, {"export_merge_tree_partition_max_retries", 3, 3, "Obsolete and ignored: export partition tasks now retry retryable failures until the task timeout and fail immediately on non-retryable errors, instead of using a fixed retry budget"}, diff --git a/src/Core/SettingsEnums.cpp b/src/Core/SettingsEnums.cpp index 9aadee2db780..351f8f6be7a2 100644 --- a/src/Core/SettingsEnums.cpp +++ b/src/Core/SettingsEnums.cpp @@ -483,7 +483,7 @@ IMPLEMENT_SETTING_ENUM(JemallocProfileFormat, ErrorCodes::BAD_ARGUMENTS, IMPLEMENT_SETTING_AUTO_ENUM(MergeTreePartExportFileAlreadyExistsPolicy, ErrorCodes::BAD_ARGUMENTS); -IMPLEMENT_SETTING_AUTO_ENUM(MergeTreePartExportSchemaMismatchMode, ErrorCodes::BAD_ARGUMENTS); +IMPLEMENT_SETTING_AUTO_ENUM(MergeTreePartExportSchemaMatchMode, ErrorCodes::BAD_ARGUMENTS); IMPLEMENT_SETTING_AUTO_ENUM(ExportPartitionAllOnError, ErrorCodes::BAD_ARGUMENTS); diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index f5c48509e0b0..a325835452d5 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -574,14 +574,13 @@ enum class MergeTreePartExportFileAlreadyExistsPolicy : uint8_t DECLARE_SETTING_ENUM(MergeTreePartExportFileAlreadyExistsPolicy) -enum class MergeTreePartExportSchemaMismatchMode : uint8_t +enum class MergeTreePartExportSchemaMatchMode : uint8_t { - strict, - ignore_extra_source_columns_by_position, - ignore_extra_source_columns_by_name, + match_by_position, + match_by_name, }; -DECLARE_SETTING_ENUM(MergeTreePartExportSchemaMismatchMode) +DECLARE_SETTING_ENUM(MergeTreePartExportSchemaMatchMode) enum class ExportPartitionAllOnError : uint8_t { diff --git a/src/Storages/ExportReplicatedMergeTreePartitionManifest.h b/src/Storages/ExportReplicatedMergeTreePartitionManifest.h index 7857302b1261..3af5b82dfd7d 100644 --- a/src/Storages/ExportReplicatedMergeTreePartitionManifest.h +++ b/src/Storages/ExportReplicatedMergeTreePartitionManifest.h @@ -248,7 +248,8 @@ struct ExportReplicatedMergeTreePartitionManifest std::optional output_format_compression_level; std::optional parquet_row_group_size; std::optional parquet_row_group_size_bytes; - std::optional schema_mismatch_mode; + std::optional schema_match_mode; + std::optional ignore_extra_source_columns; /// this is a controversial setting. As far as I can infer from the iceberg docs, the transforms are always UTC. /// this setting allows to specify different timezones. Since it is already implemented, we must respect it. @@ -298,8 +299,10 @@ struct ExportReplicatedMergeTreePartitionManifest json.set("parquet_row_group_size_bytes", *parquet_row_group_size_bytes); if (iceberg_partition_timezone) json.set("iceberg_partition_timezone", *iceberg_partition_timezone); - if (schema_mismatch_mode) - json.set("schema_mismatch_mode", String(magic_enum::enum_name(*schema_mismatch_mode))); + if (schema_match_mode) + json.set("schema_match_mode", String(magic_enum::enum_name(*schema_match_mode))); + if (ignore_extra_source_columns) + json.set("ignore_extra_source_columns", *ignore_extra_source_columns); std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM oss.exceptions(std::ios::failbit); Poco::JSON::Stringifier::stringify(json, oss); @@ -367,15 +370,20 @@ struct ExportReplicatedMergeTreePartitionManifest /// on upgrade. New tasks always persist the initiator's actual choice. manifest.allow_lossy_cast = json->has("allow_lossy_cast") ? json->getValue("allow_lossy_cast") : true; - /// Left unset (nullopt) for tasks created before this field existed - such tasks were - /// always scheduled under the old, strict column-count check (a mismatch could never + /// Left unset (nullopt) for tasks created before these fields existed - such tasks were + /// always scheduled under the old, strict column-matching check (a mismatch could never /// reach scheduling in the first place), so callers should treat an absent value as - /// `strict`. - if (json->has("schema_mismatch_mode")) + /// `match_by_position` with `ignore_extra_source_columns = false`. + if (json->has("schema_match_mode")) { - const auto schema_mismatch_mode = magic_enum::enum_cast(json->getValue("schema_mismatch_mode")); - if (schema_mismatch_mode) - manifest.schema_mismatch_mode = schema_mismatch_mode; + const auto schema_match_mode = magic_enum::enum_cast(json->getValue("schema_match_mode")); + if (schema_match_mode) + manifest.schema_match_mode = schema_match_mode; + } + + if (json->has("ignore_extra_source_columns")) + { + manifest.ignore_extra_source_columns = json->getValue("ignore_extra_source_columns"); } if (json->has("parquet_compression_method")) diff --git a/src/Storages/MergeTree/ExportPartTask.cpp b/src/Storages/MergeTree/ExportPartTask.cpp index 8f33e48646cb..2126601ce7c3 100644 --- a/src/Storages/MergeTree/ExportPartTask.cpp +++ b/src/Storages/MergeTree/ExportPartTask.cpp @@ -65,7 +65,8 @@ namespace Setting extern const SettingsUInt64 export_merge_tree_part_max_rows_per_file; extern const SettingsBool allow_experimental_analyzer; extern const SettingsString export_merge_tree_part_filename_pattern; - extern const SettingsMergeTreePartExportSchemaMismatchMode export_merge_tree_part_schema_mismatch_mode; + extern const SettingsMergeTreePartExportSchemaMatchMode export_merge_tree_part_schema_match_mode; + extern const SettingsBool ignore_extra_source_columns; } namespace @@ -122,15 +123,16 @@ namespace = destination_storage.getInMemoryMetadataPtr()->getSampleBlockNonMaterialized(); const auto & destination_columns = destination_header.getColumnsWithTypeAndName(); - const auto schema_mismatch_mode = - local_context->getSettingsRef()[Setting::export_merge_tree_part_schema_mismatch_mode].value; - const bool ignore_extra_source_columns_by_position = - schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position; + const auto schema_match_mode = + local_context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value; + const bool ignore_extra_source_columns = + local_context->getSettingsRef()[Setting::ignore_extra_source_columns]; + const bool match_by_name = schema_match_mode == MergeTreePartExportSchemaMatchMode::match_by_name; auto source_columns = plan_for_part.getCurrentHeader()->getColumnsWithTypeAndName(); const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); - if (ignore_extra_source_columns_by_position && src_has_extra_columns) + if (!match_by_name && ignore_extra_source_columns && src_has_extra_columns) { LOG_DEBUG(getLogger("ExportPartTask"), "Source has {} columns while destination has {} columns, " @@ -159,14 +161,10 @@ namespace source_columns = plan_for_part.getCurrentHeader()->getColumnsWithTypeAndName(); } - const bool ignore_extra_source_columns_by_name = - schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name - && src_has_extra_columns; - auto dag = ActionsDAG::makeConvertingActions( source_columns, destination_columns, - ignore_extra_source_columns_by_name + match_by_name ? ActionsDAG::MatchColumnsMode::Name : ActionsDAG::MatchColumnsMode::Position, local_context); diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index e482a6babbb8..8a676111d531 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -97,7 +97,8 @@ namespace Setting #if USE_AVRO extern const SettingsTimezone iceberg_partition_timezone; #endif - extern const SettingsMergeTreePartExportSchemaMismatchMode export_merge_tree_part_schema_mismatch_mode; + extern const SettingsMergeTreePartExportSchemaMatchMode export_merge_tree_part_schema_match_mode; + extern const SettingsBool ignore_extra_source_columns; } namespace FailPoints @@ -217,12 +218,16 @@ namespace ExportPartitionUtils context_copy->setSetting("output_format_parquet_row_group_size", *manifest.parquet_row_group_size); if (manifest.parquet_row_group_size_bytes) context_copy->setSetting("output_format_parquet_row_group_size_bytes", *manifest.parquet_row_group_size_bytes); - /// Manifests written before this setting existed have no value here; such tasks were always - /// scheduled under the old, strict column-count check, so an absent value must resolve to - /// `strict` regardless of the ambient context's setting (which may have since been changed). + /// Manifests written before these settings existed have no value here; such tasks were always + /// scheduled under the old, strict column-matching check, so an absent value must resolve to + /// `match_by_position` / `false` regardless of the ambient context's settings (which may have + /// since been changed). context_copy->setSetting( - "export_merge_tree_part_schema_mismatch_mode", - String(magic_enum::enum_name(manifest.schema_mismatch_mode.value_or(MergeTreePartExportSchemaMismatchMode::strict)))); + "export_merge_tree_part_schema_match_mode", + String(magic_enum::enum_name(manifest.schema_match_mode.value_or(MergeTreePartExportSchemaMatchMode::match_by_position)))); + context_copy->setSetting( + "ignore_extra_source_columns", + manifest.ignore_extra_source_columns.value_or(false)); context_copy->setSetting("max_threads", manifest.max_threads); context_copy->setSetting("export_merge_tree_part_file_already_exists_policy", String(magic_enum::enum_name(manifest.file_already_exists_policy))); @@ -969,14 +974,23 @@ namespace void verifyExportColumnCastsAreSafe( const ColumnsWithTypeAndName & source_columns, const ColumnsWithTypeAndName & destination_columns, - MergeTreePartExportSchemaMismatchMode schema_mismatch_mode, + MergeTreePartExportSchemaMatchMode schema_match_mode, + bool ignore_extra_source_columns, const StorageID & destination_storage_id) { - const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); - - if (schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name - && src_has_extra_columns) + if (schema_match_mode == MergeTreePartExportSchemaMatchMode::match_by_name) { + /// `ignore_extra_source_columns` only excuses the source having MORE columns than the + /// destination; a source with fewer columns is always a mismatch. + const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); + if (source_columns.size() != destination_columns.size() + && !(ignore_extra_source_columns && src_has_extra_columns)) + throw Exception( + ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, + "Number of columns doesn't match (source: {} and result: {})", + source_columns.size(), + destination_columns.size()); + std::unordered_map source_columns_by_name; source_columns_by_name.reserve(source_columns.size()); for (const auto & source_column : source_columns) @@ -997,8 +1011,7 @@ namespace } if (source_columns.size() < destination_columns.size() - || (schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::strict - && source_columns.size() != destination_columns.size())) + || (!ignore_extra_source_columns && source_columns.size() != destination_columns.size())) throw Exception( ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, "Number of columns doesn't match (source: {} and result: {})", @@ -1026,18 +1039,14 @@ namespace auto source_columns = source_sample_block.getColumnsWithTypeAndName(); const auto & destination_columns = destination_sample_block.getColumnsWithTypeAndName(); - /// In `ignore_extra_source_columns_by_position` mode a source with more columns than the destination - /// is allowed: the extra trailing source columns (by position) are dropped, mirroring - /// the trimming `ExportPartTask::addExportConvertingActions` applies to the real data. - /// The reverse (destination has more columns than source) is always rejected below by - /// `makeConvertingActions`, in both modes. - const auto schema_mismatch_mode = - context->getSettingsRef()[Setting::export_merge_tree_part_schema_mismatch_mode].value; + const auto schema_match_mode = + context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value; + const bool ignore_extra_source_columns = + context->getSettingsRef()[Setting::ignore_extra_source_columns]; + const bool match_by_name = schema_match_mode == MergeTreePartExportSchemaMatchMode::match_by_name; const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); - const bool ignore_extra_source_columns_by_position = - schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position; - if (ignore_extra_source_columns_by_position && src_has_extra_columns) + if (!match_by_name && ignore_extra_source_columns && src_has_extra_columns) { LOG_DEBUG(getLogger("ExportPartitionUtils"), "Source has {} columns while destination has {} columns, " @@ -1048,14 +1057,21 @@ namespace source_columns.resize(destination_columns.size()); } - const bool ignore_extra_source_columns_by_name = - schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name - && src_has_extra_columns; + /// `makeConvertingActions` in `Name` mode silently ignores an unreferenced source column, so it must be rejected here explicitly. + /// `ignore_extra_source_columns` only excuses the source having MORE columns than the destination; + /// a source with fewer columns is always a mismatch. + if (match_by_name && source_columns.size() != destination_columns.size() + && !(ignore_extra_source_columns && src_has_extra_columns)) + throw Exception( + ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, + "Number of columns doesn't match (source: {} and result: {})", + source_columns.size(), + destination_columns.size()); (void) ActionsDAG::makeConvertingActions( source_columns, destination_columns, - ignore_extra_source_columns_by_name + match_by_name ? ActionsDAG::MatchColumnsMode::Name : ActionsDAG::MatchColumnsMode::Position, context); @@ -1074,7 +1090,7 @@ namespace const bool allow_lossy_cast = context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; - if (ignore_extra_source_columns_by_name) + if (match_by_name) { std::unordered_map source_positions_by_name; source_positions_by_name.reserve(source_columns.size()); @@ -1111,7 +1127,8 @@ namespace verifyExportColumnCastsAreSafe( source_columns, destination_columns, - schema_mismatch_mode, + schema_match_mode, + ignore_extra_source_columns, destination_storage_id); } } diff --git a/src/Storages/MergeTree/ExportPartitionUtils.h b/src/Storages/MergeTree/ExportPartitionUtils.h index 5c164d43de09..5757e8f658c8 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.h +++ b/src/Storages/MergeTree/ExportPartitionUtils.h @@ -92,24 +92,14 @@ namespace ExportPartitionUtils void verifyExportColumnCastsAreSafe( const ColumnsWithTypeAndName & source_columns, const ColumnsWithTypeAndName & destination_columns, - MergeTreePartExportSchemaMismatchMode schema_mismatch_mode, + MergeTreePartExportSchemaMatchMode schema_match_mode, + bool ignore_extra_source_columns, const StorageID & destination_storage_id); - /// Validates that source columns can be exported into the destination with the - /// configured positional or name-based CAST matching. Lossy casts are rejected - /// unless `export_merge_tree_part_allow_lossy_cast` is set. - /// - /// By default the source and destination must have the same number of columns. - /// If `export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_position'`, a - /// source with more columns than the destination is allowed: the extra trailing - /// source columns (by position) are excluded from the comparison here, matching - /// what `ExportPartTask::addExportConvertingActions` drops from the actual data. - /// If the mode is `ignore_extra_source_columns_by_name` and the source has more columns - /// than the destination, destination columns are matched to source columns by their exact - /// names and may appear in a different order. With an equal (or smaller) column count this - /// mode has no extra source columns to ignore and behaves exactly like `strict`. - /// - /// Throws BAD_ARGUMENTS on any violation. + /// Validates that source columns can be exported into the destination with the configured + /// positional or name-based CAST matching (`export_merge_tree_part_schema_match_mode`) and + /// unmatched-column policy (`ignore_extra_source_columns`). Lossy casts are rejected unless + /// `export_merge_tree_part_allow_lossy_cast` is set. Throws BAD_ARGUMENTS on any violation. void verifyExportSchemaCastable( const StorageMetadataPtr & source_metadata, const StorageMetadataPtr & destination_metadata, diff --git a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp index 0dde781e4871..71206642da1a 100644 --- a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp +++ b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp @@ -18,11 +18,13 @@ namespace ErrorCodes { extern const int INCOMPATIBLE_COLUMNS; extern const int THERE_IS_NO_COLUMN; + extern const int NUMBER_OF_COLUMNS_DOESNT_MATCH; } namespace Setting { - extern const SettingsMergeTreePartExportSchemaMismatchMode export_merge_tree_part_schema_mismatch_mode; + extern const SettingsMergeTreePartExportSchemaMatchMode export_merge_tree_part_schema_match_mode; + extern const SettingsBool ignore_extra_source_columns; } namespace @@ -144,63 +146,112 @@ TEST_F(ExportPartitionOrderingTest, IterationOrderMatchesCreateTime) } -TEST_F(ExportPartitionManifestBackCompatTest, MissingSchemaMismatchModeParsesAsNullopt) +TEST_F(ExportPartitionManifestBackCompatTest, MissingSchemaMatchModeParsesAsNullopt) { auto manifest = makeValidManifest(); - manifest.schema_mismatch_mode = MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position; + manifest.schema_match_mode = MergeTreePartExportSchemaMatchMode::match_by_name; Poco::JSON::Parser parser; auto json = parser.parse(manifest.toJsonString()).extract(); - json->remove("schema_mismatch_mode"); + json->remove("schema_match_mode"); std::ostringstream oss; oss.exceptions(std::ios::failbit); Poco::JSON::Stringifier::stringify(json, oss); auto parsed = ExportReplicatedMergeTreePartitionManifest::fromJsonString(oss.str()); - EXPECT_FALSE(parsed.schema_mismatch_mode.has_value()); + EXPECT_FALSE(parsed.schema_match_mode.has_value()); } -TEST_F(ExportPartitionManifestBackCompatTest, SchemaMismatchModeRoundTripsForEveryValue) +TEST_F(ExportPartitionManifestBackCompatTest, SchemaMatchModeRoundTripsForEveryValue) { - for (const auto value : magic_enum::enum_values()) + for (const auto value : magic_enum::enum_values()) { auto manifest = makeValidManifest(); - manifest.schema_mismatch_mode = value; + manifest.schema_match_mode = value; auto parsed = ExportReplicatedMergeTreePartitionManifest::fromJsonString(manifest.toJsonString()); - ASSERT_TRUE(parsed.schema_mismatch_mode.has_value()) << "value=" << magic_enum::enum_name(value); - EXPECT_EQ(*parsed.schema_mismatch_mode, value) << "value=" << magic_enum::enum_name(value); + ASSERT_TRUE(parsed.schema_match_mode.has_value()) << "value=" << magic_enum::enum_name(value); + EXPECT_EQ(*parsed.schema_match_mode, value) << "value=" << magic_enum::enum_name(value); } } -TEST_F(ExportPartitionManifestBackCompatTest, MissingSchemaMismatchModeFallsBackToStrictInWorkerContext) +TEST_F(ExportPartitionManifestBackCompatTest, MissingIgnoreExtraSourceColumnsParsesAsNullopt) { auto manifest = makeValidManifest(); - ASSERT_FALSE(manifest.schema_mismatch_mode.has_value()); + manifest.ignore_extra_source_columns = true; + + Poco::JSON::Parser parser; + auto json = parser.parse(manifest.toJsonString()).extract(); + json->remove("ignore_extra_source_columns"); + std::ostringstream oss; + oss.exceptions(std::ios::failbit); + Poco::JSON::Stringifier::stringify(json, oss); + + auto parsed = ExportReplicatedMergeTreePartitionManifest::fromJsonString(oss.str()); + EXPECT_FALSE(parsed.ignore_extra_source_columns.has_value()); +} + +TEST_F(ExportPartitionManifestBackCompatTest, IgnoreExtraSourceColumnsRoundTripsForEveryValue) +{ + for (const bool value : {false, true}) + { + auto manifest = makeValidManifest(); + manifest.ignore_extra_source_columns = value; + + auto parsed = ExportReplicatedMergeTreePartitionManifest::fromJsonString(manifest.toJsonString()); + + ASSERT_TRUE(parsed.ignore_extra_source_columns.has_value()) << "value=" << value; + EXPECT_EQ(*parsed.ignore_extra_source_columns, value) << "value=" << value; + } +} + +TEST_F(ExportPartitionManifestBackCompatTest, MissingSchemaMatchSettingsFallBackToDefaultsInWorkerContext) +{ + auto manifest = makeValidManifest(); + ASSERT_FALSE(manifest.schema_match_mode.has_value()); + ASSERT_FALSE(manifest.ignore_extra_source_columns.has_value()); auto worker_context = ExportPartitionUtils::getContextCopyWithTaskSettings(getContext().context, manifest); EXPECT_EQ( - worker_context->getSettingsRef()[Setting::export_merge_tree_part_schema_mismatch_mode].value, - MergeTreePartExportSchemaMismatchMode::strict); + worker_context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value, + MergeTreePartExportSchemaMatchMode::match_by_position); + EXPECT_EQ( + worker_context->getSettingsRef()[Setting::ignore_extra_source_columns].value, + false); } -TEST_F(ExportPartitionManifestBackCompatTest, SchemaMismatchModeAppliedToWorkerContextForEveryValue) +TEST_F(ExportPartitionManifestBackCompatTest, SchemaMatchModeAppliedToWorkerContextForEveryValue) { - for (const auto value : magic_enum::enum_values()) + for (const auto value : magic_enum::enum_values()) { auto manifest = makeValidManifest(); - manifest.schema_mismatch_mode = value; + manifest.schema_match_mode = value; auto worker_context = ExportPartitionUtils::getContextCopyWithTaskSettings(getContext().context, manifest); EXPECT_EQ( - worker_context->getSettingsRef()[Setting::export_merge_tree_part_schema_mismatch_mode].value, + worker_context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value, value) << "value=" << magic_enum::enum_name(value); } } +TEST_F(ExportPartitionManifestBackCompatTest, IgnoreExtraSourceColumnsAppliedToWorkerContextForEveryValue) +{ + for (const bool value : {false, true}) + { + auto manifest = makeValidManifest(); + manifest.ignore_extra_source_columns = value; + + auto worker_context = ExportPartitionUtils::getContextCopyWithTaskSettings(getContext().context, manifest); + + EXPECT_EQ( + worker_context->getSettingsRef()[Setting::ignore_extra_source_columns].value, + value) << "value=" << value; + } +} + TEST(ExportColumnCastsTest, UsesSelectedMatchingMode) { const ColumnsWithTypeAndName source_columns = { @@ -222,7 +273,8 @@ TEST(ExportColumnCastsTest, UsesSelectedMatchingMode) ExportPartitionUtils::verifyExportColumnCastsAreSafe( source_columns, destination_columns, - MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position, + MergeTreePartExportSchemaMatchMode::match_by_position, + /*ignore_extra_source_columns=*/ true, destination_storage_id); }, ErrorCodes::INCOMPATIBLE_COLUMNS); @@ -230,11 +282,12 @@ TEST(ExportColumnCastsTest, UsesSelectedMatchingMode) EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( source_columns, destination_columns, - MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name, + MergeTreePartExportSchemaMatchMode::match_by_name, + /*ignore_extra_source_columns=*/ true, destination_storage_id)); } -TEST(ExportColumnCastsTest, MatchingByNameNoOpWithoutExtraSourceColumns) +TEST(ExportColumnCastsTest, MatchByNameAcceptsReorderedColumnsWithEqualColumnCount) { const ColumnsWithTypeAndName source_columns = { makeColumn("id"), @@ -248,18 +301,26 @@ TEST(ExportColumnCastsTest, MatchingByNameNoOpWithoutExtraSourceColumns) }; const StorageID destination_storage_id{"test", "destination"}; - for (const auto mode : - {MergeTreePartExportSchemaMismatchMode::strict, - MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position, - MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name}) + for (const bool ignore_extra_source_columns : {false, true}) { expectExceptionCode( [&] { ExportPartitionUtils::verifyExportColumnCastsAreSafe( - source_columns, reordered_destination_columns, mode, destination_storage_id); + source_columns, + reordered_destination_columns, + MergeTreePartExportSchemaMatchMode::match_by_position, + ignore_extra_source_columns, + destination_storage_id); }, ErrorCodes::INCOMPATIBLE_COLUMNS); + + EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, + reordered_destination_columns, + MergeTreePartExportSchemaMatchMode::match_by_name, + ignore_extra_source_columns, + destination_storage_id)); } const ColumnsWithTypeAndName same_order_destination_columns = { @@ -268,14 +329,43 @@ TEST(ExportColumnCastsTest, MatchingByNameNoOpWithoutExtraSourceColumns) makeColumn("payload"), }; - for (const auto mode : - {MergeTreePartExportSchemaMismatchMode::strict, - MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position, - MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name}) - { - EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( - source_columns, same_order_destination_columns, mode, destination_storage_id)); - } + for (const auto mode : {MergeTreePartExportSchemaMatchMode::match_by_position, MergeTreePartExportSchemaMatchMode::match_by_name}) + for (const bool ignore_extra_source_columns : {false, true}) + EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, same_order_destination_columns, mode, ignore_extra_source_columns, destination_storage_id)); +} + +TEST(ExportColumnCastsTest, MatchByNameRejectsUnmatchedSourceColumnUnlessIgnored) +{ + const ColumnsWithTypeAndName source_columns = { + makeColumn("id"), + makeColumn("year"), + makeColumn("extra"), + }; + const ColumnsWithTypeAndName destination_columns = { + makeColumn("id"), + makeColumn("year"), + }; + const StorageID destination_storage_id{"test", "destination"}; + + expectExceptionCode( + [&] + { + ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, + destination_columns, + MergeTreePartExportSchemaMatchMode::match_by_name, + /*ignore_extra_source_columns=*/ false, + destination_storage_id); + }, + ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH); + + EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, + destination_columns, + MergeTreePartExportSchemaMatchMode::match_by_name, + /*ignore_extra_source_columns=*/ true, + destination_storage_id)); } TEST(ExportColumnCastsTest, RejectsLossyCastAfterMatchingByName) @@ -296,7 +386,8 @@ TEST(ExportColumnCastsTest, RejectsLossyCastAfterMatchingByName) ExportPartitionUtils::verifyExportColumnCastsAreSafe( source_columns, destination_columns, - MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name, + MergeTreePartExportSchemaMatchMode::match_by_name, + /*ignore_extra_source_columns=*/ true, StorageID{"test", "destination"}); }, ErrorCodes::INCOMPATIBLE_COLUMNS); @@ -320,7 +411,8 @@ TEST(ExportColumnCastsTest, RejectsMissingDestinationColumnAfterMatchingByName) ExportPartitionUtils::verifyExportColumnCastsAreSafe( source_columns, destination_columns, - MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name, + MergeTreePartExportSchemaMatchMode::match_by_name, + /*ignore_extra_source_columns=*/ true, StorageID{"test", "destination"}); }, ErrorCodes::THERE_IS_NO_COLUMN); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index a1257264ef22..6d09b6950bd5 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -229,7 +229,8 @@ namespace Setting extern const SettingsBool export_merge_tree_part_throw_on_pending_mutations; extern const SettingsBool export_merge_tree_part_throw_on_pending_patch_parts; extern const SettingsBool export_merge_tree_part_allow_lossy_cast; - extern const SettingsMergeTreePartExportSchemaMismatchMode export_merge_tree_part_schema_mismatch_mode; + extern const SettingsMergeTreePartExportSchemaMatchMode export_merge_tree_part_schema_match_mode; + extern const SettingsBool ignore_extra_source_columns; extern const SettingsExportPartitionAllOnError export_merge_tree_partition_all_on_error; extern const SettingsString export_merge_tree_part_filename_pattern; extern const SettingsBool write_full_path_in_iceberg_metadata; @@ -8534,7 +8535,8 @@ void StorageReplicatedMergeTree::exportPartitionToTable(const PartitionCommand & manifest.write_full_path_in_iceberg_metadata = query_context->getSettingsRef()[Setting::write_full_path_in_iceberg_metadata]; manifest.allow_lossy_cast = query_context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; manifest.iceberg_partition_timezone = query_context->getSettingsRef()[Setting::iceberg_partition_timezone].toString(); - manifest.schema_mismatch_mode = query_context->getSettingsRef()[Setting::export_merge_tree_part_schema_mismatch_mode].value; + manifest.schema_match_mode = query_context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value; + manifest.ignore_extra_source_columns = query_context->getSettingsRef()[Setting::ignore_extra_source_columns].value; if (dest_storage->isDataLake()) { diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index e5691a3e1a11..569b05fbadcd 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -38,8 +38,8 @@ EXTRA_SOURCE_COLUMN_MODES = [ - pytest.param("ignore_extra_source_columns_by_position", id="by-position"), - pytest.param("ignore_extra_source_columns_by_name", id="by-name"), + pytest.param("match_by_position", id="by-position"), + pytest.param("match_by_name", id="by-name"), ] @@ -744,8 +744,8 @@ def test_export_part_column_count_mismatch_source_fewer_is_rejected(cluster): node.query(f"DROP TABLE IF EXISTS {iceberg}") -@pytest.mark.parametrize("mismatch_mode", EXTRA_SOURCE_COLUMN_MODES) -def test_export_part_source_more_columns_allowed_with_ignore_extra_setting(cluster, mismatch_mode): +@pytest.mark.parametrize("schema_match_mode", EXTRA_SOURCE_COLUMN_MODES) +def test_export_part_source_more_columns_allowed_with_ignore_extra_setting(cluster, schema_match_mode): node = cluster.instances["node1"] sfx = unique_suffix() mt = f"mt_ignore_extra_{sfx}" @@ -759,7 +759,10 @@ def test_export_part_source_more_columns_allowed_with_ignore_extra_setting(clust export_part( node=node, table=mt, part=part_2020, dest=iceberg, - extra_settings=f"export_merge_tree_part_schema_mismatch_mode = '{mismatch_mode}'", + extra_settings=( + f"export_merge_tree_part_schema_match_mode = '{schema_match_mode}', " + f"ignore_extra_source_columns = 1" + ), ) wait_for_export_part(node=node, table=mt, part=part_2020) @@ -776,14 +779,14 @@ def test_export_part_source_more_columns_allowed_with_ignore_extra_setting(clust @pytest.mark.parametrize( - "mismatch_mode,expected_error", + "schema_match_mode,expected_error", [ - pytest.param("ignore_extra_source_columns_by_position", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), - pytest.param("ignore_extra_source_columns_by_name", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-name"), + pytest.param("match_by_position", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), + pytest.param("match_by_name", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-name"), ], ) def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_ignore_extra_setting( - cluster, mismatch_mode, expected_error + cluster, schema_match_mode, expected_error ): node = cluster.instances["node1"] sfx = unique_suffix() @@ -800,9 +803,10 @@ def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_igno f"ALTER TABLE {mt} EXPORT PART '{part_2020}' TO TABLE {iceberg} " f"SETTINGS allow_experimental_export_merge_tree_part = 1, " f"allow_experimental_insert_into_iceberg = 1, " - f"export_merge_tree_part_schema_mismatch_mode = '{mismatch_mode}'" + f"export_merge_tree_part_schema_match_mode = '{schema_match_mode}', " + f"ignore_extra_source_columns = 1" ) - assert expected_error in error, f"Expected {expected_error} with {mismatch_mode}, got: {error!r}" + assert expected_error in error, f"Expected {expected_error} with {schema_match_mode}, got: {error!r}" node.query(f"DROP TABLE IF EXISTS {mt} SYNC") node.query(f"DROP TABLE IF EXISTS {iceberg}") @@ -854,7 +858,7 @@ def test_export_part_ignore_extra_column_breaks_hybrid_over_source_and_destinati export_part( node=node, table=mt, part=part, dest=iceberg, - extra_settings="export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_position'", + extra_settings="export_merge_tree_part_schema_match_mode = 'match_by_position', ignore_extra_source_columns = 1", ) wait_for_export_part(node=node, table=mt, part=part) @@ -932,8 +936,8 @@ def test_export_part_with_alias_column(cluster): node.query(f"DROP TABLE IF EXISTS {iceberg}") -@pytest.mark.parametrize("mismatch_mode", EXTRA_SOURCE_COLUMN_MODES) -def test_export_part_ignore_extra_setting_drops_trailing_alias_column(cluster, mismatch_mode): +@pytest.mark.parametrize("schema_match_mode", EXTRA_SOURCE_COLUMN_MODES) +def test_export_part_ignore_extra_setting_drops_trailing_alias_column(cluster, schema_match_mode): node = cluster.instances["node1"] sfx = unique_suffix() mt = f"mt_ignore_extra_alias_{sfx}" @@ -947,7 +951,10 @@ def test_export_part_ignore_extra_setting_drops_trailing_alias_column(cluster, m export_part( node=node, table=mt, part=part_2020, dest=iceberg, - extra_settings=f"export_merge_tree_part_schema_mismatch_mode = '{mismatch_mode}'", + extra_settings=( + f"export_merge_tree_part_schema_match_mode = '{schema_match_mode}', " + f"ignore_extra_source_columns = 1" + ), ) wait_for_export_part(node=node, table=mt, part=part_2020) @@ -960,8 +967,8 @@ def test_export_part_ignore_extra_setting_drops_trailing_alias_column(cluster, m node.query(f"DROP TABLE IF EXISTS {iceberg}") -@pytest.mark.parametrize("mismatch_mode", EXTRA_SOURCE_COLUMN_MODES) -def test_export_part_ignore_extra_setting_kept_alias_depends_on_dropped_column(cluster, mismatch_mode): +@pytest.mark.parametrize("schema_match_mode", EXTRA_SOURCE_COLUMN_MODES) +def test_export_part_ignore_extra_setting_kept_alias_depends_on_dropped_column(cluster, schema_match_mode): node = cluster.instances["node1"] sfx = unique_suffix() mt = f"mt_ignore_extra_dep_{sfx}" @@ -979,7 +986,10 @@ def test_export_part_ignore_extra_setting_kept_alias_depends_on_dropped_column(c export_part( node=node, table=mt, part=part_2020, dest=iceberg, - extra_settings=f"export_merge_tree_part_schema_mismatch_mode = '{mismatch_mode}'", + extra_settings=( + f"export_merge_tree_part_schema_match_mode = '{schema_match_mode}', " + f"ignore_extra_source_columns = 1" + ), ) wait_for_export_part(node=node, table=mt, part=part_2020) @@ -1053,7 +1063,8 @@ def test_export_part_reordered_subset_requires_matching_by_name(cluster): f"ALTER TABLE {mt} EXPORT PART '{part_2020}' TO TABLE {iceberg} " f"SETTINGS allow_experimental_export_merge_tree_part = 1, " f"allow_experimental_insert_into_iceberg = 1, " - f"export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_position'" + f"export_merge_tree_part_schema_match_mode = 'match_by_position', " + f"ignore_extra_source_columns = 1" ) assert "INCOMPATIBLE_COLUMNS" in error, f"Expected positional matching to fail, got: {error!r}" assert node.query(f"SELECT count() FROM {iceberg}").strip() == "0" @@ -1063,7 +1074,7 @@ def test_export_part_reordered_subset_requires_matching_by_name(cluster): mt, part_2020, iceberg, - extra_settings="export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_name'", + extra_settings="export_merge_tree_part_schema_match_mode = 'match_by_name', ignore_extra_source_columns = 1", ) wait_for_export_part(node, mt, part_2020) @@ -1098,7 +1109,8 @@ def test_export_part_match_by_name_requires_every_destination_column(cluster): f"ALTER TABLE {mt} EXPORT PART '{part_2020}' TO TABLE {iceberg} " f"SETTINGS allow_experimental_export_merge_tree_part = 1, " f"allow_experimental_insert_into_iceberg = 1, " - f"export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_name'" + f"export_merge_tree_part_schema_match_mode = 'match_by_name', " + f"ignore_extra_source_columns = 1" ) assert "THERE_IS_NO_COLUMN" in error and "renamed_id" in error, ( f"Expected name matching to reject missing column `renamed_id`, got: {error!r}" @@ -1110,8 +1122,10 @@ def test_export_part_match_by_name_requires_every_destination_column(cluster): node.query(f"DROP TABLE IF EXISTS {iceberg}") -def test_export_part_match_by_name_falls_back_to_positional_without_extra_source_columns(cluster): - # No extra source column here, so `ignore_extra_source_columns_by_name` behaves like `strict`. +def test_export_part_match_by_name_rejects_renamed_column_without_extra_source_columns(cluster): + """`match_by_name` has no positional fallback: even when the source and destination have the + exact same number of columns (no extra source column to justify falling back to position), + a destination column absent from the source by name must still be rejected.""" node = cluster.instances["node1"] sfx = unique_suffix() mt = f"mt_match_by_name_fallback_{sfx}" @@ -1123,14 +1137,48 @@ def test_export_part_match_by_name_falls_back_to_positional_without_extra_source node.query(f"INSERT INTO {mt} VALUES (1, 2020), (2, 2020)") part_2020 = get_part(node, mt, "2020") + error = node.query_and_get_error( + f"ALTER TABLE {mt} EXPORT PART '{part_2020}' TO TABLE {iceberg} " + f"SETTINGS allow_experimental_export_merge_tree_part = 1, " + f"allow_experimental_insert_into_iceberg = 1, " + f"export_merge_tree_part_schema_match_mode = 'match_by_name'" + ) + assert "THERE_IS_NO_COLUMN" in error and "renamed_id" in error, ( + f"Expected name matching to reject missing column `renamed_id` even without an extra " + f"source column (no positional fallback), got: {error!r}" + ) + assert node.query(f"SELECT count() FROM {iceberg}").strip() == "0" + + node.query(f"DROP TABLE IF EXISTS {mt} SYNC") + node.query(f"DROP TABLE IF EXISTS {iceberg}") + + +def test_export_part_match_by_name_reorders_columns_without_extra_source_columns(cluster): + """The scenario `export_merge_tree_part_schema_match_mode = 'match_by_name'` was introduced for: + the source and destination have the exact same number of columns, declared in a different order. + Previously this fell back to positional matching (a no-op for by-name mode); now it is matched + by name like any other case.""" + node = cluster.instances["node1"] + sfx = unique_suffix() + mt = f"mt_match_by_name_equal_count_{sfx}" + iceberg = f"iceberg_match_by_name_equal_count_{sfx}" + + make_mt(node, mt, "id Int32, year Int32, payload String", "year") + make_iceberg_s3(node, iceberg, "payload String, id Int32, year Int32", "year") + + node.query(f"INSERT INTO {mt} VALUES (1, 2020, 'foo'), (2, 2020, 'bar')") + part_2020 = get_part(node, mt, "2020") + export_part( node, mt, part_2020, iceberg, - extra_settings="export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_name'", + extra_settings="export_merge_tree_part_schema_match_mode = 'match_by_name'", ) wait_for_export_part(node, mt, part_2020) - result = node.query(f"SELECT renamed_id, year FROM {iceberg} ORDER BY renamed_id").strip() - assert result == "1\t2020\n2\t2020", f"Unexpected data:\n{result}" + result = node.query(f"SELECT id, year, payload FROM {iceberg} ORDER BY id").strip() + assert result == "1\t2020\tfoo\n2\t2020\tbar", f"Unexpected data:\n{result}" + + assert_part_log(node, mt, part_2020) node.query(f"DROP TABLE IF EXISTS {mt} SYNC") node.query(f"DROP TABLE IF EXISTS {iceberg}") diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index d732b6d91f03..e1e95c75bcbe 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -10,8 +10,8 @@ EXTRA_SOURCE_COLUMN_MODES = [ - pytest.param("ignore_extra_source_columns_by_position", id="by-position"), - pytest.param("ignore_extra_source_columns_by_name", id="by-name"), + pytest.param("match_by_position", id="by-position"), + pytest.param("match_by_name", id="by-name"), ] @@ -986,14 +986,14 @@ def test_export_part_column_count_mismatch_source_fewer_is_rejected(cluster): @pytest.mark.parametrize( - "mismatch_mode,expected_error", + "schema_match_mode,expected_error", [ - pytest.param("ignore_extra_source_columns_by_position", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), - pytest.param("ignore_extra_source_columns_by_name", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-name"), + pytest.param("match_by_position", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), + pytest.param("match_by_name", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-name"), ], ) def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_ignore_extra_setting( - cluster, mismatch_mode, expected_error + cluster, schema_match_mode, expected_error ): node = cluster.instances["node1"] @@ -1016,10 +1016,11 @@ def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_igno error = node.query_and_get_error( f"ALTER TABLE {mt_table} EXPORT PART '2020_1_1_0' TO TABLE {s3_table} " - f"SETTINGS export_merge_tree_part_schema_mismatch_mode = '{mismatch_mode}'" + f"SETTINGS export_merge_tree_part_schema_match_mode = '{schema_match_mode}', " + f"ignore_extra_source_columns = 1" ) assert expected_error in error, ( - f"Expected {expected_error} for source Date: Mon, 24 Aug 2026 12:46:32 +0200 Subject: [PATCH 07/13] some updates Signed-off-by: Konstantin Morozov --- .../MergeTree/ExportPartitionUtils.cpp | 140 +++++++++--------- src/Storages/MergeTree/ExportPartitionUtils.h | 1 - .../tests/gtest_export_partition_ordering.cpp | 59 +++----- 3 files changed, 84 insertions(+), 116 deletions(-) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 8a676111d531..01fc9f6e6620 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -975,51 +975,34 @@ namespace const ColumnsWithTypeAndName & source_columns, const ColumnsWithTypeAndName & destination_columns, MergeTreePartExportSchemaMatchMode schema_match_mode, - bool ignore_extra_source_columns, const StorageID & destination_storage_id) { - if (schema_match_mode == MergeTreePartExportSchemaMatchMode::match_by_name) + switch (schema_match_mode) { - /// `ignore_extra_source_columns` only excuses the source having MORE columns than the - /// destination; a source with fewer columns is always a mismatch. - const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); - if (source_columns.size() != destination_columns.size() - && !(ignore_extra_source_columns && src_has_extra_columns)) - throw Exception( - ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, - "Number of columns doesn't match (source: {} and result: {})", - source_columns.size(), - destination_columns.size()); - - std::unordered_map source_columns_by_name; - source_columns_by_name.reserve(source_columns.size()); - for (const auto & source_column : source_columns) - source_columns_by_name.emplace(source_column.name, &source_column); - - for (const auto & destination_column : destination_columns) - { - const auto source_it = source_columns_by_name.find(destination_column.name); - if (source_it == source_columns_by_name.end()) - throw Exception( - ErrorCodes::THERE_IS_NO_COLUMN, - "Cannot find column `{}` in source stream", - destination_column.name); - - verifyExportColumnCastIsSafe(*source_it->second, destination_column, destination_storage_id); + case MergeTreePartExportSchemaMatchMode::match_by_position: { + for (size_t i = 0; i < destination_columns.size(); ++i) + verifyExportColumnCastIsSafe(source_columns[i], destination_columns[i], destination_storage_id); + return; } - return; - } + case MergeTreePartExportSchemaMatchMode::match_by_name: { + std::unordered_map source_columns_by_name; + source_columns_by_name.reserve(source_columns.size()); + for (const auto & source_column : source_columns) + source_columns_by_name.emplace(source_column.name, &source_column); - if (source_columns.size() < destination_columns.size() - || (!ignore_extra_source_columns && source_columns.size() != destination_columns.size())) - throw Exception( - ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, - "Number of columns doesn't match (source: {} and result: {})", - source_columns.size(), - destination_columns.size()); + for (const auto & destination_column : destination_columns) + { + const auto source_it = source_columns_by_name.find(destination_column.name); + if (source_it == source_columns_by_name.end()) + throw Exception( + ErrorCodes::THERE_IS_NO_COLUMN, "Cannot find column `{}` in source stream", destination_column.name); - for (size_t i = 0; i < destination_columns.size(); ++i) - verifyExportColumnCastIsSafe(source_columns[i], destination_columns[i], destination_storage_id); + verifyExportColumnCastIsSafe(*source_it->second, destination_column, destination_storage_id); + } + return; + } + } + UNREACHABLE(); } void verifyExportSchemaCastable( @@ -1046,6 +1029,22 @@ namespace const bool match_by_name = schema_match_mode == MergeTreePartExportSchemaMatchMode::match_by_name; const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); + if (source_columns.size() < destination_columns.size()) + throw Exception( + ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, + "Number of columns doesn't match (source: {} and result: {}): " + "destination cannot have more columns than source", + source_columns.size(), + destination_columns.size()); + + if (src_has_extra_columns && !ignore_extra_source_columns) + throw Exception( + ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, + "Number of columns doesn't match (source: {} and result: {}): " + "source has extra columns and the `ignore_extra_source_columns` setting is disabled", + source_columns.size(), + destination_columns.size()); + if (!match_by_name && ignore_extra_source_columns && src_has_extra_columns) { LOG_DEBUG(getLogger("ExportPartitionUtils"), @@ -1057,17 +1056,6 @@ namespace source_columns.resize(destination_columns.size()); } - /// `makeConvertingActions` in `Name` mode silently ignores an unreferenced source column, so it must be rejected here explicitly. - /// `ignore_extra_source_columns` only excuses the source having MORE columns than the destination; - /// a source with fewer columns is always a mismatch. - if (match_by_name && source_columns.size() != destination_columns.size() - && !(ignore_extra_source_columns && src_has_extra_columns)) - throw Exception( - ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, - "Number of columns doesn't match (source: {} and result: {})", - source_columns.size(), - destination_columns.size()); - (void) ActionsDAG::makeConvertingActions( source_columns, destination_columns, @@ -1090,34 +1078,39 @@ namespace const bool allow_lossy_cast = context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; - if (match_by_name) + switch (schema_match_mode) { - std::unordered_map source_positions_by_name; - source_positions_by_name.reserve(source_columns.size()); - for (size_t i = 0; i < source_columns.size(); ++i) - source_positions_by_name.emplace(source_columns[i].name, i); - - for (const auto & destination_column : destination_columns) + case MergeTreePartExportSchemaMatchMode::match_by_name: { - if (!partition_key_owner_columns.contains(destination_column.name)) - continue; + std::unordered_map source_positions_by_name; + source_positions_by_name.reserve(source_columns.size()); + for (size_t i = 0; i < source_columns.size(); ++i) + source_positions_by_name.emplace(source_columns[i].name, i); + + for (const auto & destination_column : destination_columns) + { + if (!partition_key_owner_columns.contains(destination_column.name)) + continue; - const auto source_it = source_positions_by_name.find(destination_column.name); - if (source_it == source_positions_by_name.end()) - continue; + const auto source_it = source_positions_by_name.find(destination_column.name); + if (source_it == source_positions_by_name.end()) + continue; - verifyPartitionKeyColumn( - source_columns[source_it->second], destination_column, source_it->second, destination_storage_id, - /*match_by_name=*/ true); + verifyPartitionKeyColumn( + source_columns[source_it->second], destination_column, source_it->second, destination_storage_id, + /*match_by_name=*/ true); + } + break; + } + case MergeTreePartExportSchemaMatchMode::match_by_position: + { + const size_t num_columns = std::min(source_columns.size(), destination_columns.size()); + for (size_t i = 0; i < num_columns; ++i) + if (partition_key_owner_columns.contains(source_columns[i].name)) + verifyPartitionKeyColumn(source_columns[i], destination_columns[i], i, destination_storage_id, + /*match_by_name=*/ false); + break; } - } - else - { - const size_t num_columns = std::min(source_columns.size(), destination_columns.size()); - for (size_t i = 0; i < num_columns; ++i) - if (partition_key_owner_columns.contains(source_columns[i].name)) - verifyPartitionKeyColumn(source_columns[i], destination_columns[i], i, destination_storage_id, - /*match_by_name=*/ false); } /// Lossy casts may silently change values, so reject them unless the user opts in. @@ -1128,7 +1121,6 @@ namespace source_columns, destination_columns, schema_match_mode, - ignore_extra_source_columns, destination_storage_id); } } diff --git a/src/Storages/MergeTree/ExportPartitionUtils.h b/src/Storages/MergeTree/ExportPartitionUtils.h index 5757e8f658c8..455aace3846c 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.h +++ b/src/Storages/MergeTree/ExportPartitionUtils.h @@ -93,7 +93,6 @@ namespace ExportPartitionUtils const ColumnsWithTypeAndName & source_columns, const ColumnsWithTypeAndName & destination_columns, MergeTreePartExportSchemaMatchMode schema_match_mode, - bool ignore_extra_source_columns, const StorageID & destination_storage_id); /// Validates that source columns can be exported into the destination with the configured diff --git a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp index 71206642da1a..0a3be2fadad5 100644 --- a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp +++ b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp @@ -274,7 +274,6 @@ TEST(ExportColumnCastsTest, UsesSelectedMatchingMode) source_columns, destination_columns, MergeTreePartExportSchemaMatchMode::match_by_position, - /*ignore_extra_source_columns=*/ true, destination_storage_id); }, ErrorCodes::INCOMPATIBLE_COLUMNS); @@ -283,7 +282,6 @@ TEST(ExportColumnCastsTest, UsesSelectedMatchingMode) source_columns, destination_columns, MergeTreePartExportSchemaMatchMode::match_by_name, - /*ignore_extra_source_columns=*/ true, destination_storage_id)); } @@ -301,27 +299,22 @@ TEST(ExportColumnCastsTest, MatchByNameAcceptsReorderedColumnsWithEqualColumnCou }; const StorageID destination_storage_id{"test", "destination"}; - for (const bool ignore_extra_source_columns : {false, true}) - { - expectExceptionCode( - [&] - { - ExportPartitionUtils::verifyExportColumnCastsAreSafe( - source_columns, - reordered_destination_columns, - MergeTreePartExportSchemaMatchMode::match_by_position, - ignore_extra_source_columns, - destination_storage_id); - }, - ErrorCodes::INCOMPATIBLE_COLUMNS); + expectExceptionCode( + [&] + { + ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, + reordered_destination_columns, + MergeTreePartExportSchemaMatchMode::match_by_position, + destination_storage_id); + }, + ErrorCodes::INCOMPATIBLE_COLUMNS); - EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( - source_columns, - reordered_destination_columns, - MergeTreePartExportSchemaMatchMode::match_by_name, - ignore_extra_source_columns, - destination_storage_id)); - } + EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, + reordered_destination_columns, + MergeTreePartExportSchemaMatchMode::match_by_name, + destination_storage_id)); const ColumnsWithTypeAndName same_order_destination_columns = { makeColumn("id"), @@ -330,12 +323,11 @@ TEST(ExportColumnCastsTest, MatchByNameAcceptsReorderedColumnsWithEqualColumnCou }; for (const auto mode : {MergeTreePartExportSchemaMatchMode::match_by_position, MergeTreePartExportSchemaMatchMode::match_by_name}) - for (const bool ignore_extra_source_columns : {false, true}) - EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( - source_columns, same_order_destination_columns, mode, ignore_extra_source_columns, destination_storage_id)); + EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, same_order_destination_columns, mode, destination_storage_id)); } -TEST(ExportColumnCastsTest, MatchByNameRejectsUnmatchedSourceColumnUnlessIgnored) +TEST(ExportColumnCastsTest, MatchByNameToleratesUnmatchedSourceColumn) { const ColumnsWithTypeAndName source_columns = { makeColumn("id"), @@ -348,23 +340,10 @@ TEST(ExportColumnCastsTest, MatchByNameRejectsUnmatchedSourceColumnUnlessIgnored }; const StorageID destination_storage_id{"test", "destination"}; - expectExceptionCode( - [&] - { - ExportPartitionUtils::verifyExportColumnCastsAreSafe( - source_columns, - destination_columns, - MergeTreePartExportSchemaMatchMode::match_by_name, - /*ignore_extra_source_columns=*/ false, - destination_storage_id); - }, - ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH); - EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( source_columns, destination_columns, MergeTreePartExportSchemaMatchMode::match_by_name, - /*ignore_extra_source_columns=*/ true, destination_storage_id)); } @@ -387,7 +366,6 @@ TEST(ExportColumnCastsTest, RejectsLossyCastAfterMatchingByName) source_columns, destination_columns, MergeTreePartExportSchemaMatchMode::match_by_name, - /*ignore_extra_source_columns=*/ true, StorageID{"test", "destination"}); }, ErrorCodes::INCOMPATIBLE_COLUMNS); @@ -412,7 +390,6 @@ TEST(ExportColumnCastsTest, RejectsMissingDestinationColumnAfterMatchingByName) source_columns, destination_columns, MergeTreePartExportSchemaMatchMode::match_by_name, - /*ignore_extra_source_columns=*/ true, StorageID{"test", "destination"}); }, ErrorCodes::THERE_IS_NO_COLUMN); From 3c4b03f71c093f9a5c82ed62630c9503cbaeaab5 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Mon, 24 Aug 2026 13:11:36 +0200 Subject: [PATCH 08/13] rename setting Signed-off-by: Konstantin Morozov --- docs/en/antalya/part_export.md | 12 +++++------ docs/en/antalya/partition_export.md | 12 +++++------ src/Core/Settings.cpp | 4 ++-- src/Core/SettingsChangesHistory.cpp | 2 +- src/Storages/MergeTree/ExportPartTask.cpp | 4 ++-- .../MergeTree/ExportPartitionUtils.cpp | 8 ++++---- src/Storages/MergeTree/ExportPartitionUtils.h | 2 +- .../tests/gtest_export_partition_ordering.cpp | 6 +++--- src/Storages/StorageReplicatedMergeTree.cpp | 4 ++-- .../test.py | 16 +++++++-------- .../test.py | 8 ++++---- .../test.py | 20 +++++++++---------- .../test.py | 4 ++-- 13 files changed, 51 insertions(+), 51 deletions(-) diff --git a/docs/en/antalya/part_export.md b/docs/en/antalya/part_export.md index 413e5101956c..1b095342ea3d 100644 --- a/docs/en/antalya/part_export.md +++ b/docs/en/antalya/part_export.md @@ -55,7 +55,7 @@ Source and destination tables must support positional schema conversion. The fol The following requirements apply to the source and destination: -1. **Column count** - by default (`ignore_extra_source_columns = false`) every source column must have a corresponding destination column: with `export_merge_tree_part_schema_match_mode = 'match_by_position'` (the default) the source and destination must have the same number of columns; with `'match_by_name'` they must have the same set of column names. A mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. Set `ignore_extra_source_columns = 1` to allow a source table with columns that have no corresponding destination column; such columns are dropped and not exported. The destination having a column absent from the source is always rejected, regardless of this setting. +1. **Column count** - by default (`export_merge_tree_part_ignore_extra_source_columns = false`) every source column must have a corresponding destination column: with `export_merge_tree_part_schema_match_mode = 'match_by_position'` (the default) the source and destination must have the same number of columns; with `'match_by_name'` they must have the same set of column names. A mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. Set `export_merge_tree_part_ignore_extra_source_columns = 1` to allow a source table with columns that have no corresponding destination column; such columns are dropped and not exported. The destination having a column absent from the source is always rejected, regardless of this setting. 2. **`PARTITION BY` expressions** - the whole part must land in a single destination partition. Identical expressions always satisfy this; otherwise the destination expression has to be computable from the values the source partition key pins, or be proven single-valued over the part's min/max range. The same requirement applies to the partition fields and transforms of an Apache Iceberg destination. See [Source partition key compatibility](/docs/en/antalya/partition_export.md#source-partition-key-compatibility). 3. **The position of every column backing the partition key** - it is not enough for the `PARTITION BY` expressions to be textually identical: every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. If such a column contains a named `Tuple`, its element names must also be declared in the same order (an unnamed `Tuple` on either side is exempt from this, per the allowance above). This comparison is recursive through nested tuples and through container types such as `Array` and `Map`. @@ -144,9 +144,9 @@ In case a table function is used as the destination, the schema can be omitted a - `match_by_position` (default) - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Column names are not otherwise considered. - `match_by_name` - every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be declared in a different order than the source. A destination column absent from the source, including when it was renamed, throws `THERE_IS_NO_COLUMN`; there is no positional fallback. - See `ignore_extra_source_columns` below for how a source column without a corresponding destination column is handled in each mode. + See `export_merge_tree_part_ignore_extra_source_columns` below for how a source column without a corresponding destination column is handled in each mode. -### `ignore_extra_source_columns` (Optional) +### `export_merge_tree_part_ignore_extra_source_columns` (Optional) - **Type**: `Bool` - **Default**: `false` @@ -158,9 +158,9 @@ In case a table function is used as the destination, the schema can be omitted a Error behavior: - - With `ignore_extra_source_columns = false` (default), a source column without a corresponding destination column throws `NUMBER_OF_COLUMNS_DOESNT_MATCH` - in `match_by_position` mode this means any column-count mismatch, in `match_by_name` mode this means the source and destination column-name sets differ. - - In `match_by_position` mode, the destination having more columns than the source always throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`, regardless of `ignore_extra_source_columns`. - - In `match_by_name` mode, a destination column absent from the source (including a renamed one) always throws `THERE_IS_NO_COLUMN`, regardless of `ignore_extra_source_columns`; there is no positional fallback. + - With `export_merge_tree_part_ignore_extra_source_columns = false` (default), a source column without a corresponding destination column throws `NUMBER_OF_COLUMNS_DOESNT_MATCH` - in `match_by_position` mode this means any column-count mismatch, in `match_by_name` mode this means the source and destination column-name sets differ. + - In `match_by_position` mode, the destination having more columns than the source always throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`, regardless of `export_merge_tree_part_ignore_extra_source_columns`. + - In `match_by_name` mode, a destination column absent from the source (including a renamed one) always throws `THERE_IS_NO_COLUMN`, regardless of `export_merge_tree_part_ignore_extra_source_columns`; there is no positional fallback. - After columns have been matched successfully, a cast rejected by the export type-safety check throws `INCOMPATIBLE_COLUMNS`. diff --git a/docs/en/antalya/partition_export.md b/docs/en/antalya/partition_export.md index 9e03e62fe2a0..3a9d0cedb674 100644 --- a/docs/en/antalya/partition_export.md +++ b/docs/en/antalya/partition_export.md @@ -54,7 +54,7 @@ TO TABLE [destination_database.]destination_table `EXPORT PARTITION` exports each part via the same mechanism as [`EXPORT PART`](/docs/en/antalya/part_export.md#requirements), so the source and destination tables must satisfy the same compatibility requirements. Columns are matched by position by default, or by their exact, case-sensitive name if `export_merge_tree_part_schema_match_mode = 'match_by_name'` is set, and column types may differ as long as they are safely castable (or `export_merge_tree_part_allow_lossy_cast = 1` is set). Beyond that, the following requirements apply: -1. **Column count** - by default (`ignore_extra_source_columns = false`) every source column must have a corresponding destination column: with `export_merge_tree_part_schema_match_mode = 'match_by_position'` (the default) the source and destination must have the same number of columns; with `'match_by_name'` they must have the same set of column names. Set `ignore_extra_source_columns = 1` to allow a source table with columns that have no corresponding destination column; the destination having a column absent from the source is still always rejected. +1. **Column count** - by default (`export_merge_tree_part_ignore_extra_source_columns = false`) every source column must have a corresponding destination column: with `export_merge_tree_part_schema_match_mode = 'match_by_position'` (the default) the source and destination must have the same number of columns; with `'match_by_name'` they must have the same set of column names. Set `export_merge_tree_part_ignore_extra_source_columns = 1` to allow a source table with columns that have no corresponding destination column; the destination having a column absent from the source is still always rejected. 2. **`PARTITION BY` expressions** - the whole source partition must land in a single destination partition. Identical expressions always satisfy this; otherwise the destination expression has to be computable from the values the source partition key pins, or be proven single-valued over the partition's min/max range. The same requirement applies to the partition fields and transforms of an Apache Iceberg destination. See [Source partition key compatibility](#source-partition-key-compatibility). 3. **Partition key column positions and layouts** - every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. Named `Tuple` elements within such a column must also be declared in the same order, including tuples nested inside `Array` or `Map`. This applies even if both tables' `PARTITION BY` expressions are textually identical. See [`EXPORT PART` requirements](/docs/en/antalya/part_export.md#requirements) for a worked example and the corresponding exception message. @@ -143,9 +143,9 @@ Notes: - `match_by_position` (default) - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Column names are not otherwise considered. - `match_by_name` - every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be declared in a different order than the source. A destination column absent from the source, including when it was renamed, throws `THERE_IS_NO_COLUMN`; there is no positional fallback. - See `ignore_extra_source_columns` below for how a source column without a corresponding destination column is handled in each mode. + See `export_merge_tree_part_ignore_extra_source_columns` below for how a source column without a corresponding destination column is handled in each mode. -### `ignore_extra_source_columns` (Optional) +### `export_merge_tree_part_ignore_extra_source_columns` (Optional) - **Type**: `Bool` - **Default**: `false` @@ -157,9 +157,9 @@ Notes: Error behavior: - - With `ignore_extra_source_columns = false` (default), a source column without a corresponding destination column throws `NUMBER_OF_COLUMNS_DOESNT_MATCH` - in `match_by_position` mode this means any column-count mismatch, in `match_by_name` mode this means the source and destination column-name sets differ. - - In `match_by_position` mode, the destination having more columns than the source always throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`, regardless of `ignore_extra_source_columns`. - - In `match_by_name` mode, a destination column absent from the source (including a renamed one) always throws `THERE_IS_NO_COLUMN`, regardless of `ignore_extra_source_columns`; there is no positional fallback. + - With `export_merge_tree_part_ignore_extra_source_columns = false` (default), a source column without a corresponding destination column throws `NUMBER_OF_COLUMNS_DOESNT_MATCH` - in `match_by_position` mode this means any column-count mismatch, in `match_by_name` mode this means the source and destination column-name sets differ. + - In `match_by_position` mode, the destination having more columns than the source always throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`, regardless of `export_merge_tree_part_ignore_extra_source_columns`. + - In `match_by_name` mode, a destination column absent from the source (including a renamed one) always throws `THERE_IS_NO_COLUMN`, regardless of `export_merge_tree_part_ignore_extra_source_columns`; there is no positional fallback. - After columns have been matched successfully, a cast rejected by the export type-safety check throws `INCOMPATIBLE_COLUMNS`. ## Examples diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 45eb39385754..ec9efc301dad 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -7633,9 +7633,9 @@ Possible values: - `match_by_position` (default) - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Column names are not otherwise considered. - `match_by_name` - every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be declared in a different order than the source. A destination column absent from the source throws `THERE_IS_NO_COLUMN`, with no positional fallback. -See also `ignore_extra_source_columns`, which controls whether a source column without a corresponding destination column is dropped or rejected. +See also `export_merge_tree_part_ignore_extra_source_columns`, which controls whether a source column without a corresponding destination column is dropped or rejected. )", 0) \ - DECLARE(Bool, ignore_extra_source_columns, false, R"( + DECLARE(Bool, export_merge_tree_part_ignore_extra_source_columns, false, R"( Controls whether `EXPORT PART`/`EXPORT PARTITION` tolerates source `MergeTree` columns that have no corresponding destination column. - `false` (default) - such a source column is rejected: the source and destination must match exactly (in `export_merge_tree_part_schema_match_mode = 'match_by_position'`, this means the same number of columns; in `'match_by_name'`, the same set of column names). A mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - `true` - a source column without a corresponding destination column is dropped and not exported, instead of throwing. In `match_by_position` mode, this allows a source with extra trailing columns (the destination having more columns than the source is still always rejected). In `match_by_name` mode, this allows source columns whose name has no destination counterpart. diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 580d09fc8703..26df2661a4a7 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -45,7 +45,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"export_merge_tree_partition_task_timeout_seconds", "3600", "86400", "Increase default value to make it more realistic"}, {"export_merge_tree_part_allow_lossy_cast", false, false, "New setting to gate lossy casts in EXPORT PART/PARTITION behind explicit acknowledgment"}, {"export_merge_tree_part_schema_match_mode", "match_by_position", "match_by_position", "New setting to control how EXPORT PART/EXPORT PARTITION matches source columns to destination columns"}, - {"ignore_extra_source_columns", false, false, "New setting to allow EXPORT PART/EXPORT PARTITION when the source table has columns absent from the destination"}, + {"export_merge_tree_part_ignore_extra_source_columns", false, false, "New setting to allow EXPORT PART/EXPORT PARTITION when the source table has columns absent from the destination"}, {"export_merge_tree_partition_retry_initial_backoff_seconds", 5, 5, "New setting for exponential back-off between failed part export retries in an export partition task"}, {"export_merge_tree_partition_retry_max_backoff_seconds", 300, 300, "New setting capping the exponential back-off between failed part export retries in an export partition task"}, {"export_merge_tree_partition_max_retries", 3, 3, "Obsolete and ignored: export partition tasks now retry retryable failures until the task timeout and fail immediately on non-retryable errors, instead of using a fixed retry budget"}, diff --git a/src/Storages/MergeTree/ExportPartTask.cpp b/src/Storages/MergeTree/ExportPartTask.cpp index 2126601ce7c3..7cb2f051f2a5 100644 --- a/src/Storages/MergeTree/ExportPartTask.cpp +++ b/src/Storages/MergeTree/ExportPartTask.cpp @@ -66,7 +66,7 @@ namespace Setting extern const SettingsBool allow_experimental_analyzer; extern const SettingsString export_merge_tree_part_filename_pattern; extern const SettingsMergeTreePartExportSchemaMatchMode export_merge_tree_part_schema_match_mode; - extern const SettingsBool ignore_extra_source_columns; + extern const SettingsBool export_merge_tree_part_ignore_extra_source_columns; } namespace @@ -126,7 +126,7 @@ namespace const auto schema_match_mode = local_context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value; const bool ignore_extra_source_columns = - local_context->getSettingsRef()[Setting::ignore_extra_source_columns]; + local_context->getSettingsRef()[Setting::export_merge_tree_part_ignore_extra_source_columns]; const bool match_by_name = schema_match_mode == MergeTreePartExportSchemaMatchMode::match_by_name; auto source_columns = plan_for_part.getCurrentHeader()->getColumnsWithTypeAndName(); diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 01fc9f6e6620..241a8366b9cc 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -98,7 +98,7 @@ namespace Setting extern const SettingsTimezone iceberg_partition_timezone; #endif extern const SettingsMergeTreePartExportSchemaMatchMode export_merge_tree_part_schema_match_mode; - extern const SettingsBool ignore_extra_source_columns; + extern const SettingsBool export_merge_tree_part_ignore_extra_source_columns; } namespace FailPoints @@ -226,7 +226,7 @@ namespace ExportPartitionUtils "export_merge_tree_part_schema_match_mode", String(magic_enum::enum_name(manifest.schema_match_mode.value_or(MergeTreePartExportSchemaMatchMode::match_by_position)))); context_copy->setSetting( - "ignore_extra_source_columns", + "export_merge_tree_part_ignore_extra_source_columns", manifest.ignore_extra_source_columns.value_or(false)); context_copy->setSetting("max_threads", manifest.max_threads); @@ -1025,7 +1025,7 @@ namespace const auto schema_match_mode = context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value; const bool ignore_extra_source_columns = - context->getSettingsRef()[Setting::ignore_extra_source_columns]; + context->getSettingsRef()[Setting::export_merge_tree_part_ignore_extra_source_columns]; const bool match_by_name = schema_match_mode == MergeTreePartExportSchemaMatchMode::match_by_name; const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); @@ -1041,7 +1041,7 @@ namespace throw Exception( ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, "Number of columns doesn't match (source: {} and result: {}): " - "source has extra columns and the `ignore_extra_source_columns` setting is disabled", + "source has extra columns and the `export_merge_tree_part_ignore_extra_source_columns` setting is disabled", source_columns.size(), destination_columns.size()); diff --git a/src/Storages/MergeTree/ExportPartitionUtils.h b/src/Storages/MergeTree/ExportPartitionUtils.h index 455aace3846c..22187290c652 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.h +++ b/src/Storages/MergeTree/ExportPartitionUtils.h @@ -97,7 +97,7 @@ namespace ExportPartitionUtils /// Validates that source columns can be exported into the destination with the configured /// positional or name-based CAST matching (`export_merge_tree_part_schema_match_mode`) and - /// unmatched-column policy (`ignore_extra_source_columns`). Lossy casts are rejected unless + /// unmatched-column policy (`export_merge_tree_part_ignore_extra_source_columns`). Lossy casts are rejected unless /// `export_merge_tree_part_allow_lossy_cast` is set. Throws BAD_ARGUMENTS on any violation. void verifyExportSchemaCastable( const StorageMetadataPtr & source_metadata, diff --git a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp index 0a3be2fadad5..b01c564e63d9 100644 --- a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp +++ b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp @@ -24,7 +24,7 @@ namespace ErrorCodes namespace Setting { extern const SettingsMergeTreePartExportSchemaMatchMode export_merge_tree_part_schema_match_mode; - extern const SettingsBool ignore_extra_source_columns; + extern const SettingsBool export_merge_tree_part_ignore_extra_source_columns; } namespace @@ -218,7 +218,7 @@ TEST_F(ExportPartitionManifestBackCompatTest, MissingSchemaMatchSettingsFallBack worker_context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value, MergeTreePartExportSchemaMatchMode::match_by_position); EXPECT_EQ( - worker_context->getSettingsRef()[Setting::ignore_extra_source_columns].value, + worker_context->getSettingsRef()[Setting::export_merge_tree_part_ignore_extra_source_columns].value, false); } @@ -247,7 +247,7 @@ TEST_F(ExportPartitionManifestBackCompatTest, IgnoreExtraSourceColumnsAppliedToW auto worker_context = ExportPartitionUtils::getContextCopyWithTaskSettings(getContext().context, manifest); EXPECT_EQ( - worker_context->getSettingsRef()[Setting::ignore_extra_source_columns].value, + worker_context->getSettingsRef()[Setting::export_merge_tree_part_ignore_extra_source_columns].value, value) << "value=" << value; } } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 6d09b6950bd5..708a543d3340 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -230,7 +230,7 @@ namespace Setting extern const SettingsBool export_merge_tree_part_throw_on_pending_patch_parts; extern const SettingsBool export_merge_tree_part_allow_lossy_cast; extern const SettingsMergeTreePartExportSchemaMatchMode export_merge_tree_part_schema_match_mode; - extern const SettingsBool ignore_extra_source_columns; + extern const SettingsBool export_merge_tree_part_ignore_extra_source_columns; extern const SettingsExportPartitionAllOnError export_merge_tree_partition_all_on_error; extern const SettingsString export_merge_tree_part_filename_pattern; extern const SettingsBool write_full_path_in_iceberg_metadata; @@ -8536,7 +8536,7 @@ void StorageReplicatedMergeTree::exportPartitionToTable(const PartitionCommand & manifest.allow_lossy_cast = query_context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; manifest.iceberg_partition_timezone = query_context->getSettingsRef()[Setting::iceberg_partition_timezone].toString(); manifest.schema_match_mode = query_context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value; - manifest.ignore_extra_source_columns = query_context->getSettingsRef()[Setting::ignore_extra_source_columns].value; + manifest.ignore_extra_source_columns = query_context->getSettingsRef()[Setting::export_merge_tree_part_ignore_extra_source_columns].value; if (dest_storage->isDataLake()) { diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index 569b05fbadcd..8218acd85d9a 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -761,7 +761,7 @@ def test_export_part_source_more_columns_allowed_with_ignore_extra_setting(clust node=node, table=mt, part=part_2020, dest=iceberg, extra_settings=( f"export_merge_tree_part_schema_match_mode = '{schema_match_mode}', " - f"ignore_extra_source_columns = 1" + f"export_merge_tree_part_ignore_extra_source_columns = 1" ), ) wait_for_export_part(node=node, table=mt, part=part_2020) @@ -804,7 +804,7 @@ def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_igno f"SETTINGS allow_experimental_export_merge_tree_part = 1, " f"allow_experimental_insert_into_iceberg = 1, " f"export_merge_tree_part_schema_match_mode = '{schema_match_mode}', " - f"ignore_extra_source_columns = 1" + f"export_merge_tree_part_ignore_extra_source_columns = 1" ) assert expected_error in error, f"Expected {expected_error} with {schema_match_mode}, got: {error!r}" @@ -858,7 +858,7 @@ def test_export_part_ignore_extra_column_breaks_hybrid_over_source_and_destinati export_part( node=node, table=mt, part=part, dest=iceberg, - extra_settings="export_merge_tree_part_schema_match_mode = 'match_by_position', ignore_extra_source_columns = 1", + extra_settings="export_merge_tree_part_schema_match_mode = 'match_by_position', export_merge_tree_part_ignore_extra_source_columns = 1", ) wait_for_export_part(node=node, table=mt, part=part) @@ -953,7 +953,7 @@ def test_export_part_ignore_extra_setting_drops_trailing_alias_column(cluster, s node=node, table=mt, part=part_2020, dest=iceberg, extra_settings=( f"export_merge_tree_part_schema_match_mode = '{schema_match_mode}', " - f"ignore_extra_source_columns = 1" + f"export_merge_tree_part_ignore_extra_source_columns = 1" ), ) wait_for_export_part(node=node, table=mt, part=part_2020) @@ -988,7 +988,7 @@ def test_export_part_ignore_extra_setting_kept_alias_depends_on_dropped_column(c node=node, table=mt, part=part_2020, dest=iceberg, extra_settings=( f"export_merge_tree_part_schema_match_mode = '{schema_match_mode}', " - f"ignore_extra_source_columns = 1" + f"export_merge_tree_part_ignore_extra_source_columns = 1" ), ) wait_for_export_part(node=node, table=mt, part=part_2020) @@ -1064,7 +1064,7 @@ def test_export_part_reordered_subset_requires_matching_by_name(cluster): f"SETTINGS allow_experimental_export_merge_tree_part = 1, " f"allow_experimental_insert_into_iceberg = 1, " f"export_merge_tree_part_schema_match_mode = 'match_by_position', " - f"ignore_extra_source_columns = 1" + f"export_merge_tree_part_ignore_extra_source_columns = 1" ) assert "INCOMPATIBLE_COLUMNS" in error, f"Expected positional matching to fail, got: {error!r}" assert node.query(f"SELECT count() FROM {iceberg}").strip() == "0" @@ -1074,7 +1074,7 @@ def test_export_part_reordered_subset_requires_matching_by_name(cluster): mt, part_2020, iceberg, - extra_settings="export_merge_tree_part_schema_match_mode = 'match_by_name', ignore_extra_source_columns = 1", + extra_settings="export_merge_tree_part_schema_match_mode = 'match_by_name', export_merge_tree_part_ignore_extra_source_columns = 1", ) wait_for_export_part(node, mt, part_2020) @@ -1110,7 +1110,7 @@ def test_export_part_match_by_name_requires_every_destination_column(cluster): f"SETTINGS allow_experimental_export_merge_tree_part = 1, " f"allow_experimental_insert_into_iceberg = 1, " f"export_merge_tree_part_schema_match_mode = 'match_by_name', " - f"ignore_extra_source_columns = 1" + f"export_merge_tree_part_ignore_extra_source_columns = 1" ) assert "THERE_IS_NO_COLUMN" in error and "renamed_id" in error, ( f"Expected name matching to reject missing column `renamed_id`, got: {error!r}" diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index e1e95c75bcbe..2c1952e66109 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -1017,7 +1017,7 @@ def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_igno error = node.query_and_get_error( f"ALTER TABLE {mt_table} EXPORT PART '2020_1_1_0' TO TABLE {s3_table} " f"SETTINGS export_merge_tree_part_schema_match_mode = '{schema_match_mode}', " - f"ignore_extra_source_columns = 1" + f"export_merge_tree_part_ignore_extra_source_columns = 1" ) assert expected_error in error, ( f"Expected {expected_error} for source Date: Mon, 24 Aug 2026 14:00:29 +0200 Subject: [PATCH 09/13] add check before building DAG Signed-off-by: Konstantin Morozov --- src/Common/FailPoint.cpp | 1 + src/Storages/MergeTree/ExportPartTask.cpp | 9 +++ .../MergeTree/ExportPartitionUtils.cpp | 41 +++++++---- src/Storages/MergeTree/ExportPartitionUtils.h | 5 ++ .../test.py | 73 +++++++++++++++++++ 5 files changed, 114 insertions(+), 15 deletions(-) diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index b8fee32bdf13..d7ccfac38631 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -144,6 +144,7 @@ static struct InitFiu REGULAR(export_partition_processed_paths_sync_fail) \ REGULAR(export_part_non_retryable_throw) \ REGULAR(export_part_retryable_throw) \ + PAUSEABLE_ONCE(export_part_pause_before_schema_validation) \ ONCE(backup_add_empty_memory_table) \ PAUSEABLE_ONCE(backup_pause_on_start) \ PAUSEABLE_ONCE(restore_pause_on_start) \ diff --git a/src/Storages/MergeTree/ExportPartTask.cpp b/src/Storages/MergeTree/ExportPartTask.cpp index 7cb2f051f2a5..3df30df41bbd 100644 --- a/src/Storages/MergeTree/ExportPartTask.cpp +++ b/src/Storages/MergeTree/ExportPartTask.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -50,6 +51,7 @@ namespace ErrorCodes namespace FailPoints { + extern const char export_part_pause_before_schema_validation[]; /// Throw a non-retryable (denylisted) error from the part-export worker, so the whole /// export task transitions to FAILED immediately regardless of any timeout. extern const char export_part_non_retryable_throw[]; @@ -119,6 +121,8 @@ namespace const IStorage & destination_storage, const ContextPtr & local_context) { + FailPointInjection::pauseFailPoint(FailPoints::export_part_pause_before_schema_validation); + const auto destination_header = destination_storage.getInMemoryMetadataPtr()->getSampleBlockNonMaterialized(); const auto & destination_columns = destination_header.getColumnsWithTypeAndName(); @@ -132,6 +136,11 @@ namespace auto source_columns = plan_for_part.getCurrentHeader()->getColumnsWithTypeAndName(); const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); + ExportPartitionUtils::checkExportSchemaColumnsCount( + source_columns.size(), + destination_columns.size(), + ignore_extra_source_columns); + if (!match_by_name && ignore_extra_source_columns && src_has_extra_columns) { LOG_DEBUG(getLogger("ExportPartTask"), diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 241a8366b9cc..e630219dd27a 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -1005,6 +1005,28 @@ namespace UNREACHABLE(); } + void checkExportSchemaColumnsCount( + size_t source_columns_count, + size_t destination_columns_count, + bool ignore_extra_source_columns) + { + if (source_columns_count < destination_columns_count) + throw Exception( + ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, + "Number of columns doesn't match (source: {} and result: {}): " + "destination cannot have more columns than source", + source_columns_count, + destination_columns_count); + + if (source_columns_count > destination_columns_count && !ignore_extra_source_columns) + throw Exception( + ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, + "Number of columns doesn't match (source: {} and result: {}): " + "source has extra columns and the `export_merge_tree_part_ignore_extra_source_columns` setting is disabled", + source_columns_count, + destination_columns_count); + } + void verifyExportSchemaCastable( const StorageMetadataPtr & source_metadata, const StorageMetadataPtr & destination_metadata, @@ -1029,21 +1051,10 @@ namespace const bool match_by_name = schema_match_mode == MergeTreePartExportSchemaMatchMode::match_by_name; const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); - if (source_columns.size() < destination_columns.size()) - throw Exception( - ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, - "Number of columns doesn't match (source: {} and result: {}): " - "destination cannot have more columns than source", - source_columns.size(), - destination_columns.size()); - - if (src_has_extra_columns && !ignore_extra_source_columns) - throw Exception( - ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, - "Number of columns doesn't match (source: {} and result: {}): " - "source has extra columns and the `export_merge_tree_part_ignore_extra_source_columns` setting is disabled", - source_columns.size(), - destination_columns.size()); + checkExportSchemaColumnsCount( + source_columns.size(), + destination_columns.size(), + ignore_extra_source_columns); if (!match_by_name && ignore_extra_source_columns && src_has_extra_columns) { diff --git a/src/Storages/MergeTree/ExportPartitionUtils.h b/src/Storages/MergeTree/ExportPartitionUtils.h index 22187290c652..5119139a848d 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.h +++ b/src/Storages/MergeTree/ExportPartitionUtils.h @@ -95,6 +95,11 @@ namespace ExportPartitionUtils MergeTreePartExportSchemaMatchMode schema_match_mode, const StorageID & destination_storage_id); + void checkExportSchemaColumnsCount( + size_t source_columns_count, + size_t destination_columns_count, + bool ignore_extra_source_columns); + /// Validates that source columns can be exported into the destination with the configured /// positional or name-based CAST matching (`export_merge_tree_part_schema_match_mode`) and /// unmatched-column policy (`export_merge_tree_part_ignore_extra_source_columns`). Lossy casts are rejected unless diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index 8218acd85d9a..98c7626848fb 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -1289,6 +1289,79 @@ def test_export_part_runtime_cast_failure_propagates_async(cluster): node.query(f"DROP TABLE IF EXISTS {iceberg}") +def test_export_part_match_by_name_revalidates_extra_source_columns_in_background_task(cluster): + node = cluster.instances["node1"] + sfx = unique_suffix() + mt = f"mt_match_by_name_schema_drift_{sfx}" + iceberg = f"iceberg_match_by_name_schema_drift_{sfx}" + + make_mt(node, mt, "id Int32, year Int32, payload String", "year") + make_iceberg_s3(node, iceberg, "id Int32, year Int32, payload String", "year") + node.query(f"INSERT INTO {mt} VALUES (1, 2020, 'foo'), (2, 2020, 'bar')") + part = get_part(node, mt, "2020") + + try: + node.query("SYSTEM ENABLE FAILPOINT export_part_pause_before_schema_validation") + export_part( + node, + mt, + part, + iceberg, + "export_merge_tree_part_schema_match_mode = 'match_by_name'", + ) + node.query("SYSTEM WAIT FAILPOINT export_part_pause_before_schema_validation PAUSE") + + node.query( + f"ALTER TABLE {iceberg} DROP COLUMN payload", + settings={"allow_insert_into_iceberg": 1}, + ) + node.query("SYSTEM NOTIFY FAILPOINT export_part_pause_before_schema_validation") + + exception = wait_for_failed_export_part(node, mt, part) + assert "NUMBER_OF_COLUMNS_DOESNT_MATCH" in exception, ( + f"Expected the background schema check to reject the extra source column, got: {exception}" + ) + assert node.query(f"SELECT count() FROM {iceberg}").strip() == "0" + finally: + node.query("SYSTEM DISABLE FAILPOINT export_part_pause_before_schema_validation") + node.query(f"DROP TABLE IF EXISTS {mt} SYNC") + node.query(f"DROP TABLE IF EXISTS {iceberg}") + + +def test_export_part_match_by_name_uses_source_snapshot_when_source_column_is_added(cluster): + node = cluster.instances["node1"] + sfx = unique_suffix() + mt = f"mt_match_by_name_source_schema_drift_{sfx}" + iceberg = f"iceberg_match_by_name_source_schema_drift_{sfx}" + + make_mt(node, mt, "id Int32, year Int32, payload String", "year") + make_iceberg_s3(node, iceberg, "id Int32, year Int32, payload String", "year") + node.query(f"INSERT INTO {mt} VALUES (1, 2020, 'foo'), (2, 2020, 'bar')") + part = get_part(node, mt, "2020") + + try: + node.query("SYSTEM ENABLE FAILPOINT export_part_pause_before_schema_validation") + export_part( + node, + mt, + part, + iceberg, + "export_merge_tree_part_schema_match_mode = 'match_by_name'", + ) + node.query("SYSTEM WAIT FAILPOINT export_part_pause_before_schema_validation PAUSE") + + node.query(f"ALTER TABLE {mt} ADD COLUMN extra String DEFAULT 'new'") + node.query("SYSTEM NOTIFY FAILPOINT export_part_pause_before_schema_validation") + + wait_for_export_part(node, mt, part) + result = node.query(f"SELECT id, year, payload FROM {iceberg} ORDER BY id").strip() + assert result == "1\t2020\tfoo\n2\t2020\tbar", f"Unexpected exported data:\n{result}" + finally: + node.query("SYSTEM DISABLE FAILPOINT export_part_pause_before_schema_validation") + node.query(f"DROP TABLE IF EXISTS {mt} SYNC") + node.query(f"DROP TABLE IF EXISTS {iceberg}") + + def test_export_part_tuple_subcolumn_partition_key_iceberg_rejected(cluster): node = cluster.instances["node1"] sfx = unique_suffix() From f9e82d3c5a4189b2815e6ff1b95817851bb5730d Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Mon, 24 Aug 2026 16:52:50 +0200 Subject: [PATCH 10/13] apply comments: update doc and names Signed-off-by: Konstantin Morozov --- docs/en/antalya/part_export.md | 23 ++++++---------- docs/en/antalya/partition_export.md | 23 ++++++---------- src/Core/Settings.cpp | 10 +++---- src/Core/SettingsChangesHistory.cpp | 2 +- src/Core/SettingsEnums.h | 4 +-- ...portReplicatedMergeTreePartitionManifest.h | 2 +- src/Storages/MergeTree/ExportPartTask.cpp | 2 +- .../MergeTree/ExportPartitionUtils.cpp | 14 +++++----- .../tests/gtest_export_partition_ordering.cpp | 20 +++++++------- .../test.py | 26 +++++++++---------- .../test.py | 12 ++++----- .../test.py | 26 +++++++++---------- .../test.py | 8 +++--- 13 files changed, 79 insertions(+), 93 deletions(-) diff --git a/docs/en/antalya/part_export.md b/docs/en/antalya/part_export.md index 1b095342ea3d..421256d95a5c 100644 --- a/docs/en/antalya/part_export.md +++ b/docs/en/antalya/part_export.md @@ -49,13 +49,13 @@ SETTINGS allow_experimental_export_merge_tree_part = 1 Source and destination tables must support positional schema conversion. The following differences between the two schemas are allowed: -- **Column names** may differ between source and destination for non-partition-key columns when `export_merge_tree_part_schema_match_mode = 'match_by_position'` (the default) - columns are matched by position, similar to `INSERT INTO dest SELECT * FROM src`, not by name. Set `export_merge_tree_part_schema_match_mode = 'match_by_name'` to match columns by their exact, case-sensitive name instead, allowing destination columns to be declared in a different order than the source. +- **Column names** may differ between source and destination for non-partition-key columns when `export_merge_tree_part_schema_match_mode = 'POSITION'` (the default) - columns are matched by position, similar to `INSERT INTO dest SELECT * FROM src`, not by name. Set `export_merge_tree_part_schema_match_mode = 'NAME'` to match columns by their exact, case-sensitive name instead, allowing destination columns to be declared in a different order than the source. - **Column types** may differ, as long as the source type is safely castable to the destination type. Set `export_merge_tree_part_allow_lossy_cast = 1` to also permit lossy casts. - **`Tuple` element names** may differ if either the source or destination declares the tuple without named elements: an unnamed `Tuple` (e.g. `Tuple(Int32, Int32)`) is matched against the destination by element position and type only, not by name. For example, exporting from `t Tuple(Int32, Int32)` to `t Tuple(x Int32, y Int32)` is allowed as long as element types match positionally. The following requirements apply to the source and destination: -1. **Column count** - by default (`export_merge_tree_part_ignore_extra_source_columns = false`) every source column must have a corresponding destination column: with `export_merge_tree_part_schema_match_mode = 'match_by_position'` (the default) the source and destination must have the same number of columns; with `'match_by_name'` they must have the same set of column names. A mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. Set `export_merge_tree_part_ignore_extra_source_columns = 1` to allow a source table with columns that have no corresponding destination column; such columns are dropped and not exported. The destination having a column absent from the source is always rejected, regardless of this setting. +1. **Column count** - by default, every source column must have a corresponding destination column, and vice versa; a mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. Which source column corresponds to which destination column is determined by `export_merge_tree_part_schema_match_mode`. Set `export_merge_tree_part_ignore_extra_source_columns = 1` to relax this in one direction: a source column without a corresponding destination column is dropped and not exported, instead of throwing. The destination having a column absent from the source is always rejected, regardless of this setting. 2. **`PARTITION BY` expressions** - the whole part must land in a single destination partition. Identical expressions always satisfy this; otherwise the destination expression has to be computable from the values the source partition key pins, or be proven single-valued over the part's min/max range. The same requirement applies to the partition fields and transforms of an Apache Iceberg destination. See [Source partition key compatibility](/docs/en/antalya/partition_export.md#source-partition-key-compatibility). 3. **The position of every column backing the partition key** - it is not enough for the `PARTITION BY` expressions to be textually identical: every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. If such a column contains a named `Tuple`, its element names must also be declared in the same order (an unnamed `Tuple` on either side is exempt from this, per the allowance above). This comparison is recursive through nested tuples and through container types such as `Array` and `Map`. @@ -139,10 +139,10 @@ In case a table function is used as the destination, the schema can be omitted a ### `export_merge_tree_part_schema_match_mode` (Optional) - **Type**: `MergeTreePartExportSchemaMatchMode` -- **Default**: `match_by_position` +- **Default**: `POSITION` - **Description**: Controls how `EXPORT PART`/`EXPORT PARTITION` matches source `MergeTree` columns to destination columns. Possible values: - - `match_by_position` (default) - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Column names are not otherwise considered. - - `match_by_name` - every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be declared in a different order than the source. A destination column absent from the source, including when it was renamed, throws `THERE_IS_NO_COLUMN`; there is no positional fallback. + - `POSITION` (default) - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Column names are not otherwise considered. + - `NAME` - every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be declared in a different order than the source. A destination column absent from the source, including when it was renamed, throws `THERE_IS_NO_COLUMN`; there is no positional fallback. See `export_merge_tree_part_ignore_extra_source_columns` below for how a source column without a corresponding destination column is handled in each mode. @@ -150,19 +150,12 @@ In case a table function is used as the destination, the schema can be omitted a - **Type**: `Bool` - **Default**: `false` -- **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` tolerates source `MergeTree` columns that have no corresponding destination column. - - `false` (default) - such a source column is rejected: the source and destination must match exactly. With `export_merge_tree_part_schema_match_mode = 'match_by_position'` this means the same number of columns; with `'match_by_name'` this means the same set of column names, so a source table with columns absent from the destination is rejected even if the matched columns would otherwise be compatible. A mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - - `true` - a source column without a corresponding destination column is dropped and not exported, instead of throwing. In `match_by_position` mode, this allows a source with extra trailing columns (the destination having more columns than the source is still always rejected). In `match_by_name` mode, this allows source columns whose name has no destination counterpart, which may occur in any position. +- **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` tolerates a source `MergeTree` column that has no corresponding destination column. + - `false` (default) - such a source column is rejected: the source and destination must match exactly. A mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. + - `true` - a source column without a corresponding destination column is dropped and not exported, instead of throwing. The destination having a column absent from the source is still always rejected. Extra source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. Type conversion and `export_merge_tree_part_allow_lossy_cast` are applied after columns are matched. - Error behavior: - - - With `export_merge_tree_part_ignore_extra_source_columns = false` (default), a source column without a corresponding destination column throws `NUMBER_OF_COLUMNS_DOESNT_MATCH` - in `match_by_position` mode this means any column-count mismatch, in `match_by_name` mode this means the source and destination column-name sets differ. - - In `match_by_position` mode, the destination having more columns than the source always throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`, regardless of `export_merge_tree_part_ignore_extra_source_columns`. - - In `match_by_name` mode, a destination column absent from the source (including a renamed one) always throws `THERE_IS_NO_COLUMN`, regardless of `export_merge_tree_part_ignore_extra_source_columns`; there is no positional fallback. - - After columns have been matched successfully, a cast rejected by the export type-safety check throws `INCOMPATIBLE_COLUMNS`. - ## Examples diff --git a/docs/en/antalya/partition_export.md b/docs/en/antalya/partition_export.md index 3a9d0cedb674..ab9d3297ba79 100644 --- a/docs/en/antalya/partition_export.md +++ b/docs/en/antalya/partition_export.md @@ -52,9 +52,9 @@ TO TABLE [destination_database.]destination_table ## Requirements -`EXPORT PARTITION` exports each part via the same mechanism as [`EXPORT PART`](/docs/en/antalya/part_export.md#requirements), so the source and destination tables must satisfy the same compatibility requirements. Columns are matched by position by default, or by their exact, case-sensitive name if `export_merge_tree_part_schema_match_mode = 'match_by_name'` is set, and column types may differ as long as they are safely castable (or `export_merge_tree_part_allow_lossy_cast = 1` is set). Beyond that, the following requirements apply: +`EXPORT PARTITION` exports each part via the same mechanism as [`EXPORT PART`](/docs/en/antalya/part_export.md#requirements), so the source and destination tables must satisfy the same compatibility requirements. Columns are matched by position by default, or by their exact, case-sensitive name if `export_merge_tree_part_schema_match_mode = 'NAME'` is set, and column types may differ as long as they are safely castable (or `export_merge_tree_part_allow_lossy_cast = 1` is set). Beyond that, the following requirements apply: -1. **Column count** - by default (`export_merge_tree_part_ignore_extra_source_columns = false`) every source column must have a corresponding destination column: with `export_merge_tree_part_schema_match_mode = 'match_by_position'` (the default) the source and destination must have the same number of columns; with `'match_by_name'` they must have the same set of column names. Set `export_merge_tree_part_ignore_extra_source_columns = 1` to allow a source table with columns that have no corresponding destination column; the destination having a column absent from the source is still always rejected. +1. **Column count** - by default, every source column must have a corresponding destination column, and vice versa; a mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. Which source column corresponds to which destination column is determined by `export_merge_tree_part_schema_match_mode`. Set `export_merge_tree_part_ignore_extra_source_columns = 1` to relax this in one direction: a source column without a corresponding destination column is dropped and not exported, instead of throwing. The destination having a column absent from the source is always rejected, regardless of this setting. 2. **`PARTITION BY` expressions** - the whole source partition must land in a single destination partition. Identical expressions always satisfy this; otherwise the destination expression has to be computable from the values the source partition key pins, or be proven single-valued over the partition's min/max range. The same requirement applies to the partition fields and transforms of an Apache Iceberg destination. See [Source partition key compatibility](#source-partition-key-compatibility). 3. **Partition key column positions and layouts** - every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. Named `Tuple` elements within such a column must also be declared in the same order, including tuples nested inside `Array` or `Map`. This applies even if both tables' `PARTITION BY` expressions are textually identical. See [`EXPORT PART` requirements](/docs/en/antalya/part_export.md#requirements) for a worked example and the corresponding exception message. @@ -138,10 +138,10 @@ Notes: ### `export_merge_tree_part_schema_match_mode` (Optional) - **Type**: `MergeTreePartExportSchemaMatchMode` -- **Default**: `match_by_position` +- **Default**: `POSITION` - **Description**: Controls how `EXPORT PART`/`EXPORT PARTITION` matches source `MergeTree` columns to destination columns. Possible values: - - `match_by_position` (default) - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Column names are not otherwise considered. - - `match_by_name` - every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be declared in a different order than the source. A destination column absent from the source, including when it was renamed, throws `THERE_IS_NO_COLUMN`; there is no positional fallback. + - `POSITION` (default) - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Column names are not otherwise considered. + - `NAME` - every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be declared in a different order than the source. A destination column absent from the source, including when it was renamed, throws `THERE_IS_NO_COLUMN`; there is no positional fallback. See `export_merge_tree_part_ignore_extra_source_columns` below for how a source column without a corresponding destination column is handled in each mode. @@ -149,19 +149,12 @@ Notes: - **Type**: `Bool` - **Default**: `false` -- **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` tolerates source `MergeTree` columns that have no corresponding destination column. - - `false` (default) - such a source column is rejected: the source and destination must match exactly. With `export_merge_tree_part_schema_match_mode = 'match_by_position'` this means the same number of columns; with `'match_by_name'` this means the same set of column names, so a source table with columns absent from the destination is rejected even if the matched columns would otherwise be compatible. A mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - - `true` - a source column without a corresponding destination column is dropped and not exported, instead of throwing. In `match_by_position` mode, this allows a source with extra trailing columns (the destination having more columns than the source is still always rejected). In `match_by_name` mode, this allows source columns whose name has no destination counterpart, which may occur in any position. +- **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` tolerates a source `MergeTree` column that has no corresponding destination column. + - `false` (default) - such a source column is rejected: the source and destination must match exactly. A mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. + - `true` - a source column without a corresponding destination column is dropped and not exported, instead of throwing. The destination having a column absent from the source is still always rejected. Extra source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. Type conversion and `export_merge_tree_part_allow_lossy_cast` are applied after columns are matched. - Error behavior: - - - With `export_merge_tree_part_ignore_extra_source_columns = false` (default), a source column without a corresponding destination column throws `NUMBER_OF_COLUMNS_DOESNT_MATCH` - in `match_by_position` mode this means any column-count mismatch, in `match_by_name` mode this means the source and destination column-name sets differ. - - In `match_by_position` mode, the destination having more columns than the source always throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`, regardless of `export_merge_tree_part_ignore_extra_source_columns`. - - In `match_by_name` mode, a destination column absent from the source (including a renamed one) always throws `THERE_IS_NO_COLUMN`, regardless of `export_merge_tree_part_ignore_extra_source_columns`; there is no positional fallback. - - After columns have been matched successfully, a cast rejected by the export type-safety check throws `INCOMPATIBLE_COLUMNS`. - ## Examples ### Basic Export to S3 diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index ec9efc301dad..47e99d476acd 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -7627,18 +7627,18 @@ Allow `EXPORT PART`/`EXPORT PARTITION` to apply lossy (non-value-preserving) cas When exporting to Apache Iceberg, the partition value written to the metadata is derived from the source partition columns by casting them to the destination partition-field types and applying the destination partition transform — the same computation the exported data files use, so the metadata stays consistent with the data. A lossy cast on a partition column remains semantically truncating: both the data files and the metadata contain the truncated value, and such casts require this setting to be enabled. )", 0) \ - DECLARE(MergeTreePartExportSchemaMatchMode, export_merge_tree_part_schema_match_mode, MergeTreePartExportSchemaMatchMode::match_by_position, R"( + DECLARE(MergeTreePartExportSchemaMatchMode, export_merge_tree_part_schema_match_mode, MergeTreePartExportSchemaMatchMode::POSITION, R"( Controls how `EXPORT PART`/`EXPORT PARTITION` matches source `MergeTree` columns to destination columns. Possible values: -- `match_by_position` (default) - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Column names are not otherwise considered. -- `match_by_name` - every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be declared in a different order than the source. A destination column absent from the source throws `THERE_IS_NO_COLUMN`, with no positional fallback. +- `POSITION` (default) - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Column names are not otherwise considered. +- `NAME` - every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be declared in a different order than the source. A destination column absent from the source throws `THERE_IS_NO_COLUMN`, with no positional fallback. See also `export_merge_tree_part_ignore_extra_source_columns`, which controls whether a source column without a corresponding destination column is dropped or rejected. )", 0) \ DECLARE(Bool, export_merge_tree_part_ignore_extra_source_columns, false, R"( Controls whether `EXPORT PART`/`EXPORT PARTITION` tolerates source `MergeTree` columns that have no corresponding destination column. -- `false` (default) - such a source column is rejected: the source and destination must match exactly (in `export_merge_tree_part_schema_match_mode = 'match_by_position'`, this means the same number of columns; in `'match_by_name'`, the same set of column names). A mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. -- `true` - a source column without a corresponding destination column is dropped and not exported, instead of throwing. In `match_by_position` mode, this allows a source with extra trailing columns (the destination having more columns than the source is still always rejected). In `match_by_name` mode, this allows source columns whose name has no destination counterpart. +- `false` (default) - such a source column is rejected: the source and destination must match exactly (in `export_merge_tree_part_schema_match_mode = 'POSITION'`, this means the same number of columns; in `'NAME'`, the same set of column names). A mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. +- `true` - a source column without a corresponding destination column is dropped and not exported, instead of throwing. In `POSITION` mode, this allows a source with extra trailing columns (the destination having more columns than the source is still always rejected). In `NAME` mode, this allows source columns whose name has no destination counterpart. Extra source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. )", 0) \ diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 26df2661a4a7..3fe09fc73344 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -44,7 +44,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"object_storage_cluster_join_mode", "allow", "allow", "New setting"}, {"export_merge_tree_partition_task_timeout_seconds", "3600", "86400", "Increase default value to make it more realistic"}, {"export_merge_tree_part_allow_lossy_cast", false, false, "New setting to gate lossy casts in EXPORT PART/PARTITION behind explicit acknowledgment"}, - {"export_merge_tree_part_schema_match_mode", "match_by_position", "match_by_position", "New setting to control how EXPORT PART/EXPORT PARTITION matches source columns to destination columns"}, + {"export_merge_tree_part_schema_match_mode", "POSITION", "POSITION", "New setting to control how EXPORT PART/EXPORT PARTITION matches source columns to destination columns"}, {"export_merge_tree_part_ignore_extra_source_columns", false, false, "New setting to allow EXPORT PART/EXPORT PARTITION when the source table has columns absent from the destination"}, {"export_merge_tree_partition_retry_initial_backoff_seconds", 5, 5, "New setting for exponential back-off between failed part export retries in an export partition task"}, {"export_merge_tree_partition_retry_max_backoff_seconds", 300, 300, "New setting capping the exponential back-off between failed part export retries in an export partition task"}, diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index a325835452d5..5544906237ba 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -576,8 +576,8 @@ DECLARE_SETTING_ENUM(MergeTreePartExportFileAlreadyExistsPolicy) enum class MergeTreePartExportSchemaMatchMode : uint8_t { - match_by_position, - match_by_name, + POSITION, + NAME, }; DECLARE_SETTING_ENUM(MergeTreePartExportSchemaMatchMode) diff --git a/src/Storages/ExportReplicatedMergeTreePartitionManifest.h b/src/Storages/ExportReplicatedMergeTreePartitionManifest.h index 3af5b82dfd7d..f592c46d9c22 100644 --- a/src/Storages/ExportReplicatedMergeTreePartitionManifest.h +++ b/src/Storages/ExportReplicatedMergeTreePartitionManifest.h @@ -373,7 +373,7 @@ struct ExportReplicatedMergeTreePartitionManifest /// Left unset (nullopt) for tasks created before these fields existed - such tasks were /// always scheduled under the old, strict column-matching check (a mismatch could never /// reach scheduling in the first place), so callers should treat an absent value as - /// `match_by_position` with `ignore_extra_source_columns = false`. + /// `POSITION` with `ignore_extra_source_columns = false`. if (json->has("schema_match_mode")) { const auto schema_match_mode = magic_enum::enum_cast(json->getValue("schema_match_mode")); diff --git a/src/Storages/MergeTree/ExportPartTask.cpp b/src/Storages/MergeTree/ExportPartTask.cpp index 3df30df41bbd..ca6ab0a9ecc3 100644 --- a/src/Storages/MergeTree/ExportPartTask.cpp +++ b/src/Storages/MergeTree/ExportPartTask.cpp @@ -131,7 +131,7 @@ namespace local_context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value; const bool ignore_extra_source_columns = local_context->getSettingsRef()[Setting::export_merge_tree_part_ignore_extra_source_columns]; - const bool match_by_name = schema_match_mode == MergeTreePartExportSchemaMatchMode::match_by_name; + const bool match_by_name = schema_match_mode == MergeTreePartExportSchemaMatchMode::NAME; auto source_columns = plan_for_part.getCurrentHeader()->getColumnsWithTypeAndName(); const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index e630219dd27a..2b72417918e7 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -220,11 +220,11 @@ namespace ExportPartitionUtils context_copy->setSetting("output_format_parquet_row_group_size_bytes", *manifest.parquet_row_group_size_bytes); /// Manifests written before these settings existed have no value here; such tasks were always /// scheduled under the old, strict column-matching check, so an absent value must resolve to - /// `match_by_position` / `false` regardless of the ambient context's settings (which may have + /// `POSITION` / `false` regardless of the ambient context's settings (which may have /// since been changed). context_copy->setSetting( "export_merge_tree_part_schema_match_mode", - String(magic_enum::enum_name(manifest.schema_match_mode.value_or(MergeTreePartExportSchemaMatchMode::match_by_position)))); + String(magic_enum::enum_name(manifest.schema_match_mode.value_or(MergeTreePartExportSchemaMatchMode::POSITION)))); context_copy->setSetting( "export_merge_tree_part_ignore_extra_source_columns", manifest.ignore_extra_source_columns.value_or(false)); @@ -979,12 +979,12 @@ namespace { switch (schema_match_mode) { - case MergeTreePartExportSchemaMatchMode::match_by_position: { + case MergeTreePartExportSchemaMatchMode::POSITION: { for (size_t i = 0; i < destination_columns.size(); ++i) verifyExportColumnCastIsSafe(source_columns[i], destination_columns[i], destination_storage_id); return; } - case MergeTreePartExportSchemaMatchMode::match_by_name: { + case MergeTreePartExportSchemaMatchMode::NAME: { std::unordered_map source_columns_by_name; source_columns_by_name.reserve(source_columns.size()); for (const auto & source_column : source_columns) @@ -1048,7 +1048,7 @@ namespace context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value; const bool ignore_extra_source_columns = context->getSettingsRef()[Setting::export_merge_tree_part_ignore_extra_source_columns]; - const bool match_by_name = schema_match_mode == MergeTreePartExportSchemaMatchMode::match_by_name; + const bool match_by_name = schema_match_mode == MergeTreePartExportSchemaMatchMode::NAME; const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); checkExportSchemaColumnsCount( @@ -1091,7 +1091,7 @@ namespace switch (schema_match_mode) { - case MergeTreePartExportSchemaMatchMode::match_by_name: + case MergeTreePartExportSchemaMatchMode::NAME: { std::unordered_map source_positions_by_name; source_positions_by_name.reserve(source_columns.size()); @@ -1113,7 +1113,7 @@ namespace } break; } - case MergeTreePartExportSchemaMatchMode::match_by_position: + case MergeTreePartExportSchemaMatchMode::POSITION: { const size_t num_columns = std::min(source_columns.size(), destination_columns.size()); for (size_t i = 0; i < num_columns; ++i) diff --git a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp index b01c564e63d9..c31f4908f28f 100644 --- a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp +++ b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp @@ -149,7 +149,7 @@ TEST_F(ExportPartitionOrderingTest, IterationOrderMatchesCreateTime) TEST_F(ExportPartitionManifestBackCompatTest, MissingSchemaMatchModeParsesAsNullopt) { auto manifest = makeValidManifest(); - manifest.schema_match_mode = MergeTreePartExportSchemaMatchMode::match_by_name; + manifest.schema_match_mode = MergeTreePartExportSchemaMatchMode::NAME; Poco::JSON::Parser parser; auto json = parser.parse(manifest.toJsonString()).extract(); @@ -216,7 +216,7 @@ TEST_F(ExportPartitionManifestBackCompatTest, MissingSchemaMatchSettingsFallBack EXPECT_EQ( worker_context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value, - MergeTreePartExportSchemaMatchMode::match_by_position); + MergeTreePartExportSchemaMatchMode::POSITION); EXPECT_EQ( worker_context->getSettingsRef()[Setting::export_merge_tree_part_ignore_extra_source_columns].value, false); @@ -273,7 +273,7 @@ TEST(ExportColumnCastsTest, UsesSelectedMatchingMode) ExportPartitionUtils::verifyExportColumnCastsAreSafe( source_columns, destination_columns, - MergeTreePartExportSchemaMatchMode::match_by_position, + MergeTreePartExportSchemaMatchMode::POSITION, destination_storage_id); }, ErrorCodes::INCOMPATIBLE_COLUMNS); @@ -281,7 +281,7 @@ TEST(ExportColumnCastsTest, UsesSelectedMatchingMode) EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( source_columns, destination_columns, - MergeTreePartExportSchemaMatchMode::match_by_name, + MergeTreePartExportSchemaMatchMode::NAME, destination_storage_id)); } @@ -305,7 +305,7 @@ TEST(ExportColumnCastsTest, MatchByNameAcceptsReorderedColumnsWithEqualColumnCou ExportPartitionUtils::verifyExportColumnCastsAreSafe( source_columns, reordered_destination_columns, - MergeTreePartExportSchemaMatchMode::match_by_position, + MergeTreePartExportSchemaMatchMode::POSITION, destination_storage_id); }, ErrorCodes::INCOMPATIBLE_COLUMNS); @@ -313,7 +313,7 @@ TEST(ExportColumnCastsTest, MatchByNameAcceptsReorderedColumnsWithEqualColumnCou EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( source_columns, reordered_destination_columns, - MergeTreePartExportSchemaMatchMode::match_by_name, + MergeTreePartExportSchemaMatchMode::NAME, destination_storage_id)); const ColumnsWithTypeAndName same_order_destination_columns = { @@ -322,7 +322,7 @@ TEST(ExportColumnCastsTest, MatchByNameAcceptsReorderedColumnsWithEqualColumnCou makeColumn("payload"), }; - for (const auto mode : {MergeTreePartExportSchemaMatchMode::match_by_position, MergeTreePartExportSchemaMatchMode::match_by_name}) + for (const auto mode : {MergeTreePartExportSchemaMatchMode::POSITION, MergeTreePartExportSchemaMatchMode::NAME}) EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( source_columns, same_order_destination_columns, mode, destination_storage_id)); } @@ -343,7 +343,7 @@ TEST(ExportColumnCastsTest, MatchByNameToleratesUnmatchedSourceColumn) EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( source_columns, destination_columns, - MergeTreePartExportSchemaMatchMode::match_by_name, + MergeTreePartExportSchemaMatchMode::NAME, destination_storage_id)); } @@ -365,7 +365,7 @@ TEST(ExportColumnCastsTest, RejectsLossyCastAfterMatchingByName) ExportPartitionUtils::verifyExportColumnCastsAreSafe( source_columns, destination_columns, - MergeTreePartExportSchemaMatchMode::match_by_name, + MergeTreePartExportSchemaMatchMode::NAME, StorageID{"test", "destination"}); }, ErrorCodes::INCOMPATIBLE_COLUMNS); @@ -389,7 +389,7 @@ TEST(ExportColumnCastsTest, RejectsMissingDestinationColumnAfterMatchingByName) ExportPartitionUtils::verifyExportColumnCastsAreSafe( source_columns, destination_columns, - MergeTreePartExportSchemaMatchMode::match_by_name, + MergeTreePartExportSchemaMatchMode::NAME, StorageID{"test", "destination"}); }, ErrorCodes::THERE_IS_NO_COLUMN); diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index 98c7626848fb..053dec967036 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -38,8 +38,8 @@ EXTRA_SOURCE_COLUMN_MODES = [ - pytest.param("match_by_position", id="by-position"), - pytest.param("match_by_name", id="by-name"), + pytest.param("POSITION", id="by-position"), + pytest.param("NAME", id="by-name"), ] @@ -781,8 +781,8 @@ def test_export_part_source_more_columns_allowed_with_ignore_extra_setting(clust @pytest.mark.parametrize( "schema_match_mode,expected_error", [ - pytest.param("match_by_position", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), - pytest.param("match_by_name", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-name"), + pytest.param("POSITION", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), + pytest.param("NAME", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-name"), ], ) def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_ignore_extra_setting( @@ -858,7 +858,7 @@ def test_export_part_ignore_extra_column_breaks_hybrid_over_source_and_destinati export_part( node=node, table=mt, part=part, dest=iceberg, - extra_settings="export_merge_tree_part_schema_match_mode = 'match_by_position', export_merge_tree_part_ignore_extra_source_columns = 1", + extra_settings="export_merge_tree_part_schema_match_mode = 'POSITION', export_merge_tree_part_ignore_extra_source_columns = 1", ) wait_for_export_part(node=node, table=mt, part=part) @@ -1063,7 +1063,7 @@ def test_export_part_reordered_subset_requires_matching_by_name(cluster): f"ALTER TABLE {mt} EXPORT PART '{part_2020}' TO TABLE {iceberg} " f"SETTINGS allow_experimental_export_merge_tree_part = 1, " f"allow_experimental_insert_into_iceberg = 1, " - f"export_merge_tree_part_schema_match_mode = 'match_by_position', " + f"export_merge_tree_part_schema_match_mode = 'POSITION', " f"export_merge_tree_part_ignore_extra_source_columns = 1" ) assert "INCOMPATIBLE_COLUMNS" in error, f"Expected positional matching to fail, got: {error!r}" @@ -1074,7 +1074,7 @@ def test_export_part_reordered_subset_requires_matching_by_name(cluster): mt, part_2020, iceberg, - extra_settings="export_merge_tree_part_schema_match_mode = 'match_by_name', export_merge_tree_part_ignore_extra_source_columns = 1", + extra_settings="export_merge_tree_part_schema_match_mode = 'NAME', export_merge_tree_part_ignore_extra_source_columns = 1", ) wait_for_export_part(node, mt, part_2020) @@ -1109,7 +1109,7 @@ def test_export_part_match_by_name_requires_every_destination_column(cluster): f"ALTER TABLE {mt} EXPORT PART '{part_2020}' TO TABLE {iceberg} " f"SETTINGS allow_experimental_export_merge_tree_part = 1, " f"allow_experimental_insert_into_iceberg = 1, " - f"export_merge_tree_part_schema_match_mode = 'match_by_name', " + f"export_merge_tree_part_schema_match_mode = 'NAME', " f"export_merge_tree_part_ignore_extra_source_columns = 1" ) assert "THERE_IS_NO_COLUMN" in error and "renamed_id" in error, ( @@ -1141,7 +1141,7 @@ def test_export_part_match_by_name_rejects_renamed_column_without_extra_source_c f"ALTER TABLE {mt} EXPORT PART '{part_2020}' TO TABLE {iceberg} " f"SETTINGS allow_experimental_export_merge_tree_part = 1, " f"allow_experimental_insert_into_iceberg = 1, " - f"export_merge_tree_part_schema_match_mode = 'match_by_name'" + f"export_merge_tree_part_schema_match_mode = 'NAME'" ) assert "THERE_IS_NO_COLUMN" in error and "renamed_id" in error, ( f"Expected name matching to reject missing column `renamed_id` even without an extra " @@ -1154,7 +1154,7 @@ def test_export_part_match_by_name_rejects_renamed_column_without_extra_source_c def test_export_part_match_by_name_reorders_columns_without_extra_source_columns(cluster): - """The scenario `export_merge_tree_part_schema_match_mode = 'match_by_name'` was introduced for: + """The scenario `export_merge_tree_part_schema_match_mode = 'NAME'` was introduced for: the source and destination have the exact same number of columns, declared in a different order. Previously this fell back to positional matching (a no-op for by-name mode); now it is matched by name like any other case.""" @@ -1171,7 +1171,7 @@ def test_export_part_match_by_name_reorders_columns_without_extra_source_columns export_part( node, mt, part_2020, iceberg, - extra_settings="export_merge_tree_part_schema_match_mode = 'match_by_name'", + extra_settings="export_merge_tree_part_schema_match_mode = 'NAME'", ) wait_for_export_part(node, mt, part_2020) @@ -1307,7 +1307,7 @@ def test_export_part_match_by_name_revalidates_extra_source_columns_in_backgroun mt, part, iceberg, - "export_merge_tree_part_schema_match_mode = 'match_by_name'", + "export_merge_tree_part_schema_match_mode = 'NAME'", ) node.query("SYSTEM WAIT FAILPOINT export_part_pause_before_schema_validation PAUSE") @@ -1346,7 +1346,7 @@ def test_export_part_match_by_name_uses_source_snapshot_when_source_column_is_ad mt, part, iceberg, - "export_merge_tree_part_schema_match_mode = 'match_by_name'", + "export_merge_tree_part_schema_match_mode = 'NAME'", ) node.query("SYSTEM WAIT FAILPOINT export_part_pause_before_schema_validation PAUSE") diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index 2c1952e66109..3d18f7393be1 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -10,8 +10,8 @@ EXTRA_SOURCE_COLUMN_MODES = [ - pytest.param("match_by_position", id="by-position"), - pytest.param("match_by_name", id="by-name"), + pytest.param("POSITION", id="by-position"), + pytest.param("NAME", id="by-name"), ] @@ -988,8 +988,8 @@ def test_export_part_column_count_mismatch_source_fewer_is_rejected(cluster): @pytest.mark.parametrize( "schema_match_mode,expected_error", [ - pytest.param("match_by_position", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), - pytest.param("match_by_name", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-name"), + pytest.param("POSITION", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), + pytest.param("NAME", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-name"), ], ) def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_ignore_extra_setting( @@ -1089,7 +1089,7 @@ def test_export_part_match_by_name_with_equal_column_count_reordered(cluster): node.query( f"ALTER TABLE {mt_table} EXPORT PART '2020_1_1_0' TO TABLE {s3_table} " - f"SETTINGS export_merge_tree_part_schema_match_mode = 'match_by_name'" + f"SETTINGS export_merge_tree_part_schema_match_mode = 'NAME'" ) wait_for_export_part(node=node, table=mt_table, part="2020_1_1_0") @@ -1123,7 +1123,7 @@ def test_export_part_match_by_name_with_equal_column_count_rejects_unmatched_sou error = node.query_and_get_error( f"ALTER TABLE {mt_table} EXPORT PART '2020_1_1_0' TO TABLE {s3_table} " - f"SETTINGS export_merge_tree_part_schema_match_mode = 'match_by_name'" + f"SETTINGS export_merge_tree_part_schema_match_mode = 'NAME'" ) assert "NUMBER_OF_COLUMNS_DOESNT_MATCH" in error, ( f"Expected NUMBER_OF_COLUMNS_DOESNT_MATCH for an unmatched source column with " diff --git a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py b/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py index e0ed3bfa4a0b..4c746bc26607 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py @@ -27,8 +27,8 @@ EXTRA_SOURCE_COLUMN_MODES = [ - pytest.param("match_by_position", id="by-position"), - pytest.param("match_by_name", id="by-name"), + pytest.param("POSITION", id="by-position"), + pytest.param("NAME", id="by-name"), ] @@ -1588,8 +1588,8 @@ def test_export_partition_column_count_mismatch_source_fewer_is_rejected(cluster def test_export_partition_source_more_columns_allowed_with_ignore_extra_setting(cluster, schema_match_mode): """ Source has 3 columns (id, year, extra), destination has 2 (id, year). - With `export_merge_tree_part_schema_match_mode` set to either `match_by_position` or - `match_by_name` and `export_merge_tree_part_ignore_extra_source_columns = 1`, the export must succeed: the + With `export_merge_tree_part_schema_match_mode` set to either `POSITION` or + `NAME` and `export_merge_tree_part_ignore_extra_source_columns = 1`, the export must succeed: the trailing `extra` source column is dropped and only `id`/`year` land in the destination. """ node = cluster.instances["replica1"] @@ -1620,11 +1620,11 @@ def test_export_partition_source_more_columns_allowed_with_ignore_extra_setting( f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}", settings={ "allow_insert_into_iceberg": 1, - "export_merge_tree_part_schema_match_mode": "match_by_position", + "export_merge_tree_part_schema_match_mode": "POSITION", }, ) assert "NUMBER_OF_COLUMNS_DOESNT_MATCH" in error, ( - f"Expected NUMBER_OF_COLUMNS_DOESNT_MATCH with schema_match_mode='match_by_position', got: {error!r}" + f"Expected NUMBER_OF_COLUMNS_DOESNT_MATCH with schema_match_mode='POSITION', got: {error!r}" ) node.query( @@ -1648,8 +1648,8 @@ def test_export_partition_source_more_columns_allowed_with_ignore_extra_setting( @pytest.mark.parametrize( "schema_match_mode,expected_error", [ - pytest.param("match_by_position", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), - pytest.param("match_by_name", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-name"), + pytest.param("POSITION", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), + pytest.param("NAME", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-name"), ], ) def test_export_partition_column_count_mismatch_source_fewer_still_rejected_with_ignore_extra_setting( @@ -1657,7 +1657,7 @@ def test_export_partition_column_count_mismatch_source_fewer_still_rejected_with ): """ Setting `export_merge_tree_part_ignore_extra_source_columns = 1` never relaxes the source-has-fewer-columns - direction, in either `match_by_position` or `match_by_name` mode. Source has 2 columns + direction, in either `POSITION` or `NAME` mode. Source has 2 columns (id, year), destination has 3 (id, year, extra): the destination cannot be filled from the source, so this must still be rejected synchronously even with the relaxed setting. """ @@ -1894,7 +1894,7 @@ def test_export_partition_ignore_extra_setting_prefix_contains_different_name(cl f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}", settings={ "allow_insert_into_iceberg": 1, - "export_merge_tree_part_schema_match_mode": "match_by_position", + "export_merge_tree_part_schema_match_mode": "POSITION", "export_merge_tree_part_ignore_extra_source_columns": 1, }, ) @@ -1910,7 +1910,7 @@ def test_export_partition_ignore_extra_setting_prefix_contains_different_name(cl @pytest.mark.parametrize("schema_match_mode", EXTRA_SOURCE_COLUMN_MODES) def test_export_partition_matches_columns_when_column_counts_are_equal(cluster, schema_match_mode): """Source and destination declare the same 2 columns, in the same order and with the same - names, so `match_by_position` and `match_by_name` must agree and both succeed identically - + names, so `POSITION` and `NAME` must agree and both succeed identically - there is no unmatched source column for `export_merge_tree_part_ignore_extra_source_columns` to affect.""" node = cluster.instances["replica1"] @@ -1951,7 +1951,7 @@ def test_export_partition_column_count_mismatch_into_table_with_existing_data(cl ignore_extra_settings = { "allow_insert_into_iceberg": 1, - "export_merge_tree_part_schema_match_mode": "match_by_position", + "export_merge_tree_part_schema_match_mode": "POSITION", "export_merge_tree_part_ignore_extra_source_columns": 1, } @@ -2001,7 +2001,7 @@ def test_export_partition_column_count_mismatch_into_partition_that_already_has_ ignore_extra_settings = { "allow_insert_into_iceberg": 1, - "export_merge_tree_part_schema_match_mode": "match_by_position", + "export_merge_tree_part_schema_match_mode": "POSITION", "export_merge_tree_part_ignore_extra_source_columns": 1, } diff --git a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py index 3cb6f0d2b9da..b6c721c87dbf 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py @@ -17,8 +17,8 @@ EXTRA_SOURCE_COLUMN_MODES = [ - pytest.param("match_by_position", id="by-position"), - pytest.param("match_by_name", id="by-name"), + pytest.param("POSITION", id="by-position"), + pytest.param("NAME", id="by-name"), ] @@ -2351,7 +2351,7 @@ def test_export_partition_match_by_name_honored_by_non_initiating_replica(cluste replica1.query( f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}" - f" SETTINGS export_merge_tree_part_schema_match_mode = 'match_by_name'," + f" SETTINGS export_merge_tree_part_schema_match_mode = 'NAME'," f" export_merge_tree_part_ignore_extra_source_columns = 1" ) @@ -2400,7 +2400,7 @@ def test_export_partition_match_by_name_with_equal_column_count_reordered(cluste replica1.query( f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}" - f" SETTINGS export_merge_tree_part_schema_match_mode = 'match_by_name'" + f" SETTINGS export_merge_tree_part_schema_match_mode = 'NAME'" ) wait_for_export_status(node=replica1, source_table=mt_table, dest_table=s3_table, From 3ea77261c6d39994b8e59f1a7c54eccf87cde0e7 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Mon, 24 Aug 2026 17:50:51 +0200 Subject: [PATCH 11/13] add comments Signed-off-by: Konstantin Morozov --- src/Storages/MergeTree/ExportPartTask.cpp | 12 +++++++----- .../MergeTree/ExportPartitionUtils.cpp | 18 +++++++++++------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/Storages/MergeTree/ExportPartTask.cpp b/src/Storages/MergeTree/ExportPartTask.cpp index ca6ab0a9ecc3..82c85f4b6e09 100644 --- a/src/Storages/MergeTree/ExportPartTask.cpp +++ b/src/Storages/MergeTree/ExportPartTask.cpp @@ -131,7 +131,6 @@ namespace local_context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value; const bool ignore_extra_source_columns = local_context->getSettingsRef()[Setting::export_merge_tree_part_ignore_extra_source_columns]; - const bool match_by_name = schema_match_mode == MergeTreePartExportSchemaMatchMode::NAME; auto source_columns = plan_for_part.getCurrentHeader()->getColumnsWithTypeAndName(); const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); @@ -141,7 +140,12 @@ namespace destination_columns.size(), ignore_extra_source_columns); - if (!match_by_name && ignore_extra_source_columns && src_has_extra_columns) + const ActionsDAG::MatchColumnsMode mode = schema_match_mode == MergeTreePartExportSchemaMatchMode::NAME + ? ActionsDAG::MatchColumnsMode::Name + : ActionsDAG::MatchColumnsMode::Position; + + // makeConvertingActions with postitional mode requires equal columns count, + if (ActionsDAG::MatchColumnsMode::Position == mode && ignore_extra_source_columns && src_has_extra_columns) { LOG_DEBUG(getLogger("ExportPartTask"), "Source has {} columns while destination has {} columns, " @@ -173,9 +177,7 @@ namespace auto dag = ActionsDAG::makeConvertingActions( source_columns, destination_columns, - match_by_name - ? ActionsDAG::MatchColumnsMode::Name - : ActionsDAG::MatchColumnsMode::Position, + mode, local_context); auto expression_step = std::make_unique( diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 2b72417918e7..1278a59c044a 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -1048,7 +1048,6 @@ namespace context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value; const bool ignore_extra_source_columns = context->getSettingsRef()[Setting::export_merge_tree_part_ignore_extra_source_columns]; - const bool match_by_name = schema_match_mode == MergeTreePartExportSchemaMatchMode::NAME; const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); checkExportSchemaColumnsCount( @@ -1056,12 +1055,19 @@ namespace destination_columns.size(), ignore_extra_source_columns); - if (!match_by_name && ignore_extra_source_columns && src_has_extra_columns) + const ActionsDAG::MatchColumnsMode mode = schema_match_mode == MergeTreePartExportSchemaMatchMode::NAME + ? ActionsDAG::MatchColumnsMode::Name + : ActionsDAG::MatchColumnsMode::Position; + + // makeConvertingActions with postitional mode requires equal columns count, + if (ActionsDAG::MatchColumnsMode::Position == mode && ignore_extra_source_columns && src_has_extra_columns) { - LOG_DEBUG(getLogger("ExportPartitionUtils"), + LOG_DEBUG( + getLogger("ExportPartitionUtils"), "Source has {} columns while destination has {} columns, " "the {} extra trailing source column(s) will be ignored", - source_columns.size(), destination_columns.size(), + source_columns.size(), + destination_columns.size(), source_columns.size() - destination_columns.size()); source_columns.resize(destination_columns.size()); @@ -1070,9 +1076,7 @@ namespace (void) ActionsDAG::makeConvertingActions( source_columns, destination_columns, - match_by_name - ? ActionsDAG::MatchColumnsMode::Name - : ActionsDAG::MatchColumnsMode::Position, + mode, context); const auto & source_columns_description = source_metadata->getColumns(); From de98677687baa05ccd6ae078420b1ce1de8fedd5 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Tue, 25 Aug 2026 14:20:08 +0200 Subject: [PATCH 12/13] refactoting Signed-off-by: Konstantin Morozov --- .../MergeTree/ExportPartitionUtils.cpp | 106 ++++------- src/Storages/MergeTree/ExportPartitionUtils.h | 6 - .../tests/gtest_export_partition_ordering.cpp | 174 ------------------ 3 files changed, 34 insertions(+), 252 deletions(-) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 1278a59c044a..eb5ded5ec6bc 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -971,40 +971,6 @@ namespace destination_column.type->getName()); } - void verifyExportColumnCastsAreSafe( - const ColumnsWithTypeAndName & source_columns, - const ColumnsWithTypeAndName & destination_columns, - MergeTreePartExportSchemaMatchMode schema_match_mode, - const StorageID & destination_storage_id) - { - switch (schema_match_mode) - { - case MergeTreePartExportSchemaMatchMode::POSITION: { - for (size_t i = 0; i < destination_columns.size(); ++i) - verifyExportColumnCastIsSafe(source_columns[i], destination_columns[i], destination_storage_id); - return; - } - case MergeTreePartExportSchemaMatchMode::NAME: { - std::unordered_map source_columns_by_name; - source_columns_by_name.reserve(source_columns.size()); - for (const auto & source_column : source_columns) - source_columns_by_name.emplace(source_column.name, &source_column); - - for (const auto & destination_column : destination_columns) - { - const auto source_it = source_columns_by_name.find(destination_column.name); - if (source_it == source_columns_by_name.end()) - throw Exception( - ErrorCodes::THERE_IS_NO_COLUMN, "Cannot find column `{}` in source stream", destination_column.name); - - verifyExportColumnCastIsSafe(*source_it->second, destination_column, destination_storage_id); - } - return; - } - } - UNREACHABLE(); - } - void checkExportSchemaColumnsCount( size_t source_columns_count, size_t destination_columns_count, @@ -1044,16 +1010,11 @@ namespace auto source_columns = source_sample_block.getColumnsWithTypeAndName(); const auto & destination_columns = destination_sample_block.getColumnsWithTypeAndName(); - const auto schema_match_mode = - context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value; - const bool ignore_extra_source_columns = - context->getSettingsRef()[Setting::export_merge_tree_part_ignore_extra_source_columns]; + const auto schema_match_mode = context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value; + const bool ignore_extra_source_columns = context->getSettingsRef()[Setting::export_merge_tree_part_ignore_extra_source_columns]; const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); - checkExportSchemaColumnsCount( - source_columns.size(), - destination_columns.size(), - ignore_extra_source_columns); + checkExportSchemaColumnsCount(source_columns.size(), destination_columns.size(), ignore_extra_source_columns); const ActionsDAG::MatchColumnsMode mode = schema_match_mode == MergeTreePartExportSchemaMatchMode::NAME ? ActionsDAG::MatchColumnsMode::Name @@ -1073,11 +1034,7 @@ namespace source_columns.resize(destination_columns.size()); } - (void) ActionsDAG::makeConvertingActions( - source_columns, - destination_columns, - mode, - context); + (void)ActionsDAG::makeConvertingActions(source_columns, destination_columns, mode, context); const auto & source_columns_description = source_metadata->getColumns(); /// Collect the top-level columns that own columns or subcolumns required by `PARTITION BY`. @@ -1085,8 +1042,7 @@ namespace std::unordered_set partition_key_owner_columns; for (const auto & column_or_subcolumn_name : source_metadata->getColumnsRequiredForPartitionKey()) { - auto resolved = source_columns_description.tryGetColumnOrSubcolumn( - GetColumnsOptions::All, column_or_subcolumn_name); + auto resolved = source_columns_description.tryGetColumnOrSubcolumn(GetColumnsOptions::All, column_or_subcolumn_name); const auto & column_name = resolved ? resolved->getNameInStorage() : column_or_subcolumn_name; partition_key_owner_columns.insert(column_name); } @@ -1095,8 +1051,7 @@ namespace switch (schema_match_mode) { - case MergeTreePartExportSchemaMatchMode::NAME: - { + case MergeTreePartExportSchemaMatchMode::NAME: { std::unordered_map source_positions_by_name; source_positions_by_name.reserve(source_columns.size()); for (size_t i = 0; i < source_columns.size(); ++i) @@ -1104,39 +1059,46 @@ namespace for (const auto & destination_column : destination_columns) { - if (!partition_key_owner_columns.contains(destination_column.name)) - continue; - const auto source_it = source_positions_by_name.find(destination_column.name); if (source_it == source_positions_by_name.end()) - continue; + throw Exception( + ErrorCodes::THERE_IS_NO_COLUMN, "Cannot find column `{}` in source stream", destination_column.name); + + const auto & source_column = source_columns[source_it->second]; - verifyPartitionKeyColumn( - source_columns[source_it->second], destination_column, source_it->second, destination_storage_id, - /*match_by_name=*/ true); + if (partition_key_owner_columns.contains(destination_column.name)) + verifyPartitionKeyColumn( + source_column, + destination_column, + source_it->second, + destination_storage_id, + /*match_by_name=*/true); + + /// Lossy casts may silently change values, so reject them unless the user opts in. + if (!allow_lossy_cast) + verifyExportColumnCastIsSafe(source_column, destination_column, destination_storage_id); } break; } - case MergeTreePartExportSchemaMatchMode::POSITION: - { + case MergeTreePartExportSchemaMatchMode::POSITION: { const size_t num_columns = std::min(source_columns.size(), destination_columns.size()); for (size_t i = 0; i < num_columns; ++i) + { if (partition_key_owner_columns.contains(source_columns[i].name)) - verifyPartitionKeyColumn(source_columns[i], destination_columns[i], i, destination_storage_id, - /*match_by_name=*/ false); + verifyPartitionKeyColumn( + source_columns[i], + destination_columns[i], + i, + destination_storage_id, + /*match_by_name=*/false); + + /// Lossy casts may silently change values, so reject them unless the user opts in. + if (!allow_lossy_cast) + verifyExportColumnCastIsSafe(source_columns[i], destination_columns[i], destination_storage_id); + } break; } } - - /// Lossy casts may silently change values, so reject them unless the user opts in. - if (allow_lossy_cast) - return; - - verifyExportColumnCastsAreSafe( - source_columns, - destination_columns, - schema_match_mode, - destination_storage_id); } } diff --git a/src/Storages/MergeTree/ExportPartitionUtils.h b/src/Storages/MergeTree/ExportPartitionUtils.h index 5119139a848d..cf5493cae675 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.h +++ b/src/Storages/MergeTree/ExportPartitionUtils.h @@ -89,12 +89,6 @@ namespace ExportPartitionUtils const StorageMetadataPtr & source_metadata, const StorageMetadataPtr & destination_metadata); - void verifyExportColumnCastsAreSafe( - const ColumnsWithTypeAndName & source_columns, - const ColumnsWithTypeAndName & destination_columns, - MergeTreePartExportSchemaMatchMode schema_match_mode, - const StorageID & destination_storage_id); - void checkExportSchemaColumnsCount( size_t source_columns_count, size_t destination_columns_count, diff --git a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp index c31f4908f28f..49c86a51e66d 100644 --- a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp +++ b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp @@ -3,10 +3,7 @@ #include #include #include -#include #include -#include -#include #include #include #include @@ -14,13 +11,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int INCOMPATIBLE_COLUMNS; - extern const int THERE_IS_NO_COLUMN; - extern const int NUMBER_OF_COLUMNS_DOESNT_MATCH; -} - namespace Setting { extern const SettingsMergeTreePartExportSchemaMatchMode export_merge_tree_part_schema_match_mode; @@ -29,27 +19,6 @@ namespace Setting namespace { - template - ColumnWithTypeAndName makeColumn(const String & name) - { - auto type = std::make_shared(); - return {type->createColumn(), type, name}; - } - - template - void expectExceptionCode(Function && function, int expected_code) - { - try - { - function(); - FAIL() << "Expected exception code " << expected_code; - } - catch (const Exception & exception) - { - EXPECT_EQ(exception.code(), expected_code) << exception.message(); - } - } - ExportReplicatedMergeTreePartitionManifest makeValidManifest() { ExportReplicatedMergeTreePartitionManifest manifest; @@ -252,147 +221,4 @@ TEST_F(ExportPartitionManifestBackCompatTest, IgnoreExtraSourceColumnsAppliedToW } } -TEST(ExportColumnCastsTest, UsesSelectedMatchingMode) -{ - const ColumnsWithTypeAndName source_columns = { - makeColumn("id"), - makeColumn("year"), - makeColumn("payload"), - makeColumn("extra"), - }; - const ColumnsWithTypeAndName destination_columns = { - makeColumn("payload"), - makeColumn("year"), - makeColumn("id"), - }; - const StorageID destination_storage_id{"test", "destination"}; - - expectExceptionCode( - [&] - { - ExportPartitionUtils::verifyExportColumnCastsAreSafe( - source_columns, - destination_columns, - MergeTreePartExportSchemaMatchMode::POSITION, - destination_storage_id); - }, - ErrorCodes::INCOMPATIBLE_COLUMNS); - - EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( - source_columns, - destination_columns, - MergeTreePartExportSchemaMatchMode::NAME, - destination_storage_id)); -} - -TEST(ExportColumnCastsTest, MatchByNameAcceptsReorderedColumnsWithEqualColumnCount) -{ - const ColumnsWithTypeAndName source_columns = { - makeColumn("id"), - makeColumn("year"), - makeColumn("payload"), - }; - const ColumnsWithTypeAndName reordered_destination_columns = { - makeColumn("payload"), - makeColumn("year"), - makeColumn("id"), - }; - const StorageID destination_storage_id{"test", "destination"}; - - expectExceptionCode( - [&] - { - ExportPartitionUtils::verifyExportColumnCastsAreSafe( - source_columns, - reordered_destination_columns, - MergeTreePartExportSchemaMatchMode::POSITION, - destination_storage_id); - }, - ErrorCodes::INCOMPATIBLE_COLUMNS); - - EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( - source_columns, - reordered_destination_columns, - MergeTreePartExportSchemaMatchMode::NAME, - destination_storage_id)); - - const ColumnsWithTypeAndName same_order_destination_columns = { - makeColumn("id"), - makeColumn("year"), - makeColumn("payload"), - }; - - for (const auto mode : {MergeTreePartExportSchemaMatchMode::POSITION, MergeTreePartExportSchemaMatchMode::NAME}) - EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( - source_columns, same_order_destination_columns, mode, destination_storage_id)); -} - -TEST(ExportColumnCastsTest, MatchByNameToleratesUnmatchedSourceColumn) -{ - const ColumnsWithTypeAndName source_columns = { - makeColumn("id"), - makeColumn("year"), - makeColumn("extra"), - }; - const ColumnsWithTypeAndName destination_columns = { - makeColumn("id"), - makeColumn("year"), - }; - const StorageID destination_storage_id{"test", "destination"}; - - EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( - source_columns, - destination_columns, - MergeTreePartExportSchemaMatchMode::NAME, - destination_storage_id)); -} - -TEST(ExportColumnCastsTest, RejectsLossyCastAfterMatchingByName) -{ - const ColumnsWithTypeAndName source_columns = { - makeColumn("id"), - makeColumn("year"), - makeColumn("extra"), - }; - const ColumnsWithTypeAndName destination_columns = { - makeColumn("id"), - makeColumn("year"), - }; - - expectExceptionCode( - [&] - { - ExportPartitionUtils::verifyExportColumnCastsAreSafe( - source_columns, - destination_columns, - MergeTreePartExportSchemaMatchMode::NAME, - StorageID{"test", "destination"}); - }, - ErrorCodes::INCOMPATIBLE_COLUMNS); -} - -TEST(ExportColumnCastsTest, RejectsMissingDestinationColumnAfterMatchingByName) -{ - const ColumnsWithTypeAndName source_columns = { - makeColumn("id"), - makeColumn("year"), - makeColumn("extra"), - }; - const ColumnsWithTypeAndName destination_columns = { - makeColumn("renamed_id"), - makeColumn("year"), - }; - - expectExceptionCode( - [&] - { - ExportPartitionUtils::verifyExportColumnCastsAreSafe( - source_columns, - destination_columns, - MergeTreePartExportSchemaMatchMode::NAME, - StorageID{"test", "destination"}); - }, - ErrorCodes::THERE_IS_NO_COLUMN); -} - } From 4ef9f85707812ea0f5cc273e78aaabcd69369310 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Tue, 25 Aug 2026 15:09:06 +0200 Subject: [PATCH 13/13] Restore internal linkage for export helper functions The rebase onto antalya-26.3 dropped the anonymous `namespace` wrapping `haveSameTupleElementLayout`, `verifyExportColumnCastIsSafe`, and `verifyPartitionKeyColumn` during conflict resolution. None of the three are declared in `ExportPartitionUtils.h`, so they should keep internal linkage as before. --- .../MergeTree/ExportPartitionUtils.cpp | 165 +++++++++--------- 1 file changed, 84 insertions(+), 81 deletions(-) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index eb5ded5ec6bc..7f01588af557 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -868,107 +868,110 @@ namespace source_metadata->getPartitionKey(), destination_metadata->getPartitionKey(), parts, partition_id, context); } - bool haveSameTupleElementLayout(const DataTypePtr & source_type, const DataTypePtr & destination_type) + namespace { - const auto source_type_unwrapped = removeNullable(removeLowCardinality(source_type)); - const auto destination_type_unwrapped = removeNullable(removeLowCardinality(destination_type)); - - const auto * source_tuple = checkAndGetDataType(source_type_unwrapped.get()); - const auto * destination_tuple = checkAndGetDataType(destination_type_unwrapped.get()); - if (source_tuple || destination_tuple) + bool haveSameTupleElementLayout(const DataTypePtr & source_type, const DataTypePtr & destination_type) { - if (!source_tuple || !destination_tuple) - return false; + const auto source_type_unwrapped = removeNullable(removeLowCardinality(source_type)); + const auto destination_type_unwrapped = removeNullable(removeLowCardinality(destination_type)); - if (source_tuple->hasExplicitNames() && destination_tuple->hasExplicitNames()) + const auto * source_tuple = checkAndGetDataType(source_type_unwrapped.get()); + const auto * destination_tuple = checkAndGetDataType(destination_type_unwrapped.get()); + if (source_tuple || destination_tuple) { - if (source_tuple->getElementNames() != destination_tuple->getElementNames()) + if (!source_tuple || !destination_tuple) return false; - } - else if (source_tuple->getElements().size() != destination_tuple->getElements().size()) - return false; - const auto & source_elements = source_tuple->getElements(); - const auto & destination_elements = destination_tuple->getElements(); - for (size_t i = 0; i < source_elements.size(); ++i) - if (!haveSameTupleElementLayout(source_elements[i], destination_elements[i])) + if (source_tuple->hasExplicitNames() && destination_tuple->hasExplicitNames()) + { + if (source_tuple->getElementNames() != destination_tuple->getElementNames()) + return false; + } + else if (source_tuple->getElements().size() != destination_tuple->getElements().size()) return false; - return true; - } + const auto & source_elements = source_tuple->getElements(); + const auto & destination_elements = destination_tuple->getElements(); + for (size_t i = 0; i < source_elements.size(); ++i) + if (!haveSameTupleElementLayout(source_elements[i], destination_elements[i])) + return false; - const auto * source_array = checkAndGetDataType(source_type_unwrapped.get()); - const auto * destination_array = checkAndGetDataType(destination_type_unwrapped.get()); - if (source_array || destination_array) - { - if (!source_array || !destination_array) - return false; + return true; + } - return haveSameTupleElementLayout(source_array->getNestedType(), destination_array->getNestedType()); - } + const auto * source_array = checkAndGetDataType(source_type_unwrapped.get()); + const auto * destination_array = checkAndGetDataType(destination_type_unwrapped.get()); + if (source_array || destination_array) + { + if (!source_array || !destination_array) + return false; - const auto * source_map = checkAndGetDataType(source_type_unwrapped.get()); - const auto * destination_map = checkAndGetDataType(destination_type_unwrapped.get()); - if (source_map || destination_map) - { - if (!source_map || !destination_map) - return false; + return haveSameTupleElementLayout(source_array->getNestedType(), destination_array->getNestedType()); + } - return haveSameTupleElementLayout(source_map->getKeyType(), destination_map->getKeyType()) - && haveSameTupleElementLayout(source_map->getValueType(), destination_map->getValueType()); - } + const auto * source_map = checkAndGetDataType(source_type_unwrapped.get()); + const auto * destination_map = checkAndGetDataType(destination_type_unwrapped.get()); + if (source_map || destination_map) + { + if (!source_map || !destination_map) + return false; - return true; - } + return haveSameTupleElementLayout(source_map->getKeyType(), destination_map->getKeyType()) + && haveSameTupleElementLayout(source_map->getValueType(), destination_map->getValueType()); + } - void verifyExportColumnCastIsSafe( - const ColumnWithTypeAndName & source_column, - const ColumnWithTypeAndName & destination_column, - const StorageID & destination_storage_id) - { - if (canBeSafelyCast(source_column.type, destination_column.type)) - return; + return true; + } - throw Exception(ErrorCodes::INCOMPATIBLE_COLUMNS, - "Cannot export to {}: column '{}' requires a lossy cast from {} to {}, " - "which may change values. Set `export_merge_tree_part_allow_lossy_cast = 1` " - "to allow lossy casts during export.", - destination_storage_id.getFullTableName(), - destination_column.name, - source_column.type->getName(), - destination_column.type->getName()); - } + void verifyExportColumnCastIsSafe( + const ColumnWithTypeAndName & source_column, + const ColumnWithTypeAndName & destination_column, + const StorageID & destination_storage_id) + { + if (canBeSafelyCast(source_column.type, destination_column.type)) + return; - void verifyPartitionKeyColumn( - const ColumnWithTypeAndName & source_column, - const ColumnWithTypeAndName & destination_column, - size_t position, - const StorageID & destination_storage_id, - bool match_by_name) - { - if (source_column.name != destination_column.name) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Cannot export to {}: partition key column '{}' is at position {} in the source " - "table, but the destination's column at that position is named '{}'. EXPORT " - "PART/PARTITION {} so partition key columns must be declared {} in both tables.", + throw Exception(ErrorCodes::INCOMPATIBLE_COLUMNS, + "Cannot export to {}: column '{}' requires a lossy cast from {} to {}, " + "which may change values. Set `export_merge_tree_part_allow_lossy_cast = 1` " + "to allow lossy casts during export.", destination_storage_id.getFullTableName(), - source_column.name, - position, destination_column.name, - match_by_name ? "matches columns by name" : "matches columns by position", - match_by_name ? "with the same name" : "at the same position"); - - if (!haveSameTupleElementLayout(source_column.type, destination_column.type)) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Cannot export to {}: partition key column '{}' has a different Tuple element " - "layout in the source ({}) and destination ({}). Tuple element names must be " - "declared in the same order in both tables.", - destination_storage_id.getFullTableName(), - source_column.name, source_column.type->getName(), destination_column.type->getName()); + } + + void verifyPartitionKeyColumn( + const ColumnWithTypeAndName & source_column, + const ColumnWithTypeAndName & destination_column, + size_t position, + const StorageID & destination_storage_id, + bool match_by_name) + { + if (source_column.name != destination_column.name) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Cannot export to {}: partition key column '{}' is at position {} in the source " + "table, but the destination's column at that position is named '{}'. EXPORT " + "PART/PARTITION {} so partition key columns must be declared {} in both tables.", + destination_storage_id.getFullTableName(), + source_column.name, + position, + destination_column.name, + match_by_name ? "matches columns by name" : "matches columns by position", + match_by_name ? "with the same name" : "at the same position"); + + if (!haveSameTupleElementLayout(source_column.type, destination_column.type)) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Cannot export to {}: partition key column '{}' has a different Tuple element " + "layout in the source ({}) and destination ({}). Tuple element names must be " + "declared in the same order in both tables.", + destination_storage_id.getFullTableName(), + source_column.name, + source_column.type->getName(), + destination_column.type->getName()); + } } void checkExportSchemaColumnsCount(