From 107eaf181d4aea6749df9a2b4d8aa8211386d89c Mon Sep 17 00:00:00 2001 From: 3pointer Date: Sun, 9 Aug 2026 22:31:03 -0700 Subject: [PATCH 1/5] tiflash: pass through FTS index_fields metadata Signed-off-by: 3pointer --- dbms/src/TiDB/Schema/FullTextIndex.h | 32 +++++ dbms/src/TiDB/Schema/TiDB.cpp | 115 ++++++++++++++++-- .../TiDB/Schema/tests/gtest_table_info.cpp | 95 +++++++++++++++ 3 files changed, 234 insertions(+), 8 deletions(-) diff --git a/dbms/src/TiDB/Schema/FullTextIndex.h b/dbms/src/TiDB/Schema/FullTextIndex.h index 6ef55dec312..24fbfc0bd76 100644 --- a/dbms/src/TiDB/Schema/FullTextIndex.h +++ b/dbms/src/TiDB/Schema/FullTextIndex.h @@ -18,15 +18,47 @@ #include #include +#include +#include namespace TiDB { // Constructed from table definition. // See TiDB's pkg/parser/model/index_full_text.go +struct FullTextNgramOption +{ + UInt64 min_gram = 0; + UInt64 max_gram = 0; + String granularity; + bool lower_case = false; +}; + +struct FullTextPathHierarchyOption +{ + String delimiter; +}; + +struct FullTextIndexFieldOption +{ + String analyzer_type; + bool enable_bm25 = false; + std::optional ngram; + std::optional path_hierarchy; +}; + +struct FullTextIndexFieldInfo +{ + Int64 column_id = 0; + FullTextIndexFieldOption index_field_option; +}; + struct FullTextIndexDefinition { + // Legacy metadata for rolling upgrades of single-field indexes. String parser_type = "INVALID"; + // Array position is the Tantivy physical field order. Keep duplicates. + std::vector index_fields; }; // As this is constructed from TiDB's table definition, we should not diff --git a/dbms/src/TiDB/Schema/TiDB.cpp b/dbms/src/TiDB/Schema/TiDB.cpp index 2afbf157c29..e64969a95a7 100644 --- a/dbms/src/TiDB/Schema/TiDB.cpp +++ b/dbms/src/TiDB/Schema/TiDB.cpp @@ -134,29 +134,128 @@ enum class IndexType }; #if ENABLE_CLARA +namespace +{ +FullTextNgramOption parseFullTextNgramOption(const Poco::JSON::Object::Ptr & json) +{ + return FullTextNgramOption{ + .min_gram = json->getValue("min_gram"), + .max_gram = json->getValue("max_gram"), + .granularity = json->getValue("granularity"), + .lower_case = json->getValue("lower_case"), + }; +} + +FullTextPathHierarchyOption parseFullTextPathHierarchyOption(const Poco::JSON::Object::Ptr & json) +{ + return FullTextPathHierarchyOption{ + .delimiter = json->getValue("delimiter"), + }; +} + +FullTextIndexFieldOption parseFullTextIndexFieldOption(const Poco::JSON::Object::Ptr & json) +{ + FullTextIndexFieldOption option{ + .analyzer_type = json->getValue("analyzer_type"), + .enable_bm25 = json->getValue("enable_bm25"), + }; + if (json->has("ngram")) + { + auto ngram = json->getObject("ngram"); + RUNTIME_CHECK_MSG(ngram, "Invalid FullTextIndex field option, ngram must be an object"); + option.ngram = parseFullTextNgramOption(ngram); + } + if (json->has("path_hierarchy")) + { + auto path_hierarchy = json->getObject("path_hierarchy"); + RUNTIME_CHECK_MSG(path_hierarchy, "Invalid FullTextIndex field option, path_hierarchy must be an object"); + option.path_hierarchy = parseFullTextPathHierarchyOption(path_hierarchy); + } + return option; +} +} // namespace + FullTextIndexDefinitionPtr parseFullTextIndexFromJSON(const Poco::JSON::Object::Ptr & json) { RUNTIME_CHECK(json); // not nullptr - RUNTIME_CHECK_MSG(json->has("parser_type"), "Invalid FullTextIndex definition, missing parser_type"); - auto parser_type_field = json->getValue("parser_type"); - RUNTIME_CHECK_MSG( - ClaraFTS::supports_tokenizer(parser_type_field), - "Invalid FullTextIndex definition, unsupported parser_type `{}`", - parser_type_field); + const auto parser_type = json->has("parser_type") ? json->getValue("parser_type") : String{}; + std::vector index_fields; + if (json->has("index_fields")) + { + auto fields = json->getArray("index_fields"); + RUNTIME_CHECK_MSG(fields, "Invalid FullTextIndex definition, index_fields must be an array"); + index_fields.reserve(fields->size()); + for (size_t i = 0; i < fields->size(); ++i) + { + auto field = fields->getObject(i); + RUNTIME_CHECK_MSG(field, "Invalid FullTextIndex definition, index_fields[{}] must be an object", i); + auto field_option = field->getObject("index_field_option"); + RUNTIME_CHECK_MSG( + field_option, + "Invalid FullTextIndex definition, index_fields[{}].index_field_option must be an object", + i); + index_fields.emplace_back(FullTextIndexFieldInfo{ + .column_id = field->getValue("column_id"), + .index_field_option = parseFullTextIndexFieldOption(field_option), + }); + } + } + + // CSE owns Spec v1 semantic validation. TiFlash only validates the legacy + // parser that its local single-field implementation consumes directly. + if (index_fields.empty()) + { + RUNTIME_CHECK_MSG( + ClaraFTS::supports_tokenizer(parser_type), + "Invalid legacy FullTextIndex definition, unsupported parser_type `{}`", + parser_type); + } return std::make_shared(FullTextIndexDefinition{ - .parser_type = parser_type_field, + .parser_type = parser_type, + .index_fields = std::move(index_fields), }); } Poco::JSON::Object::Ptr fullTextIndexToJSON(const FullTextIndexDefinitionPtr & full_text_index) { RUNTIME_CHECK(full_text_index != nullptr); - RUNTIME_CHECK(ClaraFTS::supports_tokenizer(full_text_index->parser_type)); Poco::JSON::Object::Ptr json = new Poco::JSON::Object(); json->set("parser_type", full_text_index->parser_type); + if (!full_text_index->index_fields.empty()) + { + Poco::JSON::Array::Ptr fields = new Poco::JSON::Array(); + for (const auto & field : full_text_index->index_fields) + { + Poco::JSON::Object::Ptr field_json = new Poco::JSON::Object(); + field_json->set("column_id", field.column_id); + + const auto & option = field.index_field_option; + Poco::JSON::Object::Ptr option_json = new Poco::JSON::Object(); + option_json->set("analyzer_type", option.analyzer_type); + option_json->set("enable_bm25", option.enable_bm25); + if (option.ngram) + { + Poco::JSON::Object::Ptr ngram = new Poco::JSON::Object(); + ngram->set("min_gram", option.ngram->min_gram); + ngram->set("max_gram", option.ngram->max_gram); + ngram->set("granularity", option.ngram->granularity); + ngram->set("lower_case", option.ngram->lower_case); + option_json->set("ngram", ngram); + } + if (option.path_hierarchy) + { + Poco::JSON::Object::Ptr path_hierarchy = new Poco::JSON::Object(); + path_hierarchy->set("delimiter", option.path_hierarchy->delimiter); + option_json->set("path_hierarchy", path_hierarchy); + } + field_json->set("index_field_option", option_json); + fields->add(field_json); + } + json->set("index_fields", fields); + } return json; } #endif diff --git a/dbms/src/TiDB/Schema/tests/gtest_table_info.cpp b/dbms/src/TiDB/Schema/tests/gtest_table_info.cpp index f14c9c88cbc..c75c2165e46 100644 --- a/dbms/src/TiDB/Schema/tests/gtest_table_info.cpp +++ b/dbms/src/TiDB/Schema/tests/gtest_table_info.cpp @@ -31,6 +31,7 @@ #include #include +#include using TableInfo = TiDB::TableInfo; using DBInfo = TiDB::DBInfo; @@ -293,6 +294,100 @@ try } CATCH +String tableInfoWithFullTextIndex(std::string_view full_text_index) +{ + String json + = R"json({"cols":[{"id":101,"name":{"L":"title","O":"title"},"offset":0,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":65535,"Tp":252}},{"id":202,"name":{"L":"category","O":"category"},"offset":1,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":255,"Tp":15}},{"id":203,"name":{"L":"path","O":"path"},"offset":2,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":1024,"Tp":15}},{"id":204,"name":{"L":"body","O":"body"},"offset":3,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":65535,"Tp":252}}],"id":30,"index_info":[{"id":5,"idx_cols":[{"length":-1,"name":{"L":"title","O":"title"},"offset":0},{"length":-1,"name":{"L":"category","O":"category"},"offset":1},{"length":-1,"name":{"L":"path","O":"path"},"offset":2},{"length":-1,"name":{"L":"body","O":"body"},"offset":3}],"idx_name":{"L":"ft_docs","O":"ft_docs"},"index_type":3,"is_global":false,"is_invisible":false,"is_primary":false,"is_unique":false,"state":5,"full_text_index":)json"; + json.append(full_text_index); + json + += R"json(}],"is_common_handle":false,"name":{"L":"docs","O":"docs"},"partition":null,"pk_is_handle":false,"schema_version":-1,"state":5,"update_timestamp":1723778704444603})json"; + return json; +} + +void assertFullTextIndexFields(const TiDB::FullTextIndexDefinition & full_text_index) +{ + ASSERT_EQ(full_text_index.parser_type, ""); + ASSERT_EQ(full_text_index.index_fields.size(), 5); + + const auto & path_field = full_text_index.index_fields[0]; + ASSERT_EQ(path_field.column_id, 203); + ASSERT_EQ(path_field.index_field_option.analyzer_type, "PATH_HIERARCHY_V1"); + ASSERT_FALSE(path_field.index_field_option.enable_bm25); + ASSERT_TRUE(path_field.index_field_option.path_hierarchy.has_value()); + ASSERT_EQ(path_field.index_field_option.path_hierarchy->delimiter, "::"); + + const auto & standard_field = full_text_index.index_fields[1]; + ASSERT_EQ(standard_field.column_id, 101); + ASSERT_EQ(standard_field.index_field_option.analyzer_type, "STANDARD_V1"); + ASSERT_TRUE(standard_field.index_field_option.enable_bm25); + + const auto & ngram_field = full_text_index.index_fields[2]; + ASSERT_EQ(ngram_field.column_id, 101); + ASSERT_EQ(ngram_field.index_field_option.analyzer_type, "NGRAM_V1"); + ASSERT_TRUE(ngram_field.index_field_option.enable_bm25); + ASSERT_TRUE(ngram_field.index_field_option.ngram.has_value()); + ASSERT_EQ(ngram_field.index_field_option.ngram->min_gram, 2); + ASSERT_EQ(ngram_field.index_field_option.ngram->max_gram, 5); + ASSERT_EQ(ngram_field.index_field_option.ngram->granularity, "CHAR"); + ASSERT_TRUE(ngram_field.index_field_option.ngram->lower_case); + + const auto & multilingual_field = full_text_index.index_fields[3]; + ASSERT_EQ(multilingual_field.column_id, 204); + ASSERT_EQ(multilingual_field.index_field_option.analyzer_type, "MULTILINGUAL_V1"); + ASSERT_FALSE(multilingual_field.index_field_option.enable_bm25); + + const auto & exact_field = full_text_index.index_fields[4]; + ASSERT_EQ(exact_field.column_id, 202); + ASSERT_EQ(exact_field.index_field_option.analyzer_type, "EXACT_VALUE_V1"); + ASSERT_FALSE(exact_field.index_field_option.enable_bm25); +} + +TEST(TiDBTableInfoTest, FullTextIndexFieldsJSONRoundTrip) +try +{ + const auto table_info_json = tableInfoWithFullTextIndex(R"json({ + "parser_type":"", + "index_fields":[ + {"column_id":203,"index_field_option":{"analyzer_type":"PATH_HIERARCHY_V1","enable_bm25":false,"path_hierarchy":{"delimiter":"::"}}}, + {"column_id":101,"index_field_option":{"analyzer_type":"STANDARD_V1","enable_bm25":true}}, + {"column_id":101,"index_field_option":{"analyzer_type":"NGRAM_V1","enable_bm25":true,"ngram":{"min_gram":2,"max_gram":5,"granularity":"CHAR","lower_case":true}}}, + {"column_id":204,"index_field_option":{"analyzer_type":"MULTILINGUAL_V1","enable_bm25":false}}, + {"column_id":202,"index_field_option":{"analyzer_type":"EXACT_VALUE_V1","enable_bm25":false}} + ] + })json"); + TableInfo table_info(table_info_json, NullspaceID); + + ASSERT_EQ(table_info.index_infos.size(), 1); + ASSERT_NE(table_info.index_infos[0].full_text_index, nullptr); + assertFullTextIndexFields(*table_info.index_infos[0].full_text_index); + + const auto serialized = table_info.serialize(); + ASSERT_NE(serialized.find("\"index_fields\""), String::npos); + ASSERT_EQ(serialized.find("\"fields\""), String::npos); + ASSERT_EQ(serialized.find("\"filter_columns\""), String::npos); + + const TableInfo round_trip(serialized, NullspaceID); + ASSERT_NE(round_trip.index_infos[0].full_text_index, nullptr); + assertFullTextIndexFields(*round_trip.index_infos[0].full_text_index); + ASSERT_EQ(round_trip.serialize(), serialized); +} +CATCH + +TEST(TiDBTableInfoTest, LegacyFullTextParserTypeJSONRoundTrip) +try +{ + const TableInfo table_info(tableInfoWithFullTextIndex(R"json({"parser_type":"STANDARD_V1"})json"), NullspaceID); + + ASSERT_NE(table_info.index_infos[0].full_text_index, nullptr); + ASSERT_EQ(table_info.index_infos[0].full_text_index->parser_type, "STANDARD_V1"); + ASSERT_TRUE(table_info.index_infos[0].full_text_index->index_fields.empty()); + + const TableInfo round_trip(table_info.serialize(), NullspaceID); + ASSERT_EQ(round_trip.index_infos[0].full_text_index->parser_type, "STANDARD_V1"); + ASSERT_TRUE(round_trip.index_infos[0].full_text_index->index_fields.empty()); +} +CATCH + struct StmtCase { TableID table_or_partition_id; From 200f24886f07f322b5780bdcace3bc97ab7e41e6 Mon Sep 17 00:00:00 2001 From: 3pointer Date: Sun, 9 Aug 2026 22:31:13 -0700 Subject: [PATCH 2/5] columnar: pin CSE fts_spec_v1 for draft validation Signed-off-by: 3pointer --- contrib/cloud-storage-engine | 2 +- contrib/tiflash-columnar-hub/Cargo.lock | 55 ++++++++++++++++--------- 2 files changed, 37 insertions(+), 20 deletions(-) diff --git a/contrib/cloud-storage-engine b/contrib/cloud-storage-engine index b49d94ae594..e710be838c7 160000 --- a/contrib/cloud-storage-engine +++ b/contrib/cloud-storage-engine @@ -1 +1 @@ -Subproject commit b49d94ae594914225390e3914f92025f1486d95b +Subproject commit e710be838c77ceb9972d6069f55006b66c4eb552 diff --git a/contrib/tiflash-columnar-hub/Cargo.lock b/contrib/tiflash-columnar-hub/Cargo.lock index 94e8c7f1a3b..6586d68c77f 100644 --- a/contrib/tiflash-columnar-hub/Cargo.lock +++ b/contrib/tiflash-columnar-hub/Cargo.lock @@ -178,7 +178,7 @@ dependencies = [ "bytes", "codec", "engine_traits", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "match-template", "thiserror 1.0.69", "tikv_alloc", @@ -386,7 +386,7 @@ dependencies = [ "http 0.2.12", "hyper 0.14.26", "hyper-tls", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "lazy_static", "md5", "prometheus", @@ -1156,7 +1156,7 @@ dependencies = [ "http 0.2.12", "hyper 0.14.26", "kvengine", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "pd_client", "rand 0.8.6", "serde", @@ -1467,17 +1467,22 @@ name = "clara_fts" version = "0.0.1" dependencies = [ "anyhow", + "base64 0.13.1", "charabia", "fastrand 2.4.1", + "kvenginepb", "lazy_static", "memmap2 0.9.10", "once_cell", "ordered-float 5.3.0", "prost 0.13.5", "prost-build", + "serde", + "serde_json", "stable_deref_trait", "tantivy", "tempfile", + "tidb_query_datatype", "tipb", "workspace-hack", ] @@ -1490,7 +1495,7 @@ dependencies = [ "derive_more", "error_code", "futures-io", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "lazy_static", "openssl", "prometheus", @@ -1515,7 +1520,7 @@ dependencies = [ "fxhash", "hmac 0.10.1", "kvenginepb", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "lazy_static", "log", "openssl", @@ -2251,7 +2256,7 @@ dependencies = [ "futures 0.3.32", "futures-util", "hex 0.4.3", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "lazy_static", "online_config", "openssl", @@ -2278,7 +2283,7 @@ dependencies = [ "error_code", "fail 0.5.1", "file_system", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "log_wrappers", "protobuf 2.8.0", "quick_cache", @@ -2376,7 +2381,7 @@ name = "error_code" version = "0.0.1" dependencies = [ "grpcio", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "lazy_static", "raft", "serde", @@ -3677,7 +3682,7 @@ name = "keys" version = "0.1.0" dependencies = [ "byteorder", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "log_wrappers", "thiserror 1.0.69", "tikv_alloc", @@ -3736,7 +3741,7 @@ dependencies = [ "itertools 0.10.5", "keys", "kvenginepb", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "lazy_static", "libc", "log_wrappers", @@ -3813,6 +3818,19 @@ dependencies = [ "raft-proto", ] +[[package]] +name = "kvproto" +version = "0.0.2" +source = "git+https://github.com/pingcap/kvproto.git#059694ae4472276644613acccefa24cbc89d959f" +dependencies = [ + "bytes", + "futures 0.3.32", + "grpcio", + "protobuf 2.8.0", + "protobuf-build", + "raft-proto", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -4826,7 +4844,7 @@ dependencies = [ "grpcio", "hex 0.4.3", "http 0.2.12", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "lazy_static", "log", "log_wrappers", @@ -6073,7 +6091,7 @@ dependencies = [ "hyper 1.10.1", "hyper-rustls", "hyper-util", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "lazy_static", "prometheus", "rustls", @@ -7124,7 +7142,7 @@ dependencies = [ "derive_more", "error_code", "futures 0.3.32", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "lazy_static", "log_wrappers", "maybe-async", @@ -7158,7 +7176,7 @@ dependencies = [ "encoding_rs 0.8.29", "error_code", "hex 0.4.3", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "lazy_static", "log_wrappers", "match-template", @@ -7202,7 +7220,7 @@ dependencies = [ "keys", "kvengine", "kvenginepb", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git?rev=683dad8fa3689deb243f2ff8ab5847c97af53e38)", "lazy_static", "num_cpus", "pd_client", @@ -7319,7 +7337,7 @@ dependencies = [ "grpcio", "http 0.2.12", "hyper 0.14.26", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "lazy_static", "libc", "log", @@ -7748,7 +7766,7 @@ version = "0.0.1" dependencies = [ "collections", "crossbeam-utils", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "lazy_static", "parking_lot 0.12.5", "pin-project", @@ -7775,7 +7793,7 @@ dependencies = [ "collections", "error_code", "farmhash", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "log_wrappers", "slog", "thiserror 1.0.69", @@ -8598,7 +8616,6 @@ dependencies = [ "proc-macro2", "prost 0.11.9", "protobuf 2.8.0", - "protobuf-build", "rand 0.8.6", "rand_core 0.6.4", "regex", From ada1c780b490c251caa80c822102cc928af0546a Mon Sep 17 00:00:00 2001 From: 3pointer Date: Sun, 9 Aug 2026 22:41:58 -0700 Subject: [PATCH 3/5] tiflash: leave FTS spec validation to CSE Signed-off-by: 3pointer --- dbms/src/TiDB/Schema/FullTextIndex.h | 32 ----- dbms/src/TiDB/Schema/TiDB.cpp | 115 ++---------------- .../TiDB/Schema/tests/gtest_table_info.cpp | 95 --------------- 3 files changed, 8 insertions(+), 234 deletions(-) diff --git a/dbms/src/TiDB/Schema/FullTextIndex.h b/dbms/src/TiDB/Schema/FullTextIndex.h index 24fbfc0bd76..6ef55dec312 100644 --- a/dbms/src/TiDB/Schema/FullTextIndex.h +++ b/dbms/src/TiDB/Schema/FullTextIndex.h @@ -18,47 +18,15 @@ #include #include -#include -#include namespace TiDB { // Constructed from table definition. // See TiDB's pkg/parser/model/index_full_text.go -struct FullTextNgramOption -{ - UInt64 min_gram = 0; - UInt64 max_gram = 0; - String granularity; - bool lower_case = false; -}; - -struct FullTextPathHierarchyOption -{ - String delimiter; -}; - -struct FullTextIndexFieldOption -{ - String analyzer_type; - bool enable_bm25 = false; - std::optional ngram; - std::optional path_hierarchy; -}; - -struct FullTextIndexFieldInfo -{ - Int64 column_id = 0; - FullTextIndexFieldOption index_field_option; -}; - struct FullTextIndexDefinition { - // Legacy metadata for rolling upgrades of single-field indexes. String parser_type = "INVALID"; - // Array position is the Tantivy physical field order. Keep duplicates. - std::vector index_fields; }; // As this is constructed from TiDB's table definition, we should not diff --git a/dbms/src/TiDB/Schema/TiDB.cpp b/dbms/src/TiDB/Schema/TiDB.cpp index e64969a95a7..2afbf157c29 100644 --- a/dbms/src/TiDB/Schema/TiDB.cpp +++ b/dbms/src/TiDB/Schema/TiDB.cpp @@ -134,128 +134,29 @@ enum class IndexType }; #if ENABLE_CLARA -namespace -{ -FullTextNgramOption parseFullTextNgramOption(const Poco::JSON::Object::Ptr & json) -{ - return FullTextNgramOption{ - .min_gram = json->getValue("min_gram"), - .max_gram = json->getValue("max_gram"), - .granularity = json->getValue("granularity"), - .lower_case = json->getValue("lower_case"), - }; -} - -FullTextPathHierarchyOption parseFullTextPathHierarchyOption(const Poco::JSON::Object::Ptr & json) -{ - return FullTextPathHierarchyOption{ - .delimiter = json->getValue("delimiter"), - }; -} - -FullTextIndexFieldOption parseFullTextIndexFieldOption(const Poco::JSON::Object::Ptr & json) -{ - FullTextIndexFieldOption option{ - .analyzer_type = json->getValue("analyzer_type"), - .enable_bm25 = json->getValue("enable_bm25"), - }; - if (json->has("ngram")) - { - auto ngram = json->getObject("ngram"); - RUNTIME_CHECK_MSG(ngram, "Invalid FullTextIndex field option, ngram must be an object"); - option.ngram = parseFullTextNgramOption(ngram); - } - if (json->has("path_hierarchy")) - { - auto path_hierarchy = json->getObject("path_hierarchy"); - RUNTIME_CHECK_MSG(path_hierarchy, "Invalid FullTextIndex field option, path_hierarchy must be an object"); - option.path_hierarchy = parseFullTextPathHierarchyOption(path_hierarchy); - } - return option; -} -} // namespace - FullTextIndexDefinitionPtr parseFullTextIndexFromJSON(const Poco::JSON::Object::Ptr & json) { RUNTIME_CHECK(json); // not nullptr - const auto parser_type = json->has("parser_type") ? json->getValue("parser_type") : String{}; - std::vector index_fields; - if (json->has("index_fields")) - { - auto fields = json->getArray("index_fields"); - RUNTIME_CHECK_MSG(fields, "Invalid FullTextIndex definition, index_fields must be an array"); - index_fields.reserve(fields->size()); - for (size_t i = 0; i < fields->size(); ++i) - { - auto field = fields->getObject(i); - RUNTIME_CHECK_MSG(field, "Invalid FullTextIndex definition, index_fields[{}] must be an object", i); - auto field_option = field->getObject("index_field_option"); - RUNTIME_CHECK_MSG( - field_option, - "Invalid FullTextIndex definition, index_fields[{}].index_field_option must be an object", - i); - index_fields.emplace_back(FullTextIndexFieldInfo{ - .column_id = field->getValue("column_id"), - .index_field_option = parseFullTextIndexFieldOption(field_option), - }); - } - } - - // CSE owns Spec v1 semantic validation. TiFlash only validates the legacy - // parser that its local single-field implementation consumes directly. - if (index_fields.empty()) - { - RUNTIME_CHECK_MSG( - ClaraFTS::supports_tokenizer(parser_type), - "Invalid legacy FullTextIndex definition, unsupported parser_type `{}`", - parser_type); - } + RUNTIME_CHECK_MSG(json->has("parser_type"), "Invalid FullTextIndex definition, missing parser_type"); + auto parser_type_field = json->getValue("parser_type"); + RUNTIME_CHECK_MSG( + ClaraFTS::supports_tokenizer(parser_type_field), + "Invalid FullTextIndex definition, unsupported parser_type `{}`", + parser_type_field); return std::make_shared(FullTextIndexDefinition{ - .parser_type = parser_type, - .index_fields = std::move(index_fields), + .parser_type = parser_type_field, }); } Poco::JSON::Object::Ptr fullTextIndexToJSON(const FullTextIndexDefinitionPtr & full_text_index) { RUNTIME_CHECK(full_text_index != nullptr); + RUNTIME_CHECK(ClaraFTS::supports_tokenizer(full_text_index->parser_type)); Poco::JSON::Object::Ptr json = new Poco::JSON::Object(); json->set("parser_type", full_text_index->parser_type); - if (!full_text_index->index_fields.empty()) - { - Poco::JSON::Array::Ptr fields = new Poco::JSON::Array(); - for (const auto & field : full_text_index->index_fields) - { - Poco::JSON::Object::Ptr field_json = new Poco::JSON::Object(); - field_json->set("column_id", field.column_id); - - const auto & option = field.index_field_option; - Poco::JSON::Object::Ptr option_json = new Poco::JSON::Object(); - option_json->set("analyzer_type", option.analyzer_type); - option_json->set("enable_bm25", option.enable_bm25); - if (option.ngram) - { - Poco::JSON::Object::Ptr ngram = new Poco::JSON::Object(); - ngram->set("min_gram", option.ngram->min_gram); - ngram->set("max_gram", option.ngram->max_gram); - ngram->set("granularity", option.ngram->granularity); - ngram->set("lower_case", option.ngram->lower_case); - option_json->set("ngram", ngram); - } - if (option.path_hierarchy) - { - Poco::JSON::Object::Ptr path_hierarchy = new Poco::JSON::Object(); - path_hierarchy->set("delimiter", option.path_hierarchy->delimiter); - option_json->set("path_hierarchy", path_hierarchy); - } - field_json->set("index_field_option", option_json); - fields->add(field_json); - } - json->set("index_fields", fields); - } return json; } #endif diff --git a/dbms/src/TiDB/Schema/tests/gtest_table_info.cpp b/dbms/src/TiDB/Schema/tests/gtest_table_info.cpp index c75c2165e46..f14c9c88cbc 100644 --- a/dbms/src/TiDB/Schema/tests/gtest_table_info.cpp +++ b/dbms/src/TiDB/Schema/tests/gtest_table_info.cpp @@ -31,7 +31,6 @@ #include #include -#include using TableInfo = TiDB::TableInfo; using DBInfo = TiDB::DBInfo; @@ -294,100 +293,6 @@ try } CATCH -String tableInfoWithFullTextIndex(std::string_view full_text_index) -{ - String json - = R"json({"cols":[{"id":101,"name":{"L":"title","O":"title"},"offset":0,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":65535,"Tp":252}},{"id":202,"name":{"L":"category","O":"category"},"offset":1,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":255,"Tp":15}},{"id":203,"name":{"L":"path","O":"path"},"offset":2,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":1024,"Tp":15}},{"id":204,"name":{"L":"body","O":"body"},"offset":3,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":65535,"Tp":252}}],"id":30,"index_info":[{"id":5,"idx_cols":[{"length":-1,"name":{"L":"title","O":"title"},"offset":0},{"length":-1,"name":{"L":"category","O":"category"},"offset":1},{"length":-1,"name":{"L":"path","O":"path"},"offset":2},{"length":-1,"name":{"L":"body","O":"body"},"offset":3}],"idx_name":{"L":"ft_docs","O":"ft_docs"},"index_type":3,"is_global":false,"is_invisible":false,"is_primary":false,"is_unique":false,"state":5,"full_text_index":)json"; - json.append(full_text_index); - json - += R"json(}],"is_common_handle":false,"name":{"L":"docs","O":"docs"},"partition":null,"pk_is_handle":false,"schema_version":-1,"state":5,"update_timestamp":1723778704444603})json"; - return json; -} - -void assertFullTextIndexFields(const TiDB::FullTextIndexDefinition & full_text_index) -{ - ASSERT_EQ(full_text_index.parser_type, ""); - ASSERT_EQ(full_text_index.index_fields.size(), 5); - - const auto & path_field = full_text_index.index_fields[0]; - ASSERT_EQ(path_field.column_id, 203); - ASSERT_EQ(path_field.index_field_option.analyzer_type, "PATH_HIERARCHY_V1"); - ASSERT_FALSE(path_field.index_field_option.enable_bm25); - ASSERT_TRUE(path_field.index_field_option.path_hierarchy.has_value()); - ASSERT_EQ(path_field.index_field_option.path_hierarchy->delimiter, "::"); - - const auto & standard_field = full_text_index.index_fields[1]; - ASSERT_EQ(standard_field.column_id, 101); - ASSERT_EQ(standard_field.index_field_option.analyzer_type, "STANDARD_V1"); - ASSERT_TRUE(standard_field.index_field_option.enable_bm25); - - const auto & ngram_field = full_text_index.index_fields[2]; - ASSERT_EQ(ngram_field.column_id, 101); - ASSERT_EQ(ngram_field.index_field_option.analyzer_type, "NGRAM_V1"); - ASSERT_TRUE(ngram_field.index_field_option.enable_bm25); - ASSERT_TRUE(ngram_field.index_field_option.ngram.has_value()); - ASSERT_EQ(ngram_field.index_field_option.ngram->min_gram, 2); - ASSERT_EQ(ngram_field.index_field_option.ngram->max_gram, 5); - ASSERT_EQ(ngram_field.index_field_option.ngram->granularity, "CHAR"); - ASSERT_TRUE(ngram_field.index_field_option.ngram->lower_case); - - const auto & multilingual_field = full_text_index.index_fields[3]; - ASSERT_EQ(multilingual_field.column_id, 204); - ASSERT_EQ(multilingual_field.index_field_option.analyzer_type, "MULTILINGUAL_V1"); - ASSERT_FALSE(multilingual_field.index_field_option.enable_bm25); - - const auto & exact_field = full_text_index.index_fields[4]; - ASSERT_EQ(exact_field.column_id, 202); - ASSERT_EQ(exact_field.index_field_option.analyzer_type, "EXACT_VALUE_V1"); - ASSERT_FALSE(exact_field.index_field_option.enable_bm25); -} - -TEST(TiDBTableInfoTest, FullTextIndexFieldsJSONRoundTrip) -try -{ - const auto table_info_json = tableInfoWithFullTextIndex(R"json({ - "parser_type":"", - "index_fields":[ - {"column_id":203,"index_field_option":{"analyzer_type":"PATH_HIERARCHY_V1","enable_bm25":false,"path_hierarchy":{"delimiter":"::"}}}, - {"column_id":101,"index_field_option":{"analyzer_type":"STANDARD_V1","enable_bm25":true}}, - {"column_id":101,"index_field_option":{"analyzer_type":"NGRAM_V1","enable_bm25":true,"ngram":{"min_gram":2,"max_gram":5,"granularity":"CHAR","lower_case":true}}}, - {"column_id":204,"index_field_option":{"analyzer_type":"MULTILINGUAL_V1","enable_bm25":false}}, - {"column_id":202,"index_field_option":{"analyzer_type":"EXACT_VALUE_V1","enable_bm25":false}} - ] - })json"); - TableInfo table_info(table_info_json, NullspaceID); - - ASSERT_EQ(table_info.index_infos.size(), 1); - ASSERT_NE(table_info.index_infos[0].full_text_index, nullptr); - assertFullTextIndexFields(*table_info.index_infos[0].full_text_index); - - const auto serialized = table_info.serialize(); - ASSERT_NE(serialized.find("\"index_fields\""), String::npos); - ASSERT_EQ(serialized.find("\"fields\""), String::npos); - ASSERT_EQ(serialized.find("\"filter_columns\""), String::npos); - - const TableInfo round_trip(serialized, NullspaceID); - ASSERT_NE(round_trip.index_infos[0].full_text_index, nullptr); - assertFullTextIndexFields(*round_trip.index_infos[0].full_text_index); - ASSERT_EQ(round_trip.serialize(), serialized); -} -CATCH - -TEST(TiDBTableInfoTest, LegacyFullTextParserTypeJSONRoundTrip) -try -{ - const TableInfo table_info(tableInfoWithFullTextIndex(R"json({"parser_type":"STANDARD_V1"})json"), NullspaceID); - - ASSERT_NE(table_info.index_infos[0].full_text_index, nullptr); - ASSERT_EQ(table_info.index_infos[0].full_text_index->parser_type, "STANDARD_V1"); - ASSERT_TRUE(table_info.index_infos[0].full_text_index->index_fields.empty()); - - const TableInfo round_trip(table_info.serialize(), NullspaceID); - ASSERT_EQ(round_trip.index_infos[0].full_text_index->parser_type, "STANDARD_V1"); - ASSERT_TRUE(round_trip.index_infos[0].full_text_index->index_fields.empty()); -} -CATCH - struct StmtCase { TableID table_or_partition_id; From 95866ac0fccbe353862d70cf9c855ad764a11f43 Mon Sep 17 00:00:00 2001 From: 3pointer Date: Sun, 9 Aug 2026 23:34:59 -0700 Subject: [PATCH 4/5] columnar: pin CSE FTS Spec v1 on cloud-engine Signed-off-by: 3pointer --- contrib/cloud-storage-engine | 2 +- contrib/tiflash-columnar-hub/Cargo.lock | 73 +++++++++++-------------- contrib/tiflash-columnar-hub/Cargo.toml | 2 +- 3 files changed, 35 insertions(+), 42 deletions(-) diff --git a/contrib/cloud-storage-engine b/contrib/cloud-storage-engine index e710be838c7..48fd6016f89 160000 --- a/contrib/cloud-storage-engine +++ b/contrib/cloud-storage-engine @@ -1 +1 @@ -Subproject commit e710be838c77ceb9972d6069f55006b66c4eb552 +Subproject commit 48fd6016f89546623b81fd7d293dbbcf5d80cc5a diff --git a/contrib/tiflash-columnar-hub/Cargo.lock b/contrib/tiflash-columnar-hub/Cargo.lock index 6586d68c77f..15b56305990 100644 --- a/contrib/tiflash-columnar-hub/Cargo.lock +++ b/contrib/tiflash-columnar-hub/Cargo.lock @@ -178,7 +178,7 @@ dependencies = [ "bytes", "codec", "engine_traits", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "match-template", "thiserror 1.0.69", "tikv_alloc", @@ -386,7 +386,7 @@ dependencies = [ "http 0.2.12", "hyper 0.14.26", "hyper-tls", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "lazy_static", "md5", "prometheus", @@ -1156,7 +1156,7 @@ dependencies = [ "http 0.2.12", "hyper 0.14.26", "kvengine", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "pd_client", "rand 0.8.6", "serde", @@ -1495,7 +1495,7 @@ dependencies = [ "derive_more", "error_code", "futures-io", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "lazy_static", "openssl", "prometheus", @@ -1520,7 +1520,7 @@ dependencies = [ "fxhash", "hmac 0.10.1", "kvenginepb", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "lazy_static", "log", "openssl", @@ -2256,7 +2256,7 @@ dependencies = [ "futures 0.3.32", "futures-util", "hex 0.4.3", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "lazy_static", "online_config", "openssl", @@ -2283,7 +2283,7 @@ dependencies = [ "error_code", "fail 0.5.1", "file_system", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "log_wrappers", "protobuf 2.8.0", "quick_cache", @@ -2381,7 +2381,7 @@ name = "error_code" version = "0.0.1" dependencies = [ "grpcio", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "lazy_static", "raft", "serde", @@ -3682,7 +3682,7 @@ name = "keys" version = "0.1.0" dependencies = [ "byteorder", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "log_wrappers", "thiserror 1.0.69", "tikv_alloc", @@ -3698,6 +3698,7 @@ dependencies = [ "aliyun", "anyhow", "api_version", + "arc-swap 1.9.1", "arrow-buffer", "async-once-cell", "async-recursion 1.1.1", @@ -3723,6 +3724,7 @@ dependencies = [ "crc32fast", "crossbeam", "crossbeam-queue", + "crossbeam-skiplist", "dashmap 5.5.3", "dyn-clone", "engine_traits", @@ -3741,7 +3743,7 @@ dependencies = [ "itertools 0.10.5", "keys", "kvenginepb", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "lazy_static", "libc", "log_wrappers", @@ -3768,6 +3770,7 @@ dependencies = [ "rusoto_s3", "schema", "security", + "seize", "serde", "serde_derive", "serde_json", @@ -3818,19 +3821,6 @@ dependencies = [ "raft-proto", ] -[[package]] -name = "kvproto" -version = "0.0.2" -source = "git+https://github.com/pingcap/kvproto.git#059694ae4472276644613acccefa24cbc89d959f" -dependencies = [ - "bytes", - "futures 0.3.32", - "grpcio", - "protobuf 2.8.0", - "protobuf-build", - "raft-proto", -] - [[package]] name = "lazy_static" version = "1.5.0" @@ -4734,7 +4724,7 @@ checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "ownedbytes" version = "0.7.0" -source = "git+https://github.com/breezewish/tikv-tantivy.git?branch=patch-0.22.1#efa7643d27289c1ed680fe7ead8c8fffc72b20a2" +source = "git+https://github.com/pingcap/tantivy.git?branch=cse-0.22.1#392142b1eefa9258731b1c8483632c4a70bc32d8" dependencies = [ "stable_deref_trait", ] @@ -4844,7 +4834,7 @@ dependencies = [ "grpcio", "hex 0.4.3", "http 0.2.12", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "lazy_static", "log", "log_wrappers", @@ -6091,7 +6081,7 @@ dependencies = [ "hyper 1.10.1", "hyper-rustls", "hyper-util", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "lazy_static", "prometheus", "rustls", @@ -6874,7 +6864,7 @@ checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" [[package]] name = "tantivy" version = "0.22.1" -source = "git+https://github.com/breezewish/tikv-tantivy.git?branch=patch-0.22.1#efa7643d27289c1ed680fe7ead8c8fffc72b20a2" +source = "git+https://github.com/pingcap/tantivy.git?branch=cse-0.22.1#392142b1eefa9258731b1c8483632c4a70bc32d8" dependencies = [ "aho-corasick", "arc-swap 1.9.1", @@ -6924,7 +6914,7 @@ dependencies = [ [[package]] name = "tantivy-bitpacker" version = "0.6.0" -source = "git+https://github.com/breezewish/tikv-tantivy.git?branch=patch-0.22.1#efa7643d27289c1ed680fe7ead8c8fffc72b20a2" +source = "git+https://github.com/pingcap/tantivy.git?branch=cse-0.22.1#392142b1eefa9258731b1c8483632c4a70bc32d8" dependencies = [ "bitpacking", ] @@ -6932,7 +6922,7 @@ dependencies = [ [[package]] name = "tantivy-columnar" version = "0.3.0" -source = "git+https://github.com/breezewish/tikv-tantivy.git?branch=patch-0.22.1#efa7643d27289c1ed680fe7ead8c8fffc72b20a2" +source = "git+https://github.com/pingcap/tantivy.git?branch=cse-0.22.1#392142b1eefa9258731b1c8483632c4a70bc32d8" dependencies = [ "downcast-rs", "fastdivide", @@ -6947,7 +6937,7 @@ dependencies = [ [[package]] name = "tantivy-common" version = "0.7.0" -source = "git+https://github.com/breezewish/tikv-tantivy.git?branch=patch-0.22.1#efa7643d27289c1ed680fe7ead8c8fffc72b20a2" +source = "git+https://github.com/pingcap/tantivy.git?branch=cse-0.22.1#392142b1eefa9258731b1c8483632c4a70bc32d8" dependencies = [ "async-trait", "byteorder", @@ -6970,7 +6960,7 @@ dependencies = [ [[package]] name = "tantivy-query-grammar" version = "0.22.0" -source = "git+https://github.com/breezewish/tikv-tantivy.git?branch=patch-0.22.1#efa7643d27289c1ed680fe7ead8c8fffc72b20a2" +source = "git+https://github.com/pingcap/tantivy.git?branch=cse-0.22.1#392142b1eefa9258731b1c8483632c4a70bc32d8" dependencies = [ "nom 7.1.3", ] @@ -6978,7 +6968,7 @@ dependencies = [ [[package]] name = "tantivy-sstable" version = "0.3.0" -source = "git+https://github.com/breezewish/tikv-tantivy.git?branch=patch-0.22.1#efa7643d27289c1ed680fe7ead8c8fffc72b20a2" +source = "git+https://github.com/pingcap/tantivy.git?branch=cse-0.22.1#392142b1eefa9258731b1c8483632c4a70bc32d8" dependencies = [ "tantivy-bitpacker", "tantivy-common", @@ -6989,7 +6979,7 @@ dependencies = [ [[package]] name = "tantivy-stacker" version = "0.3.0" -source = "git+https://github.com/breezewish/tikv-tantivy.git?branch=patch-0.22.1#efa7643d27289c1ed680fe7ead8c8fffc72b20a2" +source = "git+https://github.com/pingcap/tantivy.git?branch=cse-0.22.1#392142b1eefa9258731b1c8483632c4a70bc32d8" dependencies = [ "murmurhash32", "rand_distr", @@ -6999,7 +6989,7 @@ dependencies = [ [[package]] name = "tantivy-tokenizer-api" version = "0.3.0" -source = "git+https://github.com/breezewish/tikv-tantivy.git?branch=patch-0.22.1#efa7643d27289c1ed680fe7ead8c8fffc72b20a2" +source = "git+https://github.com/pingcap/tantivy.git?branch=cse-0.22.1#392142b1eefa9258731b1c8483632c4a70bc32d8" dependencies = [ "serde", ] @@ -7046,7 +7036,10 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.9", + "slog", + "slog-global", "thiserror 1.0.69", + "tokio", "workspace-hack", ] @@ -7142,7 +7135,7 @@ dependencies = [ "derive_more", "error_code", "futures 0.3.32", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "lazy_static", "log_wrappers", "maybe-async", @@ -7176,7 +7169,7 @@ dependencies = [ "encoding_rs 0.8.29", "error_code", "hex 0.4.3", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "lazy_static", "log_wrappers", "match-template", @@ -7220,7 +7213,7 @@ dependencies = [ "keys", "kvengine", "kvenginepb", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git?rev=683dad8fa3689deb243f2ff8ab5847c97af53e38)", + "kvproto", "lazy_static", "num_cpus", "pd_client", @@ -7337,7 +7330,7 @@ dependencies = [ "grpcio", "http 0.2.12", "hyper 0.14.26", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "lazy_static", "libc", "log", @@ -7766,7 +7759,7 @@ version = "0.0.1" dependencies = [ "collections", "crossbeam-utils", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "lazy_static", "parking_lot 0.12.5", "pin-project", @@ -7793,7 +7786,7 @@ dependencies = [ "collections", "error_code", "farmhash", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "log_wrappers", "slog", "thiserror 1.0.69", diff --git a/contrib/tiflash-columnar-hub/Cargo.toml b/contrib/tiflash-columnar-hub/Cargo.toml index 6db56db4e6f..d12f4a12e56 100644 --- a/contrib/tiflash-columnar-hub/Cargo.toml +++ b/contrib/tiflash-columnar-hub/Cargo.toml @@ -26,7 +26,7 @@ fs2 = { git = "https://github.com/tikv/fs2-rs", branch = "tikv" } sysinfo = { git = "https://github.com/tikv/sysinfo", branch = "0.26-fix-cpu" } tokio-executor = { git = "https://github.com/tikv/tokio", branch = "tokio-timer-hotfix" } lindera = { git = "https://github.com/breezewish/lindera", branch = "v0.43.1-tokio-1.24" } -tantivy = { git = "https://github.com/breezewish/tikv-tantivy.git", branch = "patch-0.22.1" } +tantivy = { git = "https://github.com/pingcap/tantivy.git", branch = "cse-0.22.1" } pprof = { git = "https://github.com/tikv/pprof-rs.git", rev = "01cff82dbe6fe110a707bf2b38d8ebb1d14a18f8" } prometheus = { git = "https://github.com/solotzg/rust-prometheus.git", rev = "b4fe98a06a58d29f9b9987a0d7186f6ed5230193", features = ["nightly", "push"], default-features = true } tipb = { git = "https://github.com/pingcap/tipb.git" } From 8e303652bc993b974c356b0af6eef6de83b9cee8 Mon Sep 17 00:00:00 2001 From: 3pointer Date: Tue, 11 Aug 2026 03:38:23 -0700 Subject: [PATCH 5/5] columnar: update CSE FTS Spec v1 pin Signed-off-by: 3pointer --- contrib/cloud-storage-engine | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/cloud-storage-engine b/contrib/cloud-storage-engine index 48fd6016f89..36128849380 160000 --- a/contrib/cloud-storage-engine +++ b/contrib/cloud-storage-engine @@ -1 +1 @@ -Subproject commit 48fd6016f89546623b81fd7d293dbbcf5d80cc5a +Subproject commit 36128849380c771f03e5337f71a41386d0a6859e