feat: dispatch MiniMax H3 ConvRot weights natively

src/core/ggml_extend.hpp

@@ -5,8 +5,8 @@
 #include <inttypes.h>
 #include <stdarg.h>
 #include <algorithm>
-#include <atomic>
 #include <cstdlib>
+#include <atomic>
 #include <cstring>
 #include <fstream>
 #include <functional>
@@ -21,6 +21,7 @@
 #include <set>
 #include <sstream>
 #include <string>
+#include <stdexcept>
 #include <unordered_map>
 #include <unordered_set>
 #include <vector>
@@ -43,6 +44,52 @@
 
 #define EPS 1e-05f
 
+// Select the compact representation before any model parameter tensor is
+// created.  The default is deliberately native: an unsupported backend is a
+// configuration error rather than a silent CPU reroute or full F16 expansion.
+// Set SD_CONVROT_MODE=compat to explicitly request the compatibility loader.
+inline String2TensorStorage select_convrot_tensor_storage(ggml_backend_t backend,
+                                                           const String2TensorStorage& source,
+                                                           const std::string& component) {
+    // OrderedMap's default copy also copies its iterator index; rebuild it so
+    // this independent policy view owns a valid index into its own list.
+    String2TensorStorage selected;
+    bool has_convrot = false;
+    for (const auto& [name, storage] : source) {
+        selected.insert({name, storage});
+        has_convrot = has_convrot || storage.is_comfy_int8_convrot_weight();
+    }
+    if (!has_convrot) {
+        return selected;
+    }
+
+    const char* mode = std::getenv("SD_CONVROT_MODE");
+    const bool compatibility_mode = mode != nullptr && std::strcmp(mode, "compat") == 0;
+    if (mode != nullptr && !compatibility_mode && std::strcmp(mode, "native") != 0) {
+        throw std::runtime_error("invalid SD_CONVROT_MODE; expected 'native' or 'compat'");
+    }
+    const char* backend_name = backend != nullptr ? ggml_backend_name(backend) : "unknown";
+    if (compatibility_mode) {
+        LOG_INFO("ConvRot: using explicitly selected F16 compatibility path for %s on backend %s",
+                 component.c_str(), backend_name);
+        return selected;
+    }
+    if (!ggml_backend_supports_convrot(backend, GGML_TYPE_F32, 256)) {
+        throw std::runtime_error("ConvRot native support is required for " + component +
+                                 " but backend '" + backend_name +
+                                 "' lacks the 256-wide I8/F32 ConvRot operation; use a capable backend or set "
+                                 "SD_CONVROT_MODE=compat to select the F16 compatibility path");
+    }
+    for (auto& [_, storage] : selected) {
+        if (storage.is_comfy_int8_convrot_weight()) {
+            storage.comfy_int8_native_enabled = true;
+        }
+    }
+    LOG_INFO("ConvRot: selected native compact I8/F32 path for %s on backend %s",
+             component.c_str(), backend_name);
+    return selected;
+}
+
 #ifndef __STATIC_INLINE__
 #define __STATIC_INLINE__ static inline
 #endif
@@ -3875,12 +3922,31 @@ class Linear : public UnaryBlock {
     bool force_prec_f32;
     bool allow_weight_scale;
     bool has_weight_scale = false;
+    // This is distinct from `weight_scale`: the latter is a regular
+    // post-linear model parameter, while ConvRot's F32 vector is a private
+    // sidecar input to GGML_OP_MUL_MAT_CONVROT.
+    bool has_convrot_weight = false;
+    bool use_convrot_f16_compat = false;
     float scale;
     std::string prefix;
 
     void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
         this->prefix         = prefix;
         has_weight_scale     = false;
+        has_convrot_weight   = false;
+        use_convrot_f16_compat = false;
+        const auto storage_it = tensor_storage_map.find(prefix + "weight");
+        if (storage_it != tensor_storage_map.end() && storage_it->second.is_comfy_int8_convrot_weight() &&
+            storage_it->second.comfy_int8_native_enabled) {
+            params["weight"]                 = ggml_new_tensor_2d(ctx, GGML_TYPE_I8, in_features, out_features);
+            params["weight.convrot_scale"]   = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, out_features);
+            has_convrot_weight                 = true;
+            use_convrot_f16_compat             = storage_it->second.name.rfind("text_encoders.llm.", 0) == 0;
+            if (bias) {
+                params["bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, out_features);
+            }
+            return;
+        }
         enum ggml_type wtype = get_type(prefix + "weight", tensor_storage_map, GGML_TYPE_F32);
         if (in_features % ggml_blck_size(wtype) != 0 || force_f32) {
             wtype = GGML_TYPE_F32;
@@ -3928,7 +3994,23 @@ class Linear : public UnaryBlock {
         }
         ggml_tensor* linear_bias = has_weight_scale ? nullptr : b;
         ggml_tensor* out         = nullptr;
-        if (ctx->weight_adapter) {
+        if (has_convrot_weight) {
+            // ConvRot weights and their tensor-wise scales remain compact at
+            // rest.  The operator owns the scale semantics; do not route it
+            // through the ordinary `weight_scale` post-multiply path.
+            out = ggml_mul_mat_convrot(ctx->ggml_ctx, x, w, params["weight.convrot_scale"], 256);
+            // MiniMax H3's ConvRot text encoder is calibrated for the F16
+            // compatibility arithmetic. CUDA reconstructs one F16 matrix at
+            // a time and uses its standard F16 GEMM without retaining an F16
+            // copy of the complete text encoder. Other backends may ignore
+            // this hint and keep their native compact implementation.
+            if (use_convrot_f16_compat) {
+                ggml_mul_mat_convrot_set_f16_compat(out, true);
+            }
+            if (b != nullptr) {
+                out = ggml_add_inplace(ctx->ggml_ctx, out, b);
+            }
+        } else if (ctx->weight_adapter) {
             WeightAdapter::ForwardParams forward_params;
             forward_params.op_type               = WeightAdapter::ForwardParams::op_type_t::OP_LINEAR;
             forward_params.linear.force_prec_f32 = force_prec_f32;

src/model/diffusion/minimax_h3.hpp

@@ -986,7 +986,9 @@ namespace MiniMaxH3 {
             : DiffusionModelRunner(backend, prefix, weight_manager),
               config(Config::detect_from_weights(tensors, prefix)),
               model(config) {
-            model.init(params_ctx, tensors, prefix);
+            model.init(params_ctx,
+                       select_convrot_tensor_storage(backend, tensors, "MiniMax-H3 diffusion model"),
+                       prefix);
         }
 
         std::string get_desc() override {

src/model/te/llm.hpp

@@ -1771,7 +1771,9 @@ namespace LLM {
                 }
             }
             model = LLM(config, enable_vision, config.llama_cpp_style);
-            model.init(params_ctx, tensor_storage_map, prefix);
+            model.init(params_ctx,
+                       select_convrot_tensor_storage(backend, tensor_storage_map, "LLM"),
+                       prefix);
         }
 
         std::string get_desc() override {