diff --git a/src/model_io/safetensors_io.cpp b/src/model_io/safetensors_io.cpp index df71eab11..71f3c8895 100644 --- a/src/model_io/safetensors_io.cpp +++ b/src/model_io/safetensors_io.cpp @@ -5,7 +5,10 @@ #include #include #include +#include +#include #include +#include #include #include #include @@ -93,10 +96,195 @@ static ggml_type safetensors_dtype_to_ggml_type(const std::string& dtype) { ttype = GGML_TYPE_I32; } else if (dtype == "I64") { ttype = GGML_TYPE_I32; + } else if (dtype == "I8") { + ttype = GGML_TYPE_I8; } return ttype; } +struct SafetensorsTensorInfo { + std::string dtype; + std::vector shape; + uint64_t begin = 0; + uint64_t end = 0; +}; + +struct ComfyInt8Info { + bool convrot = false; + uint32_t group_size = 0; + uint64_t scale_offset = 0; + uint64_t scale_nbytes = 0; +}; + +static bool read_safetensors_tensor_info(const nlohmann::json& value, + const std::string& name, + uint64_t data_size, + SafetensorsTensorInfo* result, + std::string* error) { + try { + if (!value.is_object() || !value.contains("dtype") || !value["dtype"].is_string() || + !value.contains("shape") || !value["shape"].is_array() || + !value.contains("data_offsets") || !value["data_offsets"].is_array() || + value["data_offsets"].size() != 2) { + set_error(error, "invalid safetensors descriptor for tensor '" + name + "'"); + return false; + } + + SafetensorsTensorInfo info; + info.dtype = value["dtype"].get(); + if (value["shape"].size() > SD_MAX_DIMS) { + set_error(error, "too many dimensions for tensor '" + name + "'"); + return false; + } + uint64_t elements = 1; + for (const auto& dimension : value["shape"]) { + int64_t size = dimension.get(); + if (size <= 0 || elements > static_cast(std::numeric_limits::max()) / + static_cast(size)) { + set_error(error, "invalid dimension for tensor '" + name + "'"); + return false; + } + elements *= static_cast(size); + info.shape.push_back(size); + } + info.begin = value["data_offsets"][0].get(); + info.end = value["data_offsets"][1].get(); + if (info.begin > info.end || info.end > data_size) { + set_error(error, "data offsets out of bounds for tensor '" + name + "'"); + return false; + } + *result = std::move(info); + return true; + } catch (const std::exception&) { + set_error(error, "invalid safetensors descriptor for tensor '" + name + "'"); + return false; + } +} + +static bool is_power_of_four(uint64_t value) { + if (value < 4) { + return false; + } + while (value % 4 == 0) { + value /= 4; + } + return value == 1; +} + +static bool read_comfy_int8_metadata(std::ifstream& file, + const std::map& tensors, + uint64_t data_start, + std::map* result, + std::set* scale_tensor_names, + std::string* error) { + result->clear(); + scale_tensor_names->clear(); + constexpr const char* marker_suffix = ".comfy_quant"; + constexpr size_t marker_suffix_len = 12; + + for (const auto& [marker_name, marker] : tensors) { + if (!ends_with(marker_name, marker_suffix)) { + continue; + } + if (marker.dtype != "U8") { + set_error(error, "ComfyUI quantization marker '" + marker_name + "' must use U8 storage"); + return false; + } + if (marker.end - marker.begin > 4096) { + set_error(error, "ComfyUI quantization marker '" + marker_name + "' is too large"); + return false; + } + uint64_t marker_elements = 1; + for (int64_t dimension : marker.shape) { + if (marker_elements > std::numeric_limits::max() / + static_cast(dimension)) { + set_error(error, "ComfyUI quantization marker '" + marker_name + "' shape overflows"); + return false; + } + marker_elements *= static_cast(dimension); + } + if (marker_elements != marker.end - marker.begin) { + set_error(error, "ComfyUI quantization marker '" + marker_name + "' has an invalid byte length"); + return false; + } + + std::string marker_json(static_cast(marker.end - marker.begin), '\0'); + file.clear(); + file.seekg(static_cast(data_start + marker.begin)); + file.read(marker_json.data(), static_cast(marker_json.size())); + if (!file) { + set_error(error, "failed to read ComfyUI quantization marker '" + marker_name + "'"); + return false; + } + + nlohmann::json config; + try { + config = nlohmann::json::parse(marker_json); + } catch (const std::exception&) { + set_error(error, "invalid JSON in ComfyUI quantization marker '" + marker_name + "'"); + return false; + } + if (!config.is_object() || !config.contains("format") || !config["format"].is_string()) { + set_error(error, "invalid ComfyUI quantization marker '" + marker_name + "'"); + return false; + } + if (config["format"].get() != "int8_tensorwise") { + continue; + } + + const std::string base = marker_name.substr(0, marker_name.size() - marker_suffix_len); + const auto weight_it = tensors.find(base + ".weight"); + const auto scale_it = tensors.find(base + ".weight_scale"); + if (weight_it == tensors.end() || scale_it == tensors.end()) { + set_error(error, "ComfyUI Int8 marker '" + marker_name + "' is missing its weight or weight_scale tensor"); + return false; + } + const SafetensorsTensorInfo& weight = weight_it->second; + const SafetensorsTensorInfo& scale = scale_it->second; + if (weight.dtype != "I8" || weight.shape.size() != 2 || scale.dtype != "F32" || + scale.shape.size() != 2 || scale.shape[0] != weight.shape[0] || scale.shape[1] != 1) { + set_error(error, "ComfyUI Int8 marker '" + marker_name + "' has incompatible weight and scale tensors"); + return false; + } + const uint64_t output_rows = static_cast(weight.shape[0]); + if (output_rows > std::numeric_limits::max() / sizeof(float) || + scale.end - scale.begin != output_rows * sizeof(float)) { + set_error(error, "ComfyUI Int8 marker '" + marker_name + "' has incompatible weight and scale tensors"); + return false; + } + + ComfyInt8Info info; + if (config.contains("convrot")) { + if (!config["convrot"].is_boolean()) { + set_error(error, "ComfyUI Int8 marker '" + marker_name + "' has a non-boolean convrot field"); + return false; + } + info.convrot = config["convrot"].get(); + } + if (info.convrot) { + if (!config.contains("convrot_groupsize") || !config["convrot_groupsize"].is_number_unsigned()) { + set_error(error, "ComfyUI Int8 marker '" + marker_name + "' has no valid ConvRot group size"); + return false; + } + uint64_t group_size = config["convrot_groupsize"].get(); + if (group_size != 256 || !is_power_of_four(group_size) || + static_cast(weight.shape[1]) % group_size != 0) { + set_error(error, "ComfyUI Int8 marker '" + marker_name + "' uses an unsupported ConvRot group size"); + return false; + } + info.group_size = static_cast(group_size); + } else if (config.contains("convrot_groupsize")) { + set_error(error, "ComfyUI Int8 marker '" + marker_name + "' declares a group size without ConvRot"); + return false; + } + info.scale_offset = data_start + scale.begin; + info.scale_nbytes = scale.end - scale.begin; + result->emplace(weight_it->first, info); + scale_tensor_names->emplace(scale_it->first); + } + return true; +} + // https://huggingface.co/docs/safetensors/index bool read_safetensors_file(const std::string& file_path, std::vector& tensor_storages, @@ -163,29 +351,41 @@ bool read_safetensors_file(const std::string& file_path, } } - tensor_storages.clear(); - for (auto& item : header_.items()) { - std::string name = item.key(); - nlohmann::json tensor_info = item.value(); - // LOG_DEBUG("%s %s\n", name.c_str(), tensor_info.dump().c_str()); - - if (name == "__metadata__") { + const uint64_t data_size = file_size_ - data_start; + std::map tensor_infos; + for (const auto& item : header_.items()) { + if (item.key() == "__metadata__") { continue; } + SafetensorsTensorInfo info; + if (!read_safetensors_tensor_info(item.value(), item.key(), data_size, &info, error)) { + return false; + } + tensor_infos.emplace(item.key(), std::move(info)); + } + std::map comfy_int8_tensors; + std::set comfy_int8_scale_tensors; + if (!read_comfy_int8_metadata(file, + tensor_infos, + data_start, + &comfy_int8_tensors, + &comfy_int8_scale_tensors, + error)) { + return false; + } + + tensor_storages.clear(); + for (const auto& [name, tensor_info] : tensor_infos) { + // LOG_DEBUG("%s %s\n", name.c_str(), tensor_info.dump().c_str()); - std::string dtype = tensor_info["dtype"]; - nlohmann::json shape = tensor_info["shape"]; + const std::string& dtype = tensor_info.dtype; - if (dtype == "U8") { + if (dtype == "U8" || comfy_int8_scale_tensors.find(name) != comfy_int8_scale_tensors.end()) { continue; } - size_t begin = tensor_info["data_offsets"][0].get(); - size_t end = tensor_info["data_offsets"][1].get(); - if (begin > end || end > file_size_ - data_start) { - set_error(error, "data offsets out of bounds for tensor '" + name + "'"); - return false; - } + const uint64_t begin = tensor_info.begin; + const uint64_t end = tensor_info.end; ggml_type type = safetensors_dtype_to_ggml_type(dtype); if (type == GGML_TYPE_COUNT) { @@ -193,18 +393,17 @@ bool read_safetensors_file(const std::string& file_path, return false; } - if (shape.size() > SD_MAX_DIMS) { - set_error(error, "invalid tensor '" + name + "'"); - return false; - } - - int n_dims = (int)shape.size(); + int n_dims = static_cast(tensor_info.shape.size()); int64_t ne[SD_MAX_DIMS] = {1, 1, 1, 1, 1}; for (int i = 0; i < n_dims; i++) { - ne[i] = shape[i].get(); + ne[i] = tensor_info.shape[i]; } if (n_dims == 5) { + if (ne[0] > std::numeric_limits::max() / ne[1]) { + set_error(error, "tensor dimensions overflow for '" + name + "'"); + return false; + } n_dims = 4; ne[0] = ne[0] * ne[1]; ne[1] = ne[2]; @@ -220,7 +419,7 @@ bool read_safetensors_file(const std::string& file_path, TensorStorage tensor_storage(name, type, ne, n_dims, 0, data_start + begin); tensor_storage.reverse_ne(); - size_t tensor_data_size = end - begin; + uint64_t tensor_data_size = end - begin; bool tensor_size_ok; if (dtype == "F8_E4M3") { @@ -247,6 +446,21 @@ bool read_safetensors_file(const std::string& file_path, return false; } + auto comfy_int8 = comfy_int8_tensors.find(name); + if (dtype == "I8") { + if (comfy_int8 == comfy_int8_tensors.end()) { + set_error(error, "unsupported Int8 safetensors tensor '" + name + "' without a ComfyUI Int8 marker"); + return false; + } + tensor_storage.is_comfy_int8_tensorwise = true; + tensor_storage.comfy_int8_convrot = comfy_int8->second.convrot; + tensor_storage.comfy_int8_group_size = comfy_int8->second.group_size; + tensor_storage.comfy_int8_scale_offset = comfy_int8->second.scale_offset; + tensor_storage.comfy_int8_scale_nbytes = comfy_int8->second.scale_nbytes; + // The runtime reconstructs an F16 matrix before backend upload. + tensor_storage.expected_type = GGML_TYPE_F16; + } + tensor_storages.push_back(tensor_storage); // LOG_DEBUG("%s %s", tensor_storage.to_string().c_str(), dtype.c_str()); diff --git a/src/model_io/tensor_storage.h b/src/model_io/tensor_storage.h index 5c977f516..b00233344 100644 --- a/src/model_io/tensor_storage.h +++ b/src/model_io/tensor_storage.h @@ -22,6 +22,14 @@ struct TensorStorage { bool is_f8_e5m2 = false; bool is_f64 = false; bool is_i64 = false; + // ComfyUI TensorWiseINT8 stores the I8 weight and its per-output-row F32 + // scale separately. ConvRot metadata is carried by a U8 JSON side tensor. + // The loader reconstructs this format into F16/F32 before backend upload. + bool is_comfy_int8_tensorwise = false; + bool comfy_int8_convrot = false; + uint32_t comfy_int8_group_size = 0; + uint64_t comfy_int8_scale_offset = 0; + uint64_t comfy_int8_scale_nbytes = 0; int64_t ne[SD_MAX_DIMS] = {1, 1, 1, 1, 1}; int n_dims = 0; diff --git a/src/model_loader.cpp b/src/model_loader.cpp index a70ffefd3..5199237ad 100644 --- a/src/model_loader.cpp +++ b/src/model_loader.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -154,6 +155,82 @@ void i64_to_i32_vec(int64_t* src, int32_t* dst, int64_t n) { } } +static void apply_regular_hadamard_4(float* values, size_t stride) { + const float a = values[0 * stride]; + const float b = values[1 * stride]; + const float c = values[2 * stride]; + const float d = values[3 * stride]; + values[0 * stride] = (a + b + c - d) * 0.5f; + values[1 * stride] = (a + b - c + d) * 0.5f; + values[2 * stride] = (a - b + c + d) * 0.5f; + values[3 * stride] = (-a + b + c + d) * 0.5f; +} + +static bool dequantize_comfy_int8_tensorwise(const TensorStorage& tensor_storage, + const int8_t* quantized, + const float* scales, + void* dst, + ggml_type dst_type, + std::string* error) { + if (!tensor_storage.is_comfy_int8_tensorwise || tensor_storage.n_dims != 2 || + tensor_storage.ne[0] <= 0 || tensor_storage.ne[1] <= 0) { + *error = "invalid ComfyUI Int8 tensor metadata"; + return false; + } + if (dst_type != GGML_TYPE_F16 && dst_type != GGML_TYPE_F32) { + *error = "ComfyUI Int8 compatibility loading requires an F16 or F32 destination"; + return false; + } + + const size_t columns = static_cast(tensor_storage.ne[0]); + const size_t rows = static_cast(tensor_storage.ne[1]); + if (rows > std::numeric_limits::max() / columns || + tensor_storage.comfy_int8_scale_nbytes != rows * sizeof(float)) { + *error = "invalid ComfyUI Int8 tensor dimensions or scale size"; + return false; + } + const size_t group_size = tensor_storage.comfy_int8_convrot + ? static_cast(tensor_storage.comfy_int8_group_size) + : std::min(columns, 4096); + if (group_size == 0 || columns % group_size != 0) { + *error = "invalid ComfyUI Int8 ConvRot group size"; + return false; + } + + std::vector values(group_size); + for (size_t row = 0; row < rows; ++row) { + const float scale = scales[row]; + if (!std::isfinite(scale) || scale <= 0.f) { + *error = "ComfyUI Int8 tensor has a non-positive or non-finite scale"; + return false; + } + for (size_t column = 0; column < columns; column += group_size) { + const size_t offset = row * columns + column; + for (size_t i = 0; i < group_size; ++i) { + values[i] = static_cast(quantized[offset + i]) * scale; + } + if (tensor_storage.comfy_int8_convrot) { + for (size_t stride = 1; stride < group_size; stride *= 4) { + const size_t block = stride * 4; + for (size_t base = 0; base < group_size; base += block) { + for (size_t i = 0; i < stride; ++i) { + apply_regular_hadamard_4(values.data() + base + i, stride); + } + } + } + } + if (dst_type == GGML_TYPE_F16) { + auto* output = static_cast(dst) + offset; + ggml_fp32_to_fp16_row(values.data(), output, static_cast(group_size)); + } else { + auto* output = static_cast(dst) + offset; + memcpy(output, values.data(), group_size * sizeof(float)); + } + } + } + return true; +} + void convert_tensor(void* src, ggml_type src_type, void* dst, @@ -1240,10 +1317,42 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, return true; }; + auto read_comfy_int8_scales = [&](char* buf, size_t n) -> bool { + if (zip != nullptr) { + LOG_ERROR("ComfyUI Int8 tensor '%s' cannot be stored in a zip file", tensor_storage.name.c_str()); + return false; + } + if (mmapped) { + if (!mmapped->copy_data(buf, n, tensor_storage.comfy_int8_scale_offset)) { + LOG_ERROR("read ComfyUI Int8 scales failed: '%s'", file_path.c_str()); + return false; + } + } else { + file.clear(); + file.seekg(static_cast(tensor_storage.comfy_int8_scale_offset)); + file.read(buf, static_cast(n)); + if (!file) { + LOG_ERROR("read ComfyUI Int8 scales failed: '%s'", file_path.c_str()); + return false; + } + } + return true; + }; + char* read_buf = nullptr; char* target_buf = nullptr; char* convert_buf = nullptr; - if (dst_tensor->buffer == nullptr || ggml_backend_buffer_is_host(dst_tensor->buffer)) { + const bool is_comfy_int8 = tensor_storage.is_comfy_int8_tensorwise; + if (is_comfy_int8) { + read_buffer.resize(nbytes_to_read); + read_buf = reinterpret_cast(read_buffer.data()); + if (dst_tensor->buffer == nullptr || ggml_backend_buffer_is_host(dst_tensor->buffer)) { + target_buf = reinterpret_cast(dst_tensor->data); + } else { + convert_buffer.resize(ggml_nbytes(dst_tensor)); + target_buf = reinterpret_cast(convert_buffer.data()); + } + } else if (dst_tensor->buffer == nullptr || ggml_backend_buffer_is_host(dst_tensor->buffer)) { if (tensor_storage.type == dst_tensor->type) { GGML_ASSERT(ggml_nbytes(dst_tensor) == tensor_storage.nbytes()); if (tensor_storage.is_f64 || tensor_storage.is_i64) { @@ -1279,7 +1388,27 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, read_time_ms.fetch_add(t1 - t0); t0 = ggml_time_ms(); - if (tensor_storage.is_f8_e4m3) { + if (is_comfy_int8) { + std::vector scale_buffer(tensor_storage.comfy_int8_scale_nbytes); + if (!read_comfy_int8_scales(reinterpret_cast(scale_buffer.data()), scale_buffer.size())) { + failed = true; + break; + } + std::string dequantization_error; + if (!dequantize_comfy_int8_tensorwise(tensor_storage, + reinterpret_cast(read_buf), + reinterpret_cast(scale_buffer.data()), + target_buf, + dst_tensor->type, + &dequantization_error)) { + LOG_ERROR("ComfyUI Int8 tensor '%s' cannot be reconstructed: %s", + tensor_storage.name.c_str(), + dequantization_error.c_str()); + failed = true; + break; + } + convert_buf = target_buf; + } else if (tensor_storage.is_f8_e4m3) { f8_e4m3_to_f16_vec((uint8_t*)read_buf, (uint16_t*)target_buf, tensor_storage.nelements()); } else if (tensor_storage.is_f8_e5m2) { f8_e5m2_to_f16_vec((uint8_t*)read_buf, (uint16_t*)target_buf, tensor_storage.nelements()); @@ -1288,7 +1417,7 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, } else if (tensor_storage.is_i64) { i64_to_i32_vec((int64_t*)read_buf, (int32_t*)target_buf, tensor_storage.nelements()); } - if (tensor_storage.type != dst_tensor->type) { + if (!is_comfy_int8 && tensor_storage.type != dst_tensor->type) { if (convert_buf == nullptr) { LOG_ERROR("read tensor data failed: too less memory for conversion"); failed = true; @@ -1303,7 +1432,7 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, tensor_storage.nelements() / tensor_storage.ne[0], tensor_storage.ne[0], std::move(imatrix)); - } else { + } else if (!is_comfy_int8) { convert_buf = read_buf; } t1 = ggml_time_ms(); @@ -1319,7 +1448,8 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, copy_to_backend_time_ms.fetch_add(t1 - t0); } - bytes_processed.fetch_add((uint64_t)nbytes_to_read); + bytes_processed.fetch_add((uint64_t)nbytes_to_read + + (is_comfy_int8 ? tensor_storage.comfy_int8_scale_nbytes : 0)); } if (zip != nullptr) { zip_close(zip); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 6d0572659..76525b2c7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -47,3 +47,9 @@ target_include_directories(test-gguf-comfy-shape PRIVATE "${PROJECT_SOURCE_DIR}/src") target_link_libraries(test-gguf-comfy-shape PRIVATE stable-diffusion) add_test(NAME test-gguf-comfy-shape COMMAND test-gguf-comfy-shape) + +add_executable(test-safetensors-convrot test-safetensors-convrot.cpp) +target_include_directories(test-safetensors-convrot PRIVATE + "${PROJECT_SOURCE_DIR}/src") +target_link_libraries(test-safetensors-convrot PRIVATE stable-diffusion) +add_test(NAME test-safetensors-convrot COMMAND test-safetensors-convrot) diff --git a/tests/test-safetensors-convrot.cpp b/tests/test-safetensors-convrot.cpp new file mode 100644 index 000000000..461494e45 --- /dev/null +++ b/tests/test-safetensors-convrot.cpp @@ -0,0 +1,124 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "model_io/binary_io.h" +#include "model_loader.h" + +namespace { + +std::string make_header(const std::string& marker) { + return "{\"layer.weight\":{\"dtype\":\"I8\",\"shape\":[4,256],\"data_offsets\":[0,1024]}," + "\"layer.weight_scale\":{\"dtype\":\"F32\",\"shape\":[4,1],\"data_offsets\":[1024,1040]}," + "\"layer.comfy_quant\":{\"dtype\":\"U8\",\"shape\":[" + + std::to_string(marker.size()) + "],\"data_offsets\":[1040," + + std::to_string(1040 + marker.size()) + "]}}"; +} + +void write_fixture(const std::filesystem::path& path, const std::string& marker) { + const std::string header = make_header(marker); + std::vector weights(4 * 256, 0); + weights[0] = 2; + const float scales[] = {0.5f, 0.5f, 0.5f, 0.5f}; + + std::ofstream file(path, std::ios::binary | std::ios::trunc); + GGML_ASSERT(file.is_open()); + model_io::write_u64(file, header.size()); + file.write(header.data(), static_cast(header.size())); + file.write(reinterpret_cast(weights.data()), static_cast(weights.size())); + file.write(reinterpret_cast(scales), sizeof(scales)); + file.write(marker.data(), static_cast(marker.size())); + GGML_ASSERT(file.good()); +} + +const TensorStorage& find_tensor(const ModelLoader& loader, const std::string& name) { + const auto& tensors = loader.get_tensor_storage_map(); + const auto it = tensors.find(name); + GGML_ASSERT(it != tensors.end()); + return it->second; +} + +} // namespace + +int main() { + const std::filesystem::path path = std::filesystem::temp_directory_path() / + "stable-diffusion-convrot-test.safetensors"; + const std::string marker = + "{\"format\":\"int8_tensorwise\",\"convrot\":true,\"convrot_groupsize\":256}"; + write_fixture(path, marker); + + ModelLoader loader; + GGML_ASSERT(loader.init_from_file(path.string())); + const TensorStorage& weight = find_tensor(loader, "layer.weight"); + GGML_ASSERT(weight.type == GGML_TYPE_I8); + GGML_ASSERT(weight.expected_type == GGML_TYPE_F16); + GGML_ASSERT(weight.is_comfy_int8_tensorwise); + GGML_ASSERT(weight.comfy_int8_convrot); + GGML_ASSERT(weight.comfy_int8_group_size == 256); + GGML_ASSERT(weight.comfy_int8_scale_nbytes == 4 * sizeof(float)); + GGML_ASSERT(loader.get_tensor_storage_map().find("layer.weight_scale") == loader.get_tensor_storage_map().end()); + + ggml_init_params params = {4096, nullptr, false}; + ggml_context* ctx = ggml_init(params); + GGML_ASSERT(ctx != nullptr); + ggml_tensor* decoded = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, 256, 4); + GGML_ASSERT(loader.load_tensor(weight, decoded)); + + std::vector values(4 * 256); + ggml_fp16_to_fp32_row(static_cast(decoded->data), values.data(), values.size()); + float energy = 0.f; + for (float value : values) { + energy += value * value; + } + // The normalized regular Hadamard matrix is orthogonal: ConvRot inversion + // preserves the squared norm of the dequantized first row. + GGML_ASSERT(std::fabs(energy - 1.f) < 0.002f); + ggml_free(ctx); + + const std::string unsupported_group = + "{\"format\":\"int8_tensorwise\",\"convrot\":true,\"convrot_groupsize\":16}"; + write_fixture(path, unsupported_group); + ModelLoader invalid_loader; + GGML_ASSERT(!invalid_loader.init_from_file(path.string())); + + write_fixture(path, "not-json"); + ModelLoader malformed_loader; + GGML_ASSERT(!malformed_loader.init_from_file(path.string())); + + if (const char* real_model_path = std::getenv("CONVROT_MODEL_PATH")) { + ModelLoader real_loader; + GGML_ASSERT(real_loader.init_from_file(real_model_path)); + const TensorStorage* real_weight = nullptr; + for (const auto& [_, tensor] : real_loader.get_tensor_storage_map()) { + if (tensor.is_comfy_int8_tensorwise) { + real_weight = &tensor; + break; + } + } + GGML_ASSERT(real_weight != nullptr); + const size_t elements = static_cast(real_weight->ne[0]) * + static_cast(real_weight->ne[1]); + ggml_init_params real_params = {ggml_tensor_overhead() + elements * sizeof(ggml_fp16_t) + 4096, + nullptr, + false}; + ggml_context* real_ctx = ggml_init(real_params); + GGML_ASSERT(real_ctx != nullptr); + ggml_tensor* real_decoded = ggml_new_tensor_2d(real_ctx, + GGML_TYPE_F16, + real_weight->ne[0], + real_weight->ne[1]); + GGML_ASSERT(real_loader.load_tensor(*real_weight, real_decoded)); + const float first_value = ggml_fp16_to_fp32(static_cast(real_decoded->data)[0]); + GGML_ASSERT(std::isfinite(first_value)); + ggml_free(real_ctx); + } + + std::error_code ec; + std::filesystem::remove(path, ec); + return 0; +}