vulkan: add Q1_0_g128 (1-bit ternary) shader support

ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl

@@ -22,6 +22,7 @@ vec2 dequantize(uint ib, uint iqs, uint a_offset) {
 }
 #endif
 
+#if defined(DATA_A_Q1_0)vec2 dequantize(uint ib, uint iqs, uint a_offset) {    const uint byte_idx = iqs / 8;    const uint bit_idx = iqs % 8;    const uint bits = uint(data_a[a_offset + ib].qs[byte_idx]);    const float sign0 = ((bits >> bit_idx) & 1) == 1 ? 1.0f : -1.0f;    const uint byte_idx2 = (iqs + 1) / 8;    const uint bit_idx2 = (iqs + 1) % 8;    const uint bits2 = uint(data_a[a_offset + ib].qs[byte_idx2]);    const float sign1 = ((bits2 >> bit_idx2) & 1) == 1 ? 1.0f : -1.0f;    return vec2(sign0, sign1);}vec4 dequantize4(uint ib, uint iqs, uint a_offset) {    const uint byte_idx0 = iqs / 8;    const uint bit_idx0 = iqs % 8;    const uint bits0 = uint(data_a[a_offset + ib].qs[byte_idx0]);    const float s0 = ((bits0 >> bit_idx0) & 1) == 1 ? 1.0f : -1.0f;    const uint byte_idx1 = (iqs + 1) / 8;    const uint bit_idx1 = (iqs + 1) % 8;    const uint bits1 = uint(data_a[a_offset + ib].qs[byte_idx1]);    const float s1 = ((bits1 >> bit_idx1) & 1) == 1 ? 1.0f : -1.0f;    const uint byte_idx2 = (iqs + 2) / 8;    const uint bit_idx2 = (iqs + 2) % 8;    const uint bits2 = uint(data_a[a_offset + ib].qs[byte_idx2]);    const float s2 = ((bits2 >> bit_idx2) & 1) == 1 ? 1.0f : -1.0f;    const uint byte_idx3 = (iqs + 3) / 8;    const uint bit_idx3 = (iqs + 3) % 8;    const uint bits3 = uint(data_a[a_offset + ib].qs[byte_idx3]);    const float s3 = ((bits3 >> bit_idx3) & 1) == 1 ? 1.0f : -1.0f;    return vec4(s0, s1, s2, s3);}#endif
 #if defined(DATA_A_Q4_0)
 vec2 dequantize(uint ib, uint iqs, uint a_offset) {
     const uint vui = uint(data_a[a_offset + ib].qs[iqs]);
@@ -448,7 +449,8 @@ vec2 get_dm(uint ib, uint a_offset) {
 }
 #endif
 
-#if defined(DATA_A_Q4_0) || defined(DATA_A_Q5_0) || defined(DATA_A_Q8_0) || defined(DATA_A_IQ1_S) || defined(DATA_A_IQ2_XXS) || defined(DATA_A_IQ2_XS) || defined(DATA_A_IQ2_S) || defined(DATA_A_IQ3_XXS) || defined(DATA_A_IQ3_S) || defined(DATA_A_IQ4_XS) || defined(DATA_A_IQ4_NL)
+#if defined(DATA_A_Q1_0)vec2 dequantize(uint ib, uint iqs, uint a_offset) {    const uint byte_idx = iqs / 8;    const uint bit_idx = iqs % 8;    const uint bits = uint(data_a[a_offset + ib].qs[byte_idx]);    const float sign0 = ((bits >> bit_idx) & 1) == 1 ? 1.0f : -1.0f;    const uint byte_idx2 = (iqs + 1) / 8;    const uint bit_idx2 = (iqs + 1) % 8;    const uint bits2 = uint(data_a[a_offset + ib].qs[byte_idx2]);    const float sign1 = ((bits2 >> bit_idx2) & 1) == 1 ? 1.0f : -1.0f;    return vec2(sign0, sign1);}vec4 dequantize4(uint ib, uint iqs, uint a_offset) {    const uint byte_idx0 = iqs / 8;    const uint bit_idx0 = iqs % 8;    const uint bits0 = uint(data_a[a_offset + ib].qs[byte_idx0]);    const float s0 = ((bits0 >> bit_idx0) & 1) == 1 ? 1.0f : -1.0f;    const uint byte_idx1 = (iqs + 1) / 8;    const uint bit_idx1 = (iqs + 1) % 8;    const uint bits1 = uint(data_a[a_offset + ib].qs[byte_idx1]);    const float s1 = ((bits1 >> bit_idx1) & 1) == 1 ? 1.0f : -1.0f;    const uint byte_idx2 = (iqs + 2) / 8;    const uint bit_idx2 = (iqs + 2) % 8;    const uint bits2 = uint(data_a[a_offset + ib].qs[byte_idx2]);    const float s2 = ((bits2 >> bit_idx2) & 1) == 1 ? 1.0f : -1.0f;    const uint byte_idx3 = (iqs + 3) / 8;    const uint bit_idx3 = (iqs + 3) % 8;    const uint bits3 = uint(data_a[a_offset + ib].qs[byte_idx3]);    const float s3 = ((bits3 >> bit_idx3) & 1) == 1 ? 1.0f : -1.0f;    return vec4(s0, s1, s2, s3);}#endif
+#if defined(DATA_A_Q1_0) || defined(DATA_A_Q4_0) || defined(DATA_A_Q5_0) || defined(DATA_A_Q8_0) || defined(DATA_A_IQ1_S) || defined(DATA_A_IQ2_XXS) || defined(DATA_A_IQ2_XS) || defined(DATA_A_IQ2_S) || defined(DATA_A_IQ3_XXS) || defined(DATA_A_IQ3_S) || defined(DATA_A_IQ4_XS) || defined(DATA_A_IQ4_NL)
 vec2 get_dm(uint ib, uint a_offset) {
     return vec2(float(data_a[a_offset + ib].d), 0);
 }

ggml/src/ggml-vulkan/vulkan-shaders/dequant_q1_0.comp

@@ -0,0 +1,29 @@
+#version 450
+
+#include "dequant_head.glsl"
+
+layout(local_size_x = 256, local_size_y = 1, local_size_z = 1) in;
+
+layout (binding = 0) readonly buffer A {block_q1_0 data_a[];};
+layout (binding = 1) writeonly buffer D {D_TYPE data_b[];};
+
+void main() {
+    // Each thread handles one 128-element block
+    const uint ib = gl_WorkGroupID.x * 256 + gl_LocalInvocationID.x;
+
+    if (ib >= p.nel / 128) {
+        return;
+    }
+
+    const uint b_idx = ib * 128;
+    const float d = float(data_a[ib].d);
+
+    // Each block has 16 bytes = 128 bits = 128 elements
+    [[unroll]] for (uint byte_idx = 0; byte_idx < 16; ++byte_idx) {
+        const uint bits = uint(data_a[ib].qs[byte_idx]);
+        [[unroll]] for (uint bit_idx = 0; bit_idx < 8; ++bit_idx) {
+            const float sign = ((bits >> bit_idx) & 1) == 1 ? 1.0f : -1.0f;
+            data_b[b_idx + byte_idx * 8 + bit_idx] = D_TYPE(d * sign);
+        }
+    }
+}

ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q1_0.comp

@@ -0,0 +1,108 @@
+#version 450
+#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
+#extension GL_EXT_shader_8bit_storage : require
+
+#include "mul_mat_vec_base.glsl"
+
+layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
+
+// Fused 1-bit matrix-vector multiply for Q1_0.
+// 4 threads per block, each handles 32 elements (one uint32 of packed bits).
+// Uses simple ternary sign selection which compiles to v_cndmask on RDNA.
+
+FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
+
+void calc_block(const uint a_offset, const uint b_offset, const uint itid, const uint i,
+                const uint num_blocks_per_row, const uint first_row, const uint num_rows) {
+
+    const uint y_idx_base = i * 128 + itid * 32;
+
+    [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
+        const uint base_b = (j * p.batch_stride_b + b_offset + y_idx_base) / 4;
+        const vec4 bv0 = vec4(data_b_v4[base_b]);
+        const vec4 bv1 = vec4(data_b_v4[base_b + 1]);
+        const vec4 bv2 = vec4(data_b_v4[base_b + 2]);
+        const vec4 bv3 = vec4(data_b_v4[base_b + 3]);
+        const vec4 bv4 = vec4(data_b_v4[base_b + 4]);
+        const vec4 bv5 = vec4(data_b_v4[base_b + 5]);
+        const vec4 bv6 = vec4(data_b_v4[base_b + 6]);
+        const vec4 bv7 = vec4(data_b_v4[base_b + 7]);
+
+        uint ibi = a_offset + first_row * num_blocks_per_row + i;
+
+        [[unroll]] for (uint n = 0; n < num_rows; ++n) {
+            const float d = float(data_a[ibi].d);
+
+            const uint byte_base = itid * 4;
+            const uint bits = uint(data_a[ibi].qs[byte_base])
+                            | (uint(data_a[ibi].qs[byte_base + 1]) << 8)
+                            | (uint(data_a[ibi].qs[byte_base + 2]) << 16)
+                            | (uint(data_a[ibi].qs[byte_base + 3]) << 24);
+
+            FLOAT_TYPE partial = FLOAT_TYPE(0);
+
+            partial += FLOAT_TYPE(dot(vec4(
+                (bits & 0x1u) != 0 ? 1.0 : -1.0, (bits & 0x2u) != 0 ? 1.0 : -1.0,
+                (bits & 0x4u) != 0 ? 1.0 : -1.0, (bits & 0x8u) != 0 ? 1.0 : -1.0), bv0));
+            partial += FLOAT_TYPE(dot(vec4(
+                (bits & 0x10u) != 0 ? 1.0 : -1.0, (bits & 0x20u) != 0 ? 1.0 : -1.0,
+                (bits & 0x40u) != 0 ? 1.0 : -1.0, (bits & 0x80u) != 0 ? 1.0 : -1.0), bv1));
+            partial += FLOAT_TYPE(dot(vec4(
+                (bits & 0x100u) != 0 ? 1.0 : -1.0, (bits & 0x200u) != 0 ? 1.0 : -1.0,
+                (bits & 0x400u) != 0 ? 1.0 : -1.0, (bits & 0x800u) != 0 ? 1.0 : -1.0), bv2));
+            partial += FLOAT_TYPE(dot(vec4(
+                (bits & 0x1000u) != 0 ? 1.0 : -1.0, (bits & 0x2000u) != 0 ? 1.0 : -1.0,
+                (bits & 0x4000u) != 0 ? 1.0 : -1.0, (bits & 0x8000u) != 0 ? 1.0 : -1.0), bv3));
+            partial += FLOAT_TYPE(dot(vec4(
+                (bits & 0x10000u) != 0 ? 1.0 : -1.0, (bits & 0x20000u) != 0 ? 1.0 : -1.0,
+                (bits & 0x40000u) != 0 ? 1.0 : -1.0, (bits & 0x80000u) != 0 ? 1.0 : -1.0), bv4));
+            partial += FLOAT_TYPE(dot(vec4(
+                (bits & 0x100000u) != 0 ? 1.0 : -1.0, (bits & 0x200000u) != 0 ? 1.0 : -1.0,
+                (bits & 0x400000u) != 0 ? 1.0 : -1.0, (bits & 0x800000u) != 0 ? 1.0 : -1.0), bv5));
+            partial += FLOAT_TYPE(dot(vec4(
+                (bits & 0x1000000u) != 0 ? 1.0 : -1.0, (bits & 0x2000000u) != 0 ? 1.0 : -1.0,
+                (bits & 0x4000000u) != 0 ? 1.0 : -1.0, (bits & 0x8000000u) != 0 ? 1.0 : -1.0), bv6));
+            partial += FLOAT_TYPE(dot(vec4(
+                (bits & 0x10000000u) != 0 ? 1.0 : -1.0, (bits & 0x20000000u) != 0 ? 1.0 : -1.0,
+                (bits & 0x40000000u) != 0 ? 1.0 : -1.0, (bits & 0x80000000u) != 0 ? 1.0 : -1.0), bv7));
+
+            temp[j][n] = fma(FLOAT_TYPE(d), partial, temp[j][n]);
+            ibi += num_blocks_per_row;
+        }
+    }
+}
+
+void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
+    uint a_offset, b_offset, d_offset;
+    get_offsets(a_offset, b_offset, d_offset);
+
+    const uint num_blocks_per_row = p.ncols / 128;
+    const uint blocks_per_wg = gl_WorkGroupSize.x / 4;
+    const uint tid = gl_LocalInvocationID.x;
+    const uint itid = tid % 4;
+    const uint ix = tid / 4;
+
+    [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
+        [[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
+            temp[j][i] = FLOAT_TYPE(0);
+        }
+    }
+
+    [[unroll]] for (uint i = ix; i < num_blocks_per_row; i += blocks_per_wg)
+        calc_block(a_offset, b_offset, itid, i, num_blocks_per_row, first_row, num_rows);
+
+    reduce_result(temp, d_offset, first_row, num_rows, tid);
+}
+
+void main() {
+    const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
+
+    if (first_row + NUM_ROWS <= p.stride_d) {
+        compute_outputs(first_row, NUM_ROWS);
+    } else {
+        if (first_row >= p.stride_d) {
+            return;
+        }
+        compute_outputs(first_row, p.stride_d - first_row);
+    }
+}

ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl

@@ -128,6 +128,36 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uin
             const i8vec2 v1 = unpack8(int32_t(data_a_packed16[ib].qs[2*iqs + 1])).xy;
             const vec4 v = vec4(v0.x, v0.y, v1.x, v1.y) * d;
 
+            buf_a[buf_idx    ] = FLOAT_TYPE_VEC2(v.xy);
+            buf_a[buf_idx + 1] = FLOAT_TYPE_VEC2(v.zw);
+#elif defined(DATA_A_Q1_0)
+            const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
+            const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
+
+            // LOAD_VEC_A = 4, so each load processes 4 elements.
+            // 128 elements per block / 4 = 32 loads per block.
+            const uint ib = idx / 32;   // block index
+            const uint iel = (idx % 32) * 4;  // element offset within block (0,4,8,...124)
+
+            const float d = float(data_a[ib].d);
+            const float d2 = d + d;
+            const float neg_d = -d;
+
+            // Decode the containing 16-bit chunk, then select the 4-bit sub-group for this load.
+            const uint chunk16 = iel / 16;
+            const uint chunk_bit = iel % 16;
+            const uint byte_offset = chunk16 * 2;
+            const uint bits16 = uint(data_a[ib].qs[byte_offset])
+                              | (uint(data_a[ib].qs[byte_offset + 1]) << 8);
+            const uint bits = (bits16 >> chunk_bit) & 0xFu;
+
+            // Branchless FMA: d*(2*bit-1) = fma(2d, bit_float, -d)
+            const vec4 bit_floats = vec4(
+                float(bits & 1u), float((bits >> 1) & 1u),
+                float((bits >> 2) & 1u), float((bits >> 3) & 1u)
+            );
+            const vec4 v = fma(vec4(d2), bit_floats, vec4(neg_d));
+
             buf_a[buf_idx    ] = FLOAT_TYPE_VEC2(v.xy);
             buf_a[buf_idx + 1] = FLOAT_TYPE_VEC2(v.zw);
 #elif defined(DATA_A_Q2_K)

ggml/src/ggml-vulkan/vulkan-shaders/types.glsl

@@ -5,6 +5,7 @@
 #extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
 #extension GL_EXT_shader_explicit_arithmetic_types_int16 : require
 #extension GL_EXT_shader_explicit_arithmetic_types_int8 : require
+#extension GL_EXT_shader_8bit_storage : require
 #extension GL_EXT_shader_16bit_storage : require
 
 #if defined(DATA_A_F32)
@@ -46,6 +47,7 @@
 #endif
 #endif
 
+#define QUANT_K_Q1_0 128#define QUANT_R_Q1_0 1struct block_q1_0{    float16_t d;    uint8_t qs[16];};#if defined(DATA_A_Q1_0)#define QUANT_K QUANT_K_Q1_0#define QUANT_R QUANT_R_Q1_0#define QUANT_AUXF 1#define A_TYPE block_q1_0#define DATA_A_QUANT_LEGACY#endif
 #define QUANT_K_Q4_0 32
 #define QUANT_R_Q4_0 2