From 9223ce5889427112716b2dd03a59943bf238bddb Mon Sep 17 00:00:00 2001 From: Zonglin Peng Date: Mon, 6 Jul 2026 18:57:08 -0700 Subject: [PATCH 1/2] Inline per-tensor SIMD fast path in fusion_g3 op_dequantize (#20499) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: When the input and output buffers are 16-byte aligned (`dequant_simd_aligned`), the per-tensor path runs an inline PDX SIMD loop (`xb_vecMxf32`/`xb_vecMx32`/`PDX_MUL_MXF32`); otherwise it falls back to the NNLib path (`xa_nn_elm_dequantize_*`). The result is numerically identical to the original op — the same float-domain affine `(x - zero_point) * scale`. The macro fast paths (`ASYM_DEQUANTIZE_IMPL_CHANNEL`/`SYM_DEQUANTIZE_IMPL_CHANNEL`) get the `static_cast((x - zp) * scale)` parenthesization required to build clean under the G3 `dev` mode's `-Werror,-Wdouble-promotion`. For A/B measurement this also adds `op_dequantize_baseline.cpp` under the Jarvis operator test dir: a benchmark-only snapshot of the ORIGINAL executorch op (pre-SIMD, with only the `-Wdouble-promotion` fix). It defines `impl::G3::native::dequantize_per_tensor_out`, compared on the cycle-accurate G3 ISS. `operators_header` visibility is extended to the Jarvis test package so the snapshot can include `operators.h`. Reviewed By: mvartani-meta Differential Revision: D109500113 --- .../fusion_g3/operators/op_dequantize.cpp | 415 +++++++++++++++++- 1 file changed, 400 insertions(+), 15 deletions(-) diff --git a/backends/cadence/fusion_g3/operators/op_dequantize.cpp b/backends/cadence/fusion_g3/operators/op_dequantize.cpp index c940cdcf839..821df71d994 100644 --- a/backends/cadence/fusion_g3/operators/op_dequantize.cpp +++ b/backends/cadence/fusion_g3/operators/op_dequantize.cpp @@ -47,7 +47,7 @@ namespace { /** * Asserts that the parameters are valid. */ -void check_dequantize_per_tensor_args( +[[maybe_unused]] void check_dequantize_per_tensor_args( const Tensor& input, int64_t quant_min, int64_t quant_max, @@ -86,6 +86,18 @@ void check_dequantize_per_tensor_args( quant_max); } +// PDX aligned vector load/store (PDX_LV*) require 16-byte (128-bit) aligned +// buffers on Fusion G3; an unaligned access raises LoadStoreAlignmentCause. The +// inline SIMD fast path below is only safe when both buffers are aligned; +// otherwise the caller falls back to the alignment-safe NNLib kernel. +[[maybe_unused]] constexpr uintptr_t kSimdAlignmentBytes = 16; +[[maybe_unused]] inline bool dequant_simd_aligned( + const void* in_ptr, + const void* out_ptr) { + return (reinterpret_cast(in_ptr) % kSimdAlignmentBytes == 0) && + (reinterpret_cast(out_ptr) % kSimdAlignmentBytes == 0); +} + } // namespace /* Local function which calls the kernels based on the input datatype */ @@ -295,10 +307,8 @@ Tensor& dequantize_impl( if (zero_point_data != nullptr) { \ zero_point = zero_point_data[current_ix]; \ } \ - out_data_ptr[current_ix] = \ - static_cast( \ - input_data_ptr[current_ix] - zero_point) * \ - _scale; \ + out_data_ptr[current_ix] = static_cast( \ + (input_data_ptr[current_ix] - zero_point) * _scale); \ } \ }, \ input, \ @@ -503,10 +513,8 @@ Tensor& dequantize_impl( if (zero_point_data != nullptr) { \ zero_point = zero_point_data[current_ix]; \ } \ - out_data_ptr[current_ix] = \ - static_cast( \ - input_data_ptr[current_ix] - zero_point) * \ - _scale; \ + out_data_ptr[current_ix] = static_cast( \ + (input_data_ptr[current_ix] - zero_point) * _scale); \ } \ }, \ input, \ @@ -579,26 +587,403 @@ Tensor& dequantize_per_tensor_out( ScalarType dtype, Tensor& out) { constexpr ScalarType out_dtype = ScalarType::Float; - #ifdef OP_ARG_CHECK torch::executor::Error err = resize_tensor(out, input.sizes()); ET_CHECK_MSG( err == torch::executor::Error::Ok, "Failed to resize out Tensor in dequantize_per_tensor_out"); - check_dequantize_per_tensor_args( input, quant_min, quant_max, dtype, out_dtype, out); #endif - float scale_data = (float)scale; int zero_point_data = (int)zero_point; - + // Fast path: bypass dequantize_impl dispatch overhead for the most common + // per-tensor cases (int8/uint8 -> float32). This avoids the inp_shape copy + // loop, is_asym_dequant check, optimized flag check, and the multi-branch + // type dispatch in dequantize_impl. + if (out.scalar_type() == ScalarType::Float) { + float* __restrict__ out_data = out.mutable_data_ptr(); + // For per-tensor dequantization (axis=NULL), flatten to 1D shape. + // This avoids the shape copy loop and lets the NNLIB kernel skip + // multi-dimensional stride calculations internally. + int inp_shape[1] = {static_cast(input.numel())}; + constexpr int num_inp_dims = 1; + if (input.scalar_type() == ScalarType::Char) { + const int8_t* __restrict__ input_data = input.const_data_ptr(); +#if defined(__XTENSA__) + // Direct inline PDX SIMD dequantization for per-tensor int8->float32. + // 4x-unrolled to improve ILP: the Fusion G3 scheduler can interleave + // independent loads, converts, and FP MACs across unrolled iterations, + // hiding load-to-use latency and reducing loop overhead. + if (dequant_simd_aligned(input_data, out_data)) { + const int numel = inp_shape[0]; + auto vIn = reinterpret_cast(input_data); + auto vOut = reinterpret_cast(out_data); + const xb_vecMxf32 v_scale{ + scale_data, scale_data, scale_data, scale_data}; + int i = 0; + if (zero_point_data != 0) { + const float zp_f = static_cast(zero_point_data); + const xb_vecMxf32 v_zp{zp_f, zp_f, zp_f, zp_f}; + // 4x unrolled main loop: 16 elements per iteration + const int e16 = (numel >> 4) << 4; + for (; i < e16; i += 16) { + xb_vecMx32 vP0 = PDX_LV32_MX8_I(vIn, 0); + xb_vecMx32 vP1 = PDX_LV32_MX8_I(vIn + 1, 0); + xb_vecMx32 vP2 = PDX_LV32_MX8_I(vIn + 2, 0); + xb_vecMx32 vP3 = PDX_LV32_MX8_I(vIn + 3, 0); + xb_vecMxf32 vF0 = (xb_vecMxf32)vP0; + xb_vecMxf32 vF1 = (xb_vecMxf32)vP1; + xb_vecMxf32 vF2 = (xb_vecMxf32)vP2; + xb_vecMxf32 vF3 = (xb_vecMxf32)vP3; + vOut[0] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF0, v_zp), v_scale); + vOut[1] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF1, v_zp), v_scale); + vOut[2] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF2, v_zp), v_scale); + vOut[3] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF3, v_zp), v_scale); + vIn += 4; + vOut += 4; + } + // Scalar residual for remaining 0-15 elements + for (; i < numel; ++i) { + out_data[i] = + (static_cast(input_data[i]) - zp_f) * scale_data; + } + } else { + // 4x unrolled main loop: 16 elements per iteration + const int e16 = (numel >> 4) << 4; + for (; i < e16; i += 16) { + xb_vecMx32 vP0 = PDX_LV32_MX8_I(vIn, 0); + xb_vecMx32 vP1 = PDX_LV32_MX8_I(vIn + 1, 0); + xb_vecMx32 vP2 = PDX_LV32_MX8_I(vIn + 2, 0); + xb_vecMx32 vP3 = PDX_LV32_MX8_I(vIn + 3, 0); + xb_vecMxf32 vF0 = (xb_vecMxf32)vP0; + xb_vecMxf32 vF1 = (xb_vecMxf32)vP1; + xb_vecMxf32 vF2 = (xb_vecMxf32)vP2; + xb_vecMxf32 vF3 = (xb_vecMxf32)vP3; + vOut[0] = PDX_MUL_MXF32(vF0, v_scale); + vOut[1] = PDX_MUL_MXF32(vF1, v_scale); + vOut[2] = PDX_MUL_MXF32(vF2, v_scale); + vOut[3] = PDX_MUL_MXF32(vF3, v_scale); + vIn += 4; + vOut += 4; + } + // Scalar residual for remaining 0-15 elements + for (; i < numel; ++i) { + out_data[i] = static_cast(input_data[i]) * scale_data; + } + } + return out; + } +#endif + if (zero_point_data != 0) { + XT_KERNEL_CHECK( + context, + out, + xa_nn_elm_dequantize_asym8_f32, + out_data, + input_data, + inp_shape, + num_inp_dims, + NULL, + &zero_point_data, + &scale_data); + } else { + XT_KERNEL_CHECK( + context, + out, + xa_nn_elm_dequantize_sym8_f32, + out_data, + input_data, + inp_shape, + num_inp_dims, + NULL, + &scale_data); + } + return out; + } else if (input.scalar_type() == ScalarType::Byte) { + const uint8_t* __restrict__ input_data = input.const_data_ptr(); +#if defined(__XTENSA__) + // Direct inline PDX SIMD dequantization for per-tensor uint8->float32. + // 4x-unrolled to improve ILP: the Fusion G3 scheduler can interleave + // independent loads, converts, and FP MACs across unrolled iterations, + // hiding load-to-use latency and reducing loop overhead. + if (dequant_simd_aligned(input_data, out_data)) { + const int numel = inp_shape[0]; + auto vIn = reinterpret_cast(input_data); + auto vOut = reinterpret_cast(out_data); + const xb_vecMxf32 v_scale{ + scale_data, scale_data, scale_data, scale_data}; + int i = 0; + if (zero_point_data != 0) { + const float zp_f = static_cast(zero_point_data); + const xb_vecMxf32 v_zp{zp_f, zp_f, zp_f, zp_f}; + // 4x unrolled main loop: 16 elements per iteration + const int e16 = (numel >> 4) << 4; + for (; i < e16; i += 16) { + xb_vecMxu32 vP0 = PDX_LVU32_MX8_I(vIn, 0); + xb_vecMxu32 vP1 = PDX_LVU32_MX8_I(vIn + 1, 0); + xb_vecMxu32 vP2 = PDX_LVU32_MX8_I(vIn + 2, 0); + xb_vecMxu32 vP3 = PDX_LVU32_MX8_I(vIn + 3, 0); + xb_vecMxf32 vF0 = (xb_vecMxf32)vP0; + xb_vecMxf32 vF1 = (xb_vecMxf32)vP1; + xb_vecMxf32 vF2 = (xb_vecMxf32)vP2; + xb_vecMxf32 vF3 = (xb_vecMxf32)vP3; + vOut[0] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF0, v_zp), v_scale); + vOut[1] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF1, v_zp), v_scale); + vOut[2] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF2, v_zp), v_scale); + vOut[3] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF3, v_zp), v_scale); + vIn += 4; + vOut += 4; + } + // Scalar residual for remaining 0-15 elements + for (; i < numel; ++i) { + out_data[i] = + (static_cast(input_data[i]) - zp_f) * scale_data; + } + } else { + // 4x unrolled main loop: 16 elements per iteration + const int e16 = (numel >> 4) << 4; + for (; i < e16; i += 16) { + xb_vecMxu32 vP0 = PDX_LVU32_MX8_I(vIn, 0); + xb_vecMxu32 vP1 = PDX_LVU32_MX8_I(vIn + 1, 0); + xb_vecMxu32 vP2 = PDX_LVU32_MX8_I(vIn + 2, 0); + xb_vecMxu32 vP3 = PDX_LVU32_MX8_I(vIn + 3, 0); + xb_vecMxf32 vF0 = (xb_vecMxf32)vP0; + xb_vecMxf32 vF1 = (xb_vecMxf32)vP1; + xb_vecMxf32 vF2 = (xb_vecMxf32)vP2; + xb_vecMxf32 vF3 = (xb_vecMxf32)vP3; + vOut[0] = PDX_MUL_MXF32(vF0, v_scale); + vOut[1] = PDX_MUL_MXF32(vF1, v_scale); + vOut[2] = PDX_MUL_MXF32(vF2, v_scale); + vOut[3] = PDX_MUL_MXF32(vF3, v_scale); + vIn += 4; + vOut += 4; + } + // Scalar residual for remaining 0-15 elements + for (; i < numel; ++i) { + out_data[i] = static_cast(input_data[i]) * scale_data; + } + } + return out; + } +#endif + if (zero_point_data != 0) { + XT_KERNEL_CHECK( + context, + out, + xa_nn_elm_dequantize_asym8u_f32, + out_data, + input_data, + inp_shape, + num_inp_dims, + NULL, + &zero_point_data, + &scale_data); + } else { + XT_KERNEL_CHECK( + context, + out, + xa_nn_elm_dequantize_sym8u_f32, + out_data, + input_data, + inp_shape, + num_inp_dims, + NULL, + &scale_data); + } + return out; + } else if (input.scalar_type() == ScalarType::Short) { + const int16_t* __restrict__ input_data = input.const_data_ptr(); +#if defined(__XTENSA__) + // Direct inline PDX SIMD dequantization for per-tensor int16->float32. + // 4x-unrolled to improve ILP: the Fusion G3 scheduler can interleave + // independent loads, converts, and FP MACs across unrolled iterations, + // hiding load-to-use latency and reducing loop overhead. + if (dequant_simd_aligned(input_data, out_data)) { + const int numel = inp_shape[0]; + auto vIn = reinterpret_cast(input_data); + auto vOut = reinterpret_cast(out_data); + const xb_vecMxf32 v_scale{ + scale_data, scale_data, scale_data, scale_data}; + int i = 0; + if (zero_point_data != 0) { + const float zp_f = static_cast(zero_point_data); + const xb_vecMxf32 v_zp{zp_f, zp_f, zp_f, zp_f}; + // 4x unrolled main loop: 16 elements per iteration + const int e16 = (numel >> 4) << 4; + for (; i < e16; i += 16) { + xb_vecMx32 vP0 = PDX_LV32_MX16_I(vIn, 0); + xb_vecMx32 vP1 = PDX_LV32_MX16_I(vIn + 1, 0); + xb_vecMx32 vP2 = PDX_LV32_MX16_I(vIn + 2, 0); + xb_vecMx32 vP3 = PDX_LV32_MX16_I(vIn + 3, 0); + xb_vecMxf32 vF0 = (xb_vecMxf32)vP0; + xb_vecMxf32 vF1 = (xb_vecMxf32)vP1; + xb_vecMxf32 vF2 = (xb_vecMxf32)vP2; + xb_vecMxf32 vF3 = (xb_vecMxf32)vP3; + vOut[0] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF0, v_zp), v_scale); + vOut[1] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF1, v_zp), v_scale); + vOut[2] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF2, v_zp), v_scale); + vOut[3] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF3, v_zp), v_scale); + vIn += 4; + vOut += 4; + } + // Scalar residual for remaining 0-15 elements + for (; i < numel; ++i) { + out_data[i] = + (static_cast(input_data[i]) - zp_f) * scale_data; + } + } else { + // 4x unrolled main loop: 16 elements per iteration + const int e16 = (numel >> 4) << 4; + for (; i < e16; i += 16) { + xb_vecMx32 vP0 = PDX_LV32_MX16_I(vIn, 0); + xb_vecMx32 vP1 = PDX_LV32_MX16_I(vIn + 1, 0); + xb_vecMx32 vP2 = PDX_LV32_MX16_I(vIn + 2, 0); + xb_vecMx32 vP3 = PDX_LV32_MX16_I(vIn + 3, 0); + xb_vecMxf32 vF0 = (xb_vecMxf32)vP0; + xb_vecMxf32 vF1 = (xb_vecMxf32)vP1; + xb_vecMxf32 vF2 = (xb_vecMxf32)vP2; + xb_vecMxf32 vF3 = (xb_vecMxf32)vP3; + vOut[0] = PDX_MUL_MXF32(vF0, v_scale); + vOut[1] = PDX_MUL_MXF32(vF1, v_scale); + vOut[2] = PDX_MUL_MXF32(vF2, v_scale); + vOut[3] = PDX_MUL_MXF32(vF3, v_scale); + vIn += 4; + vOut += 4; + } + // Scalar residual for remaining 0-15 elements + for (; i < numel; ++i) { + out_data[i] = static_cast(input_data[i]) * scale_data; + } + } + return out; + } +#endif + if (zero_point_data != 0) { + XT_KERNEL_CHECK( + context, + out, + xa_nn_elm_dequantize_asym16_f32, + out_data, + input_data, + inp_shape, + num_inp_dims, + NULL, + &zero_point_data, + &scale_data); + } else { + XT_KERNEL_CHECK( + context, + out, + xa_nn_elm_dequantize_sym16_f32, + out_data, + input_data, + inp_shape, + num_inp_dims, + NULL, + &scale_data); + } + return out; + } else if (input.scalar_type() == ScalarType::UInt16) { + const uint16_t* __restrict__ input_data = + input.const_data_ptr(); +#if defined(__XTENSA__) + // Direct inline PDX SIMD dequantization for per-tensor uint16->float32. + // 4x-unrolled to improve ILP: the Fusion G3 scheduler can interleave + // independent loads, converts, and FP MACs across unrolled iterations, + // hiding load-to-use latency and reducing loop overhead. + if (dequant_simd_aligned(input_data, out_data)) { + const int numel = inp_shape[0]; + auto vIn = reinterpret_cast(input_data); + auto vOut = reinterpret_cast(out_data); + const xb_vecMxf32 v_scale{ + scale_data, scale_data, scale_data, scale_data}; + int i = 0; + if (zero_point_data != 0) { + const float zp_f = static_cast(zero_point_data); + const xb_vecMxf32 v_zp{zp_f, zp_f, zp_f, zp_f}; + // 4x unrolled main loop: 16 elements per iteration + const int e16 = (numel >> 4) << 4; + for (; i < e16; i += 16) { + xb_vecMxu32 vP0 = PDX_LVU32_MX16_I(vIn, 0); + xb_vecMxu32 vP1 = PDX_LVU32_MX16_I(vIn + 1, 0); + xb_vecMxu32 vP2 = PDX_LVU32_MX16_I(vIn + 2, 0); + xb_vecMxu32 vP3 = PDX_LVU32_MX16_I(vIn + 3, 0); + xb_vecMxf32 vF0 = (xb_vecMxf32)vP0; + xb_vecMxf32 vF1 = (xb_vecMxf32)vP1; + xb_vecMxf32 vF2 = (xb_vecMxf32)vP2; + xb_vecMxf32 vF3 = (xb_vecMxf32)vP3; + vOut[0] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF0, v_zp), v_scale); + vOut[1] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF1, v_zp), v_scale); + vOut[2] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF2, v_zp), v_scale); + vOut[3] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF3, v_zp), v_scale); + vIn += 4; + vOut += 4; + } + // Scalar residual for remaining 0-15 elements + for (; i < numel; ++i) { + out_data[i] = + (static_cast(input_data[i]) - zp_f) * scale_data; + } + } else { + // 4x unrolled main loop: 16 elements per iteration + const int e16 = (numel >> 4) << 4; + for (; i < e16; i += 16) { + xb_vecMxu32 vP0 = PDX_LVU32_MX16_I(vIn, 0); + xb_vecMxu32 vP1 = PDX_LVU32_MX16_I(vIn + 1, 0); + xb_vecMxu32 vP2 = PDX_LVU32_MX16_I(vIn + 2, 0); + xb_vecMxu32 vP3 = PDX_LVU32_MX16_I(vIn + 3, 0); + xb_vecMxf32 vF0 = (xb_vecMxf32)vP0; + xb_vecMxf32 vF1 = (xb_vecMxf32)vP1; + xb_vecMxf32 vF2 = (xb_vecMxf32)vP2; + xb_vecMxf32 vF3 = (xb_vecMxf32)vP3; + vOut[0] = PDX_MUL_MXF32(vF0, v_scale); + vOut[1] = PDX_MUL_MXF32(vF1, v_scale); + vOut[2] = PDX_MUL_MXF32(vF2, v_scale); + vOut[3] = PDX_MUL_MXF32(vF3, v_scale); + vIn += 4; + vOut += 4; + } + // Scalar residual for remaining 0-15 elements + for (; i < numel; ++i) { + out_data[i] = static_cast(input_data[i]) * scale_data; + } + } + return out; + } +#endif + if (zero_point_data != 0) { + XT_KERNEL_CHECK( + context, + out, + xa_nn_elm_dequantize_asym16u_f32, + out_data, + input_data, + inp_shape, + num_inp_dims, + NULL, + &zero_point_data, + &scale_data); + } else { + XT_KERNEL_CHECK( + context, + out, + xa_nn_elm_dequantize_sym16u_f32, + out_data, + input_data, + inp_shape, + num_inp_dims, + NULL, + &scale_data); + } + return out; + } + // Fall through to generic path for other input types + } dequantize_impl( context, out, input, &scale_data, &zero_point_data, NULL, out_dtype); - return out; } - Tensor& dequantize_per_tensor_tensor_args_out( KernelRuntimeContext& context, const Tensor& input, From a1acc905885d733e6f5f8c0f778f42eaab6d0504 Mon Sep 17 00:00:00 2001 From: Zonglin Peng Date: Mon, 6 Jul 2026 18:57:08 -0700 Subject: [PATCH 2/2] AI: [G3] dequantize_per_tensor asym SIMD: integer zero-point subtract (#20703) Summary: Optimizes the per-tensor `dequantize_per_tensor_out` SIMD fast path on Fusion-G3 for int8 / uint8 / int16 / uint16 -> float32. What is optimized: the zero-point subtract is kept in the int32 domain (`PDX_SUB_MX32`) so it issues on the integer unit and overlaps the float pipe, and the widened int32 codes are fed straight into the mixed-type `PDX_MUL_MXF32(int32, scale)` (the idiom the vendor NNLib dequantize kernel uses). Note: `PDX_MUL_MXF32` takes an `xb_vecMxf32`, so the compiler still emits an int->float convert -- writing the multiply mixed-type keeps the source concise but does not eliminate the convert (thanks mvartani-meta for the correction). The measured win comes from keeping the subtract in the integer domain instead of the float domain, which frees the float pipe and lets the compiler schedule the convert as the multiply's operand rather than as a separate explicit cast. Bit-identical to the previous code for 8/16-bit inputs: `q - zp` cannot overflow int32 (inputs are <=16-bit and zp is in the same quant range) and converts exactly to float, matching `float(q) - float(zp)`. What is improved from 00b95cf2b4: that revision moved the subtract into the integer domain but kept an explicit `(xb_vecMxf32)` convert, so the asymmetric FP pipe still did an explicit convert + multiply and the symmetric path was left untouched. This diff feeds the int32 codes directly into the mixed-type multiply on both paths. That clears the residual large-tensor asymmetric regression 00b95cf2b4 still had -- e.g. `objv1_1x400x400x1_u8` (160000 elems) goes 0.75x -> 1.00x vs stock and `dpev26_4x60x4x4_u16` goes 0.89x -> 1.17x -- while keeping every symmetric case at parity or better, and the object file is ~144 B smaller. Reviewed By: mvartani-meta, DrJessop Differential Revision: D110529287 --- .../fusion_g3/operators/op_dequantize.cpp | 192 +++++++++--------- 1 file changed, 96 insertions(+), 96 deletions(-) diff --git a/backends/cadence/fusion_g3/operators/op_dequantize.cpp b/backends/cadence/fusion_g3/operators/op_dequantize.cpp index 821df71d994..ac88528127b 100644 --- a/backends/cadence/fusion_g3/operators/op_dequantize.cpp +++ b/backends/cadence/fusion_g3/operators/op_dequantize.cpp @@ -612,9 +612,15 @@ Tensor& dequantize_per_tensor_out( const int8_t* __restrict__ input_data = input.const_data_ptr(); #if defined(__XTENSA__) // Direct inline PDX SIMD dequantization for per-tensor int8->float32. - // 4x-unrolled to improve ILP: the Fusion G3 scheduler can interleave - // independent loads, converts, and FP MACs across unrolled iterations, - // hiding load-to-use latency and reducing loop overhead. + // The zero-point subtract is kept in the int32 domain (PDX_SUB_MX32) so + // it issues on the integer unit and overlaps the float pipe; the widened + // int32 codes feed straight into the mixed-type PDX_MUL_MXF32. That + // intrinsic takes an xb_vecMxf32, so the compiler still emits an + // int->float convert -- writing it mixed-type only keeps the source + // concise, it does not remove the convert. q - zp cannot overflow int32 + // (inputs are <=16-bit and zp is in the same quant range) and converts + // exactly to float, so the result is bit-identical to + // float(q) - float(zp). 4x-unrolled to hide load-to-use latency. if (dequant_simd_aligned(input_data, out_data)) { const int numel = inp_shape[0]; auto vIn = reinterpret_cast(input_data); @@ -622,24 +628,24 @@ Tensor& dequantize_per_tensor_out( const xb_vecMxf32 v_scale{ scale_data, scale_data, scale_data, scale_data}; int i = 0; + // 4x unrolled main loop: 16 elements per iteration + const int e16 = (numel >> 4) << 4; if (zero_point_data != 0) { + const xb_vecMx32 v_zp{ + zero_point_data, + zero_point_data, + zero_point_data, + zero_point_data}; const float zp_f = static_cast(zero_point_data); - const xb_vecMxf32 v_zp{zp_f, zp_f, zp_f, zp_f}; - // 4x unrolled main loop: 16 elements per iteration - const int e16 = (numel >> 4) << 4; for (; i < e16; i += 16) { xb_vecMx32 vP0 = PDX_LV32_MX8_I(vIn, 0); xb_vecMx32 vP1 = PDX_LV32_MX8_I(vIn + 1, 0); xb_vecMx32 vP2 = PDX_LV32_MX8_I(vIn + 2, 0); xb_vecMx32 vP3 = PDX_LV32_MX8_I(vIn + 3, 0); - xb_vecMxf32 vF0 = (xb_vecMxf32)vP0; - xb_vecMxf32 vF1 = (xb_vecMxf32)vP1; - xb_vecMxf32 vF2 = (xb_vecMxf32)vP2; - xb_vecMxf32 vF3 = (xb_vecMxf32)vP3; - vOut[0] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF0, v_zp), v_scale); - vOut[1] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF1, v_zp), v_scale); - vOut[2] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF2, v_zp), v_scale); - vOut[3] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF3, v_zp), v_scale); + vOut[0] = PDX_MUL_MXF32(PDX_SUB_MX32(vP0, v_zp), v_scale); + vOut[1] = PDX_MUL_MXF32(PDX_SUB_MX32(vP1, v_zp), v_scale); + vOut[2] = PDX_MUL_MXF32(PDX_SUB_MX32(vP2, v_zp), v_scale); + vOut[3] = PDX_MUL_MXF32(PDX_SUB_MX32(vP3, v_zp), v_scale); vIn += 4; vOut += 4; } @@ -649,21 +655,15 @@ Tensor& dequantize_per_tensor_out( (static_cast(input_data[i]) - zp_f) * scale_data; } } else { - // 4x unrolled main loop: 16 elements per iteration - const int e16 = (numel >> 4) << 4; for (; i < e16; i += 16) { xb_vecMx32 vP0 = PDX_LV32_MX8_I(vIn, 0); xb_vecMx32 vP1 = PDX_LV32_MX8_I(vIn + 1, 0); xb_vecMx32 vP2 = PDX_LV32_MX8_I(vIn + 2, 0); xb_vecMx32 vP3 = PDX_LV32_MX8_I(vIn + 3, 0); - xb_vecMxf32 vF0 = (xb_vecMxf32)vP0; - xb_vecMxf32 vF1 = (xb_vecMxf32)vP1; - xb_vecMxf32 vF2 = (xb_vecMxf32)vP2; - xb_vecMxf32 vF3 = (xb_vecMxf32)vP3; - vOut[0] = PDX_MUL_MXF32(vF0, v_scale); - vOut[1] = PDX_MUL_MXF32(vF1, v_scale); - vOut[2] = PDX_MUL_MXF32(vF2, v_scale); - vOut[3] = PDX_MUL_MXF32(vF3, v_scale); + vOut[0] = PDX_MUL_MXF32(vP0, v_scale); + vOut[1] = PDX_MUL_MXF32(vP1, v_scale); + vOut[2] = PDX_MUL_MXF32(vP2, v_scale); + vOut[3] = PDX_MUL_MXF32(vP3, v_scale); vIn += 4; vOut += 4; } @@ -704,9 +704,15 @@ Tensor& dequantize_per_tensor_out( const uint8_t* __restrict__ input_data = input.const_data_ptr(); #if defined(__XTENSA__) // Direct inline PDX SIMD dequantization for per-tensor uint8->float32. - // 4x-unrolled to improve ILP: the Fusion G3 scheduler can interleave - // independent loads, converts, and FP MACs across unrolled iterations, - // hiding load-to-use latency and reducing loop overhead. + // The zero-point subtract is kept in the int32 domain (PDX_SUB_MX32) so + // it issues on the integer unit and overlaps the float pipe; the widened + // int32 codes feed straight into the mixed-type PDX_MUL_MXF32. That + // intrinsic takes an xb_vecMxf32, so the compiler still emits an + // int->float convert -- writing it mixed-type only keeps the source + // concise, it does not remove the convert. q - zp cannot overflow int32 + // (inputs are <=16-bit and zp is in the same quant range) and converts + // exactly to float, so the result is bit-identical to + // float(q) - float(zp). 4x-unrolled to hide load-to-use latency. if (dequant_simd_aligned(input_data, out_data)) { const int numel = inp_shape[0]; auto vIn = reinterpret_cast(input_data); @@ -714,24 +720,24 @@ Tensor& dequantize_per_tensor_out( const xb_vecMxf32 v_scale{ scale_data, scale_data, scale_data, scale_data}; int i = 0; + // 4x unrolled main loop: 16 elements per iteration + const int e16 = (numel >> 4) << 4; if (zero_point_data != 0) { + const xb_vecMxu32 v_zp{ + static_cast(zero_point_data), + static_cast(zero_point_data), + static_cast(zero_point_data), + static_cast(zero_point_data)}; const float zp_f = static_cast(zero_point_data); - const xb_vecMxf32 v_zp{zp_f, zp_f, zp_f, zp_f}; - // 4x unrolled main loop: 16 elements per iteration - const int e16 = (numel >> 4) << 4; for (; i < e16; i += 16) { xb_vecMxu32 vP0 = PDX_LVU32_MX8_I(vIn, 0); xb_vecMxu32 vP1 = PDX_LVU32_MX8_I(vIn + 1, 0); xb_vecMxu32 vP2 = PDX_LVU32_MX8_I(vIn + 2, 0); xb_vecMxu32 vP3 = PDX_LVU32_MX8_I(vIn + 3, 0); - xb_vecMxf32 vF0 = (xb_vecMxf32)vP0; - xb_vecMxf32 vF1 = (xb_vecMxf32)vP1; - xb_vecMxf32 vF2 = (xb_vecMxf32)vP2; - xb_vecMxf32 vF3 = (xb_vecMxf32)vP3; - vOut[0] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF0, v_zp), v_scale); - vOut[1] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF1, v_zp), v_scale); - vOut[2] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF2, v_zp), v_scale); - vOut[3] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF3, v_zp), v_scale); + vOut[0] = PDX_MUL_MXF32(PDX_SUB_MX32(vP0, v_zp), v_scale); + vOut[1] = PDX_MUL_MXF32(PDX_SUB_MX32(vP1, v_zp), v_scale); + vOut[2] = PDX_MUL_MXF32(PDX_SUB_MX32(vP2, v_zp), v_scale); + vOut[3] = PDX_MUL_MXF32(PDX_SUB_MX32(vP3, v_zp), v_scale); vIn += 4; vOut += 4; } @@ -741,21 +747,15 @@ Tensor& dequantize_per_tensor_out( (static_cast(input_data[i]) - zp_f) * scale_data; } } else { - // 4x unrolled main loop: 16 elements per iteration - const int e16 = (numel >> 4) << 4; for (; i < e16; i += 16) { xb_vecMxu32 vP0 = PDX_LVU32_MX8_I(vIn, 0); xb_vecMxu32 vP1 = PDX_LVU32_MX8_I(vIn + 1, 0); xb_vecMxu32 vP2 = PDX_LVU32_MX8_I(vIn + 2, 0); xb_vecMxu32 vP3 = PDX_LVU32_MX8_I(vIn + 3, 0); - xb_vecMxf32 vF0 = (xb_vecMxf32)vP0; - xb_vecMxf32 vF1 = (xb_vecMxf32)vP1; - xb_vecMxf32 vF2 = (xb_vecMxf32)vP2; - xb_vecMxf32 vF3 = (xb_vecMxf32)vP3; - vOut[0] = PDX_MUL_MXF32(vF0, v_scale); - vOut[1] = PDX_MUL_MXF32(vF1, v_scale); - vOut[2] = PDX_MUL_MXF32(vF2, v_scale); - vOut[3] = PDX_MUL_MXF32(vF3, v_scale); + vOut[0] = PDX_MUL_MXF32(vP0, v_scale); + vOut[1] = PDX_MUL_MXF32(vP1, v_scale); + vOut[2] = PDX_MUL_MXF32(vP2, v_scale); + vOut[3] = PDX_MUL_MXF32(vP3, v_scale); vIn += 4; vOut += 4; } @@ -796,9 +796,15 @@ Tensor& dequantize_per_tensor_out( const int16_t* __restrict__ input_data = input.const_data_ptr(); #if defined(__XTENSA__) // Direct inline PDX SIMD dequantization for per-tensor int16->float32. - // 4x-unrolled to improve ILP: the Fusion G3 scheduler can interleave - // independent loads, converts, and FP MACs across unrolled iterations, - // hiding load-to-use latency and reducing loop overhead. + // The zero-point subtract is kept in the int32 domain (PDX_SUB_MX32) so + // it issues on the integer unit and overlaps the float pipe; the widened + // int32 codes feed straight into the mixed-type PDX_MUL_MXF32. That + // intrinsic takes an xb_vecMxf32, so the compiler still emits an + // int->float convert -- writing it mixed-type only keeps the source + // concise, it does not remove the convert. q - zp cannot overflow int32 + // (inputs are <=16-bit and zp is in the same quant range) and converts + // exactly to float, so the result is bit-identical to + // float(q) - float(zp). 4x-unrolled to hide load-to-use latency. if (dequant_simd_aligned(input_data, out_data)) { const int numel = inp_shape[0]; auto vIn = reinterpret_cast(input_data); @@ -806,24 +812,24 @@ Tensor& dequantize_per_tensor_out( const xb_vecMxf32 v_scale{ scale_data, scale_data, scale_data, scale_data}; int i = 0; + // 4x unrolled main loop: 16 elements per iteration + const int e16 = (numel >> 4) << 4; if (zero_point_data != 0) { + const xb_vecMx32 v_zp{ + zero_point_data, + zero_point_data, + zero_point_data, + zero_point_data}; const float zp_f = static_cast(zero_point_data); - const xb_vecMxf32 v_zp{zp_f, zp_f, zp_f, zp_f}; - // 4x unrolled main loop: 16 elements per iteration - const int e16 = (numel >> 4) << 4; for (; i < e16; i += 16) { xb_vecMx32 vP0 = PDX_LV32_MX16_I(vIn, 0); xb_vecMx32 vP1 = PDX_LV32_MX16_I(vIn + 1, 0); xb_vecMx32 vP2 = PDX_LV32_MX16_I(vIn + 2, 0); xb_vecMx32 vP3 = PDX_LV32_MX16_I(vIn + 3, 0); - xb_vecMxf32 vF0 = (xb_vecMxf32)vP0; - xb_vecMxf32 vF1 = (xb_vecMxf32)vP1; - xb_vecMxf32 vF2 = (xb_vecMxf32)vP2; - xb_vecMxf32 vF3 = (xb_vecMxf32)vP3; - vOut[0] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF0, v_zp), v_scale); - vOut[1] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF1, v_zp), v_scale); - vOut[2] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF2, v_zp), v_scale); - vOut[3] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF3, v_zp), v_scale); + vOut[0] = PDX_MUL_MXF32(PDX_SUB_MX32(vP0, v_zp), v_scale); + vOut[1] = PDX_MUL_MXF32(PDX_SUB_MX32(vP1, v_zp), v_scale); + vOut[2] = PDX_MUL_MXF32(PDX_SUB_MX32(vP2, v_zp), v_scale); + vOut[3] = PDX_MUL_MXF32(PDX_SUB_MX32(vP3, v_zp), v_scale); vIn += 4; vOut += 4; } @@ -833,21 +839,15 @@ Tensor& dequantize_per_tensor_out( (static_cast(input_data[i]) - zp_f) * scale_data; } } else { - // 4x unrolled main loop: 16 elements per iteration - const int e16 = (numel >> 4) << 4; for (; i < e16; i += 16) { xb_vecMx32 vP0 = PDX_LV32_MX16_I(vIn, 0); xb_vecMx32 vP1 = PDX_LV32_MX16_I(vIn + 1, 0); xb_vecMx32 vP2 = PDX_LV32_MX16_I(vIn + 2, 0); xb_vecMx32 vP3 = PDX_LV32_MX16_I(vIn + 3, 0); - xb_vecMxf32 vF0 = (xb_vecMxf32)vP0; - xb_vecMxf32 vF1 = (xb_vecMxf32)vP1; - xb_vecMxf32 vF2 = (xb_vecMxf32)vP2; - xb_vecMxf32 vF3 = (xb_vecMxf32)vP3; - vOut[0] = PDX_MUL_MXF32(vF0, v_scale); - vOut[1] = PDX_MUL_MXF32(vF1, v_scale); - vOut[2] = PDX_MUL_MXF32(vF2, v_scale); - vOut[3] = PDX_MUL_MXF32(vF3, v_scale); + vOut[0] = PDX_MUL_MXF32(vP0, v_scale); + vOut[1] = PDX_MUL_MXF32(vP1, v_scale); + vOut[2] = PDX_MUL_MXF32(vP2, v_scale); + vOut[3] = PDX_MUL_MXF32(vP3, v_scale); vIn += 4; vOut += 4; } @@ -889,9 +889,15 @@ Tensor& dequantize_per_tensor_out( input.const_data_ptr(); #if defined(__XTENSA__) // Direct inline PDX SIMD dequantization for per-tensor uint16->float32. - // 4x-unrolled to improve ILP: the Fusion G3 scheduler can interleave - // independent loads, converts, and FP MACs across unrolled iterations, - // hiding load-to-use latency and reducing loop overhead. + // The zero-point subtract is kept in the int32 domain (PDX_SUB_MX32) so + // it issues on the integer unit and overlaps the float pipe; the widened + // int32 codes feed straight into the mixed-type PDX_MUL_MXF32. That + // intrinsic takes an xb_vecMxf32, so the compiler still emits an + // int->float convert -- writing it mixed-type only keeps the source + // concise, it does not remove the convert. q - zp cannot overflow int32 + // (inputs are <=16-bit and zp is in the same quant range) and converts + // exactly to float, so the result is bit-identical to + // float(q) - float(zp). 4x-unrolled to hide load-to-use latency. if (dequant_simd_aligned(input_data, out_data)) { const int numel = inp_shape[0]; auto vIn = reinterpret_cast(input_data); @@ -899,24 +905,24 @@ Tensor& dequantize_per_tensor_out( const xb_vecMxf32 v_scale{ scale_data, scale_data, scale_data, scale_data}; int i = 0; + // 4x unrolled main loop: 16 elements per iteration + const int e16 = (numel >> 4) << 4; if (zero_point_data != 0) { + const xb_vecMxu32 v_zp{ + static_cast(zero_point_data), + static_cast(zero_point_data), + static_cast(zero_point_data), + static_cast(zero_point_data)}; const float zp_f = static_cast(zero_point_data); - const xb_vecMxf32 v_zp{zp_f, zp_f, zp_f, zp_f}; - // 4x unrolled main loop: 16 elements per iteration - const int e16 = (numel >> 4) << 4; for (; i < e16; i += 16) { xb_vecMxu32 vP0 = PDX_LVU32_MX16_I(vIn, 0); xb_vecMxu32 vP1 = PDX_LVU32_MX16_I(vIn + 1, 0); xb_vecMxu32 vP2 = PDX_LVU32_MX16_I(vIn + 2, 0); xb_vecMxu32 vP3 = PDX_LVU32_MX16_I(vIn + 3, 0); - xb_vecMxf32 vF0 = (xb_vecMxf32)vP0; - xb_vecMxf32 vF1 = (xb_vecMxf32)vP1; - xb_vecMxf32 vF2 = (xb_vecMxf32)vP2; - xb_vecMxf32 vF3 = (xb_vecMxf32)vP3; - vOut[0] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF0, v_zp), v_scale); - vOut[1] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF1, v_zp), v_scale); - vOut[2] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF2, v_zp), v_scale); - vOut[3] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF3, v_zp), v_scale); + vOut[0] = PDX_MUL_MXF32(PDX_SUB_MX32(vP0, v_zp), v_scale); + vOut[1] = PDX_MUL_MXF32(PDX_SUB_MX32(vP1, v_zp), v_scale); + vOut[2] = PDX_MUL_MXF32(PDX_SUB_MX32(vP2, v_zp), v_scale); + vOut[3] = PDX_MUL_MXF32(PDX_SUB_MX32(vP3, v_zp), v_scale); vIn += 4; vOut += 4; } @@ -926,21 +932,15 @@ Tensor& dequantize_per_tensor_out( (static_cast(input_data[i]) - zp_f) * scale_data; } } else { - // 4x unrolled main loop: 16 elements per iteration - const int e16 = (numel >> 4) << 4; for (; i < e16; i += 16) { xb_vecMxu32 vP0 = PDX_LVU32_MX16_I(vIn, 0); xb_vecMxu32 vP1 = PDX_LVU32_MX16_I(vIn + 1, 0); xb_vecMxu32 vP2 = PDX_LVU32_MX16_I(vIn + 2, 0); xb_vecMxu32 vP3 = PDX_LVU32_MX16_I(vIn + 3, 0); - xb_vecMxf32 vF0 = (xb_vecMxf32)vP0; - xb_vecMxf32 vF1 = (xb_vecMxf32)vP1; - xb_vecMxf32 vF2 = (xb_vecMxf32)vP2; - xb_vecMxf32 vF3 = (xb_vecMxf32)vP3; - vOut[0] = PDX_MUL_MXF32(vF0, v_scale); - vOut[1] = PDX_MUL_MXF32(vF1, v_scale); - vOut[2] = PDX_MUL_MXF32(vF2, v_scale); - vOut[3] = PDX_MUL_MXF32(vF3, v_scale); + vOut[0] = PDX_MUL_MXF32(vP0, v_scale); + vOut[1] = PDX_MUL_MXF32(vP1, v_scale); + vOut[2] = PDX_MUL_MXF32(vP2, v_scale); + vOut[3] = PDX_MUL_MXF32(vP3, v_scale); vIn += 4; vOut += 4; }