Skip to content

hexagon: extend scalar DMA broadcasting support for binary ops #30245

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
max-krasnyansky merged 9 commits into from
Oct 13, 2026
Merged

hexagon: extend scalar DMA broadcasting support for binary ops #30245

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
6d12149
tests: cover weighted scalar broadcasts
kurquhar Oct 9, 2026
abcfad4
ggml : support extended scalar DMA binary ops
kurquhar Oct 9, 2026
befb82f
hex-binary: remove all non-dma paths
max-krasnyansky Oct 12, 2026
91f5b93
hex-binary: add support for non-contig strides and cleanup dma preabl…
max-krasnyansky Oct 12, 2026
bb8e869
hex-binary: cleanup the gating and simplify dma retry
max-krasnyansky Oct 12, 2026
1e55bf8
hex-geglu-erf: add a gate for F32 only, we dont support F16 for this op
max-krasnyansky Oct 12, 2026
bf33109
tests: cover weighted repeat broadcast
kurquhar Oct 12, 2026
2dd0cc0
hex-binary: update REPEAT and COMPLEX kernels to use dma as well (mis…
max-krasnyansky Oct 13, 2026
d86c81d
hex-add-id: add dma support for src2 (idx)
max-krasnyansky Oct 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 73 additions & 15 deletions ggml/src/ggml-hexagon/ggml-hexagon.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -344,7 +344,8 @@ static bool ggml_hexagon_precompute_binary_params(
const struct ggml_tensor * src0,
const struct ggml_tensor * src1,
const struct ggml_tensor * dst,
struct htp_binary_kernel_params * kparams
struct htp_binary_kernel_params * kparams,
const struct ggml_tensor * src2 = nullptr
);

static void ggml_hexagon_precompute_get_rows_params(
Expand Down Expand Up @@ -5864,7 +5865,8 @@ static bool ggml_hexagon_precompute_binary_params(
const struct ggml_tensor * src0,
const struct ggml_tensor * src1,
const struct ggml_tensor * dst,
struct htp_binary_kernel_params * kparams
struct htp_binary_kernel_params * kparams,
const struct ggml_tensor * src2
) {
memset(kparams, 0, sizeof(*kparams));

Expand All @@ -5877,14 +5879,17 @@ static bool ggml_hexagon_precompute_binary_params(
const size_t src1_row_size_aligned = hex_round_up(src1_row_size, 128);
const size_t dst_row_size_aligned = hex_round_up(dst_row_size, 128);

if (!htp_binary_strides_safe(src0, src1, dst, elem_size)) {
return false;
}

const bool is_add_id = op == HTP_OP_ADD_ID;
const bool is_scalar = !is_add_id && src1->ne[0] == 1;
const bool is_transposed = src0->nb[1] < src0_row_size || src1->nb[1] < src1_row_size || dst->nb[1] < dst_row_size;
const bool is_row_bcast = !is_add_id && !is_scalar && !is_transposed &&
const bool is_row_bcast = !is_add_id && !is_scalar &&
src1->ne[0] == src0->ne[0] &&
(src0->ne[1] > 1 || src0->ne[2] > 1 || src0->ne[3] > 1) &&
src1->ne[1] == 1 && src1->ne[2] == 1 && src1->ne[3] == 1;
const bool is_same_shape = !is_add_id && !is_scalar && !is_transposed &&
const bool is_same_shape = !is_add_id && !is_scalar &&
src1->ne[0] == src0->ne[0] &&
src1->ne[1] == src0->ne[1] &&
(src1->ne[2] == src0->ne[2] || src1->ne[2] == 1) &&
Expand Down Expand Up @@ -5925,28 +5930,42 @@ static bool ggml_hexagon_precompute_binary_params(

enum htp_binary_kernel_type kernel_type;
size_t src1_size = 0;
size_t src2_size = 0;

if (is_add_id) {
GGML_ASSERT(src2 != nullptr);
kernel_type = HTP_BINARY_KERNEL_ADD_ID;
src1_size = hex_round_up(src1->ne[1] * src1_row_size_aligned, 128);
src2_size = hex_round_up(ggml_nelements(src2) * sizeof(int32_t), 128);
if (src1_size + src2_size > sess->vtcm_size) {
return false;
}
} else if (is_row_bcast) {
kernel_type = HTP_BINARY_KERNEL_ROW_BCAST;
src1_size = src1_row_size_aligned;
} else if (is_scalar) {
const bool is_scalar_static = (src1->ne[2] == 1 && src1->ne[3] == 1) &&
(src1->ne[1] == 1 || src1->nb[1] == elem_size);
if (is_scalar_static) {
kernel_type = HTP_BINARY_KERNEL_SCALAR_DMA;
src1_size = hex_round_up(src1->ne[1] * elem_size, 128);
} else {
kernel_type = HTP_BINARY_KERNEL_SCALAR;
src1_size = hex_round_up(ggml_nelements(src1) * elem_size, 128);
if (src1_size > sess->vtcm_size) {
return false;
}
kernel_type = HTP_BINARY_KERNEL_SCALAR;
kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0;
} else if (is_same_shape) {
kernel_type = HTP_BINARY_KERNEL_SAME_SHAPE;
} else if (is_complex) {
src1_size = hex_round_up(ggml_nelements(src1) * elem_size + 128, 128);
if (src1_size > sess->vtcm_size) {
return false;
}
kernel_type = HTP_BINARY_KERNEL_COMPLEX;
kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0;
} else {
src1_size = hex_round_up(ggml_nelements(src1) * elem_size + 128, 128);
if (src1_size > sess->vtcm_size) {
return false;
}
kernel_type = HTP_BINARY_KERNEL_REPEAT;
kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0;
}

kparams->kernel_type = kernel_type;
Expand All @@ -5955,6 +5974,39 @@ static bool ggml_hexagon_precompute_binary_params(
kparams->src1_row_size_aligned = src1_row_size_aligned;
kparams->dst_row_size_aligned = dst_row_size_aligned;
kparams->src1_size = src1_size;
kparams->src2_size = src2_size;

const bool src0_contig_dim1 = (src0->nb[2] == src0->ne[1] * src0->nb[1]);
const bool src1_contig_dim1 = (src1->nb[2] == src1->ne[1] * src1->nb[1]);
const bool dst_contig_dim1 = (dst->nb[2] == src0->ne[1] * dst->nb[1]);

const bool src0_contig_dim2 = (src0->nb[3] == src0->ne[2] * src0->nb[2]);
const bool src1_contig_dim2 = (src1->nb[3] == src1->ne[2] * src1->nb[2]);
const bool dst_contig_dim2 = (dst->nb[3] == src0->ne[2] * dst->nb[2]);

bool split_at_ne01 = false;
if (src0->ne[2] > 1) {
if (kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) {
split_at_ne01 = (src1->ne[2] != src0->ne[2]) || !src0_contig_dim1 || !src1_contig_dim1 || !dst_contig_dim1;
} else if (kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) {
split_at_ne01 = !src0_contig_dim1 || !dst_contig_dim1;
} else {
split_at_ne01 = true;
}
}
kparams->split_at_ne01 = split_at_ne01 ? 1 : 0;

bool split_at_ne02 = false;
if (src0->ne[3] > 1) {
if (kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) {
split_at_ne02 = (src1->ne[3] != src0->ne[3]) || !src0_contig_dim2 || !src1_contig_dim2 || !dst_contig_dim2;
} else if (kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) {
split_at_ne02 = !src0_contig_dim2 || !dst_contig_dim2;
} else {
split_at_ne02 = true;
}
}
kparams->split_at_ne02 = split_at_ne02 ? 1 : 0;

struct htp_binary_vtcm_layout L;
htp_binary_vtcm_layout_build(&L, kparams, sess->vtcm_size);
Expand Down Expand Up @@ -7139,9 +7191,12 @@ static bool ggml_hexagon_supported_add_id(const struct ggml_hexagon_session * se
if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) {
return false;
}
if (src2->nb[1] < src2->ne[0] * sizeof(int32_t)) {
return false;
}

struct htp_binary_kernel_params kparams;
return ggml_hexagon_precompute_binary_params(sess, HTP_OP_ADD_ID, src0, src1, dst, &kparams);
return ggml_hexagon_precompute_binary_params(sess, HTP_OP_ADD_ID, src0, src1, dst, &kparams, src2);
}

static bool ggml_hexagon_supported_unary(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
Expand Down Expand Up @@ -8504,9 +8559,10 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg
);
} else if (node.opcode == HTP_OP_MUL || node.opcode == HTP_OP_ADD || node.opcode == HTP_OP_ADD_ID || node.opcode == HTP_OP_SUB || node.opcode == HTP_OP_DIV) {
const ggml_tensor * src1 = node.node->src[1];
const ggml_tensor * src2 = (node.opcode == HTP_OP_ADD_ID) ? node.node->src[2] : nullptr;
GGML_ASSERT(ggml_hexagon_precompute_binary_params(sess,
node.opcode, node.node->src[0], src1, node.node,
(struct htp_binary_kernel_params *) node.kernel_params));
(struct htp_binary_kernel_params *) node.kernel_params, src2));
} else if (node.opcode == HTP_OP_FLASH_ATTN_EXT) {
ggml_hexagon_precompute_flash_attn_params(sess,
node.node,
Expand Down Expand Up @@ -9391,9 +9447,11 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons
case GGML_GLU_OP_SWIGLU_OAI:
case GGML_GLU_OP_SWIGLU_CLAMP:
case GGML_GLU_OP_GEGLU:
supp = ggml_hexagon_supported_activations(sess, op);
break;
case GGML_GLU_OP_GEGLU_QUICK:
case GGML_GLU_OP_GEGLU_ERF:
supp = ggml_hexagon_supported_activations(sess, op);
supp = (op->src[0]->type == GGML_TYPE_F32) && ggml_hexagon_supported_activations(sess, op);
break;
default:
supp = false;
Expand Down
Loading
Loading