diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index a32ed59190af..1c1c92da20c1 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -344,7 +344,8 @@ static bool ggml_hexagon_precompute_binary_params( const struct ggml_tensor * src0, const struct ggml_tensor * src1, const struct ggml_tensor * dst, - struct htp_binary_kernel_params * kparams + struct htp_binary_kernel_params * kparams, + const struct ggml_tensor * src2 = nullptr ); static void ggml_hexagon_precompute_get_rows_params( @@ -5864,7 +5865,8 @@ static bool ggml_hexagon_precompute_binary_params( const struct ggml_tensor * src0, const struct ggml_tensor * src1, const struct ggml_tensor * dst, - struct htp_binary_kernel_params * kparams + struct htp_binary_kernel_params * kparams, + const struct ggml_tensor * src2 ) { memset(kparams, 0, sizeof(*kparams)); @@ -5877,14 +5879,17 @@ static bool ggml_hexagon_precompute_binary_params( const size_t src1_row_size_aligned = hex_round_up(src1_row_size, 128); const size_t dst_row_size_aligned = hex_round_up(dst_row_size, 128); + if (!htp_binary_strides_safe(src0, src1, dst, elem_size)) { + return false; + } + const bool is_add_id = op == HTP_OP_ADD_ID; const bool is_scalar = !is_add_id && src1->ne[0] == 1; - const bool is_transposed = src0->nb[1] < src0_row_size || src1->nb[1] < src1_row_size || dst->nb[1] < dst_row_size; - const bool is_row_bcast = !is_add_id && !is_scalar && !is_transposed && + const bool is_row_bcast = !is_add_id && !is_scalar && src1->ne[0] == src0->ne[0] && (src0->ne[1] > 1 || src0->ne[2] > 1 || src0->ne[3] > 1) && src1->ne[1] == 1 && src1->ne[2] == 1 && src1->ne[3] == 1; - const bool is_same_shape = !is_add_id && !is_scalar && !is_transposed && + const bool is_same_shape = !is_add_id && !is_scalar && src1->ne[0] == src0->ne[0] && src1->ne[1] == src0->ne[1] && (src1->ne[2] == src0->ne[2] || src1->ne[2] == 1) && @@ -5925,28 +5930,42 @@ static bool ggml_hexagon_precompute_binary_params( enum htp_binary_kernel_type kernel_type; size_t src1_size = 0; + size_t src2_size = 0; if (is_add_id) { + GGML_ASSERT(src2 != nullptr); kernel_type = HTP_BINARY_KERNEL_ADD_ID; src1_size = hex_round_up(src1->ne[1] * src1_row_size_aligned, 128); + src2_size = hex_round_up(ggml_nelements(src2) * sizeof(int32_t), 128); + if (src1_size + src2_size > sess->vtcm_size) { + return false; + } } else if (is_row_bcast) { kernel_type = HTP_BINARY_KERNEL_ROW_BCAST; src1_size = src1_row_size_aligned; } else if (is_scalar) { - const bool is_scalar_static = (src1->ne[2] == 1 && src1->ne[3] == 1) && - (src1->ne[1] == 1 || src1->nb[1] == elem_size); - if (is_scalar_static) { - kernel_type = HTP_BINARY_KERNEL_SCALAR_DMA; - src1_size = hex_round_up(src1->ne[1] * elem_size, 128); - } else { - kernel_type = HTP_BINARY_KERNEL_SCALAR; + src1_size = hex_round_up(ggml_nelements(src1) * elem_size, 128); + if (src1_size > sess->vtcm_size) { + return false; } + kernel_type = HTP_BINARY_KERNEL_SCALAR; + kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0; } else if (is_same_shape) { kernel_type = HTP_BINARY_KERNEL_SAME_SHAPE; } else if (is_complex) { + src1_size = hex_round_up(ggml_nelements(src1) * elem_size + 128, 128); + if (src1_size > sess->vtcm_size) { + return false; + } kernel_type = HTP_BINARY_KERNEL_COMPLEX; + kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0; } else { + src1_size = hex_round_up(ggml_nelements(src1) * elem_size + 128, 128); + if (src1_size > sess->vtcm_size) { + return false; + } kernel_type = HTP_BINARY_KERNEL_REPEAT; + kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0; } kparams->kernel_type = kernel_type; @@ -5955,6 +5974,39 @@ static bool ggml_hexagon_precompute_binary_params( kparams->src1_row_size_aligned = src1_row_size_aligned; kparams->dst_row_size_aligned = dst_row_size_aligned; kparams->src1_size = src1_size; + kparams->src2_size = src2_size; + + const bool src0_contig_dim1 = (src0->nb[2] == src0->ne[1] * src0->nb[1]); + const bool src1_contig_dim1 = (src1->nb[2] == src1->ne[1] * src1->nb[1]); + const bool dst_contig_dim1 = (dst->nb[2] == src0->ne[1] * dst->nb[1]); + + const bool src0_contig_dim2 = (src0->nb[3] == src0->ne[2] * src0->nb[2]); + const bool src1_contig_dim2 = (src1->nb[3] == src1->ne[2] * src1->nb[2]); + const bool dst_contig_dim2 = (dst->nb[3] == src0->ne[2] * dst->nb[2]); + + bool split_at_ne01 = false; + if (src0->ne[2] > 1) { + if (kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) { + split_at_ne01 = (src1->ne[2] != src0->ne[2]) || !src0_contig_dim1 || !src1_contig_dim1 || !dst_contig_dim1; + } else if (kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) { + split_at_ne01 = !src0_contig_dim1 || !dst_contig_dim1; + } else { + split_at_ne01 = true; + } + } + kparams->split_at_ne01 = split_at_ne01 ? 1 : 0; + + bool split_at_ne02 = false; + if (src0->ne[3] > 1) { + if (kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) { + split_at_ne02 = (src1->ne[3] != src0->ne[3]) || !src0_contig_dim2 || !src1_contig_dim2 || !dst_contig_dim2; + } else if (kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) { + split_at_ne02 = !src0_contig_dim2 || !dst_contig_dim2; + } else { + split_at_ne02 = true; + } + } + kparams->split_at_ne02 = split_at_ne02 ? 1 : 0; struct htp_binary_vtcm_layout L; htp_binary_vtcm_layout_build(&L, kparams, sess->vtcm_size); @@ -7139,9 +7191,12 @@ static bool ggml_hexagon_supported_add_id(const struct ggml_hexagon_session * se if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) { return false; } + if (src2->nb[1] < src2->ne[0] * sizeof(int32_t)) { + return false; + } struct htp_binary_kernel_params kparams; - return ggml_hexagon_precompute_binary_params(sess, HTP_OP_ADD_ID, src0, src1, dst, &kparams); + return ggml_hexagon_precompute_binary_params(sess, HTP_OP_ADD_ID, src0, src1, dst, &kparams, src2); } static bool ggml_hexagon_supported_unary(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { @@ -8504,9 +8559,10 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg ); } else if (node.opcode == HTP_OP_MUL || node.opcode == HTP_OP_ADD || node.opcode == HTP_OP_ADD_ID || node.opcode == HTP_OP_SUB || node.opcode == HTP_OP_DIV) { const ggml_tensor * src1 = node.node->src[1]; + const ggml_tensor * src2 = (node.opcode == HTP_OP_ADD_ID) ? node.node->src[2] : nullptr; GGML_ASSERT(ggml_hexagon_precompute_binary_params(sess, node.opcode, node.node->src[0], src1, node.node, - (struct htp_binary_kernel_params *) node.kernel_params)); + (struct htp_binary_kernel_params *) node.kernel_params, src2)); } else if (node.opcode == HTP_OP_FLASH_ATTN_EXT) { ggml_hexagon_precompute_flash_attn_params(sess, node.node, @@ -9391,9 +9447,11 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons case GGML_GLU_OP_SWIGLU_OAI: case GGML_GLU_OP_SWIGLU_CLAMP: case GGML_GLU_OP_GEGLU: + supp = ggml_hexagon_supported_activations(sess, op); + break; case GGML_GLU_OP_GEGLU_QUICK: case GGML_GLU_OP_GEGLU_ERF: - supp = ggml_hexagon_supported_activations(sess, op); + supp = (op->src[0]->type == GGML_TYPE_F32) && ggml_hexagon_supported_activations(sess, op); break; default: supp = false; diff --git a/ggml/src/ggml-hexagon/htp/binary-ops.c b/ggml/src/ggml-hexagon/htp/binary-ops.c index 1286176af51b..78ce4bb63d1b 100644 --- a/ggml/src/ggml-hexagon/htp/binary-ops.c +++ b/ggml/src/ggml-hexagon/htp/binary-ops.c @@ -103,13 +103,13 @@ static inline uint32_t calc_block_size(struct htp_binary_context * bctx, uint32_ // Out-of-line compute micro-kernels -typedef void (*compute_scalar_dma_t)( +typedef void (*compute_scalar_t)( uint8_t * dst, const uint8_t * src0, const void * s1_table, uint32_t cur_i11, uint32_t ne11, uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00); -#define DEFINE_COMPUTE_SCALAR_DMA(NAME, TYPE, HVX_STMT) \ -static void compute_scalar_dma_##NAME( \ +#define DEFINE_COMPUTE_SCALAR(NAME, TYPE, HVX_STMT) \ +static void compute_scalar_##NAME( \ uint8_t * dst, const uint8_t * src0, const void * s1_table, \ uint32_t cur_i11, uint32_t ne11, uint32_t n_rows, \ size_t dst_stride, size_t src0_stride, uint32_t ne00) { \ @@ -125,31 +125,6 @@ static void compute_scalar_dma_##NAME( \ } \ } -DEFINE_COMPUTE_SCALAR_DMA(add_f32, float, hvx_add_scalar_f32_aa(r_dst, r_src0, val, ne00)) -DEFINE_COMPUTE_SCALAR_DMA(add_f16, _Float16, hvx_add_scalar_f16_aa(r_dst, r_src0, val, ne00)) -DEFINE_COMPUTE_SCALAR_DMA(sub_f32, float, hvx_sub_scalar_f32_aa(r_dst, r_src0, val, ne00)) -DEFINE_COMPUTE_SCALAR_DMA(sub_f16, _Float16, hvx_sub_scalar_f16_aa(r_dst, r_src0, val, ne00)) -DEFINE_COMPUTE_SCALAR_DMA(mul_f32, float, hvx_mul_scalar_f32_aa(r_dst, r_src0, val, ne00)) -DEFINE_COMPUTE_SCALAR_DMA(mul_f16, _Float16, hvx_mul_scalar_f16_aa(r_dst, r_src0, val, ne00)) -DEFINE_COMPUTE_SCALAR_DMA(div_f32, float, hvx_mul_scalar_f32_aa(r_dst, r_src0, 1.0f / (val), ne00)) -DEFINE_COMPUTE_SCALAR_DMA(div_f16, _Float16, hvx_div_scalar_f16_aa(r_dst, r_src0, val, ne00)) - -typedef void (*compute_scalar_t)( - uint8_t * dst, const uint8_t * src0, const uint8_t * src1_ptr, uint32_t s1_stride, - uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00); - -#define DEFINE_COMPUTE_SCALAR(NAME, TYPE, HVX_STMT) \ -static void compute_scalar_##NAME( \ - uint8_t * dst, const uint8_t * src0, const uint8_t * src1_ptr, uint32_t s1_stride, \ - uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00) { \ - for (uint32_t r = 0; r < n_rows; r++) { \ - uint8_t * r_dst = dst + r * dst_stride; \ - const uint8_t * r_src0 = src0 + r * src0_stride; \ - TYPE val = *(const TYPE *)(src1_ptr + r * s1_stride); \ - HVX_STMT; \ - } \ -} - DEFINE_COMPUTE_SCALAR(add_f32, float, hvx_add_scalar_f32_aa(r_dst, r_src0, val, ne00)) DEFINE_COMPUTE_SCALAR(add_f16, _Float16, hvx_add_scalar_f16_aa(r_dst, r_src0, val, ne00)) DEFINE_COMPUTE_SCALAR(sub_f32, float, hvx_sub_scalar_f32_aa(r_dst, r_src0, val, ne00)) @@ -205,17 +180,17 @@ DEFINE_COMPUTE_ROW_BCAST(div_f16, hvx_div_f16_aaa) typedef void (*compute_complex_t)( uint8_t * dst, const uint8_t * src0, const uint8_t * src1_plane, - uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t nb11, + uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t src1_stride, uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00); #define DEFINE_COMPUTE_COMPLEX(NAME, HVX_FN) \ static void compute_complex_##NAME( \ uint8_t * dst, const uint8_t * src0, const uint8_t * src1_plane, \ - uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t nb11, \ + uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t src1_stride, \ uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00) { \ for (uint32_t r = 0; r < n_rows; r++) { \ uint32_t i11 = fastmodulo(i01 + r, ne11, div11); \ - const uint8_t * r_src1 = src1_plane + i11 * nb11; \ + const uint8_t * r_src1 = src1_plane + i11 * src1_stride; \ HVX_FN(dst + r * dst_stride, src0 + r * src0_stride, r_src1, ne00); \ } \ } @@ -231,17 +206,17 @@ DEFINE_COMPUTE_COMPLEX(div_f16, hvx_div_f16_aau) typedef void (*compute_repeat_t)( uint8_t * dst, const uint8_t * src0, const uint8_t * src1_plane, - uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t nb11, + uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t src1_stride, uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00, uint32_t ne10); #define DEFINE_COMPUTE_REPEAT(NAME, TYPE, HVX_FN) \ static void compute_repeat_##NAME( \ uint8_t * dst, const uint8_t * src0, const uint8_t * src1_plane, \ - uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t nb11, \ + uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t src1_stride, \ uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00, uint32_t ne10) { \ for (uint32_t r = 0; r < n_rows; r++) { \ uint32_t i11 = fastmodulo(i01 + r, ne11, div11); \ - const uint8_t * r_src1_row = src1_plane + i11 * nb11; \ + const uint8_t * r_src1_row = src1_plane + i11 * src1_stride; \ uint8_t * r_dst = dst + r * dst_stride; \ const uint8_t * r_src0 = src0 + r * src0_stride; \ for (uint32_t c = 0; c < ne00; c += ne10) { \ @@ -261,17 +236,17 @@ DEFINE_COMPUTE_REPEAT(div_f32, float, hvx_div_f32_uuu) DEFINE_COMPUTE_REPEAT(div_f16, _Float16, hvx_div_f16_uuu) typedef void (*compute_add_id_t)( - uint8_t * dst, const uint8_t * src0, const uint8_t * src1_data, const char * src2_data, - uint32_t i01, uint32_t i02, uint32_t nb20, uint32_t nb21, uint32_t src1_stride, + uint8_t * dst, const uint8_t * src0, const uint8_t * src1_data, const int32_t * src2_ids, + uint32_t i01, uint32_t i02, uint32_t ne01, uint32_t src1_stride, uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00); static void compute_add_id_f32( - uint8_t * dst, const uint8_t * src0, const uint8_t * src1_data, const char * src2_data, - uint32_t i01, uint32_t i02, uint32_t nb20, uint32_t nb21, uint32_t src1_stride, + uint8_t * dst, const uint8_t * src0, const uint8_t * src1_data, const int32_t * src2_ids, + uint32_t i01, uint32_t i02, uint32_t ne01, uint32_t src1_stride, uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00) { for (uint32_t r = 0; r < n_rows; r++) { uint32_t r_i01 = i01 + r; - const int32_t idx = *(const int32_t *)(src2_data + r_i01 * nb20 + i02 * nb21); + const int32_t idx = src2_ids[i02 * ne01 + r_i01]; if (idx < 0) { memcpy(dst + r * dst_stride, src0 + r * src0_stride, ne00 * sizeof(float)); continue; @@ -283,18 +258,22 @@ static void compute_add_id_f32( } } -// 1a. Scalar src1 in VTCM via DMA (ne10 == 1, ne12 == 1, ne13 == 1) -static void binary_thread_scalar_dma(unsigned int nth, unsigned int ith, void * data) { +// 1. Scalar src1 in VTCM via DMA (ne10 == 1) +static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; htp_binary_preamble; + const uint32_t elem_size = (src0->type == HTP_TYPE_F32) ? sizeof(float) : sizeof(_Float16); + const size_t plane_stride = (size_t) ne12 * ne11 * elem_size; + const size_t row_stride = (size_t) ne11 * elem_size; + const uint32_t row_size_bytes = bctx->row_size_bytes; const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; - FARF(HIGH, "binary-scalar-dma: %d/%d (%u:%u) row-size %u (%u)", + FARF(HIGH, "binary-scalar: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; @@ -329,94 +308,6 @@ static void binary_thread_scalar_dma(unsigned int nth, unsigned int ith, void * spad_idx ^= 1; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; - compute_scalar_dma_t compute = (compute_scalar_dma_t) bctx->compute; - - for (uint32_t ir = start_row; ir < end_row; ) { - uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); - - uint8_t * d_spad = (uint8_t *) dma_queue_pop(dma_q).src; - uint8_t * s0_spad = (uint8_t *) dma_queue_pop(dma_q).dst; - - uint32_t i03, i02, i01, rem; - i03 = fastdiv(ir, &bctx->src0_dim12_div); - rem = ir - i03 * (ne02 * ne01); - i02 = fastdiv(rem, &bctx->src0_dim1_div); - i01 = rem - i02 * ne01; - - uint32_t cur_i11 = fastmodulo(i01, ne11, &bctx->src1_dim1_div); - - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); - compute(d_spad, s0_spad, s1_table, cur_i11, ne11, current_block_size, - bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00); - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); - - dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; - dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, - current_block_size); - - if (ir_prefetch < end_row) { - uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); - uint32_t p03, p02, p01, prem; - p03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); - prem = ir_prefetch - p03 * (ne02 * ne01); - p02 = fastdiv(prem, &bctx->src0_dim1_div); - p01 = prem - p02 * ne01; - dma_addr_t s0_next = src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; - dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, - next_block_size); - ir_prefetch += next_block_size; - } - ir += current_block_size; - } - - dma_queue_flush(dma_q); -} - -// 1b. Scalar src1 dynamic / pointer (ne10 == 1) -static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * data) { - struct htp_binary_context * bctx = (struct htp_binary_context *) data; - struct htp_ops_context * octx = bctx->octx; - htp_binary_preamble; - - const uint32_t row_size_bytes = bctx->row_size_bytes; - const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); - if (start_row >= end_row) return; - - FARF(HIGH, "binary-scalar: %d/%d (%u:%u) row-size %u (%u)", - ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); - - const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; - uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); - uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); - size_t src0_spad_half = layout->src0_spad_half_size; - size_t dst_spad_half = layout->dst_spad_half_size; - - dma_queue * dma_q = octx->ctx->dma[ith]; - uint32_t ir_prefetch = start_row; - int spad_idx = 0; - - for (int k = 0; k < 2 && ir_prefetch < end_row; k++) { - uint32_t current_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); - uint32_t i03, i02, i01, rem; - i03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); - rem = ir_prefetch - i03 * (ne02 * ne01); - i02 = fastdiv(rem, &bctx->src0_dim1_div); - i01 = rem - i02 * ne01; - - dma_addr_t src0_curr = src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01; - dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; - - uint8_t * s0_spad = src0_spad_base + spad_idx * src0_spad_half; - uint8_t * d_spad = dst_spad_base + spad_idx * dst_spad_half; - - dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0); - dma_queue_push(dma_q, dma_make_data(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size); - ir_prefetch += current_block_size; - spad_idx ^= 1; - } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; compute_scalar_t compute = (compute_scalar_t) bctx->compute; @@ -435,17 +326,17 @@ static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * data uint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div); uint32_t i12 = fastmodulo(i02, ne12, &bctx->src1_dim2_div); uint32_t i11 = fastmodulo(i01, ne11, &bctx->src1_dim1_div); - - const uint8_t * src1_ptr = (const uint8_t *)(uintptr_t) src1->data + i13 * nb13 + i12 * nb12 + i11 * nb11; - uint32_t s1_stride = (ne11 == 1) ? 0 : nb11; + const uint8_t * scalar_table = (const uint8_t *) s1_table + + i13 * plane_stride + i12 * row_stride; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); - compute(d_spad, s0_spad, src1_ptr, s1_stride, current_block_size, + compute(d_spad, s0_spad, scalar_table, i11, ne11, current_block_size, bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; - dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size); + dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, + current_block_size); if (ir_prefetch < end_row) { uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); @@ -455,7 +346,8 @@ static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * data p02 = fastdiv(prem, &bctx->src0_dim1_div); p01 = prem - p02 * ne01; dma_addr_t s0_next = src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; - dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size); + dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, + next_block_size); ir_prefetch += next_block_size; } ir += current_block_size; @@ -668,6 +560,12 @@ static void binary_thread_vector_complex(unsigned int nth, unsigned int ith, voi uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); size_t src0_spad_half = layout->src0_spad_half_size; size_t dst_spad_half = layout->dst_spad_half_size; + const uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(const uint8_t, bctx->vtcm_base, layout->off_src1); + + const uint32_t elem_size = (src0->type == HTP_TYPE_F32) ? sizeof(float) : sizeof(_Float16); + const size_t vtcm_row_stride = (size_t) ne10 * elem_size; + const size_t vtcm_plane_stride = (size_t) ne11 * vtcm_row_stride; + const size_t vtcm_vol_stride = (size_t) ne12 * vtcm_plane_stride; dma_queue * dma_q = octx->ctx->dma[ith]; uint32_t ir_prefetch = start_row; @@ -707,10 +605,10 @@ static void binary_thread_vector_complex(unsigned int nth, unsigned int ith, voi uint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div); uint32_t i12 = fastmodulo(i02, ne12, &bctx->src1_dim2_div); - const uint8_t * src1_plane = (const uint8_t *)(uintptr_t) src1->data + i13 * nb13 + i12 * nb12; + const uint8_t * src1_plane = vtcm_src1 + i13 * vtcm_vol_stride + i12 * vtcm_plane_stride; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); - compute(d_spad, s0_spad, src1_plane, i01, ne11, &bctx->src1_dim1_div, nb11, + compute(d_spad, s0_spad, src1_plane, i01, ne11, &bctx->src1_dim1_div, vtcm_row_stride, current_block_size, bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); @@ -750,6 +648,12 @@ static void binary_thread_element_repeat(unsigned int nth, unsigned int ith, voi uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); size_t src0_spad_half = layout->src0_spad_half_size; size_t dst_spad_half = layout->dst_spad_half_size; + const uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(const uint8_t, bctx->vtcm_base, layout->off_src1); + + const uint32_t elem_size = (src0->type == HTP_TYPE_F32) ? sizeof(float) : sizeof(_Float16); + const size_t vtcm_row_stride = (size_t) ne10 * elem_size; + const size_t vtcm_plane_stride = (size_t) ne11 * vtcm_row_stride; + const size_t vtcm_vol_stride = (size_t) ne12 * vtcm_plane_stride; FARF(HIGH, "binary-repeat: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); @@ -792,10 +696,10 @@ static void binary_thread_element_repeat(unsigned int nth, unsigned int ith, voi uint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div); uint32_t i12 = fastmodulo(i02, ne12, &bctx->src1_dim2_div); - const uint8_t * src1_plane = (const uint8_t *)(uintptr_t) src1->data + i13 * nb13 + i12 * nb12; + const uint8_t * src1_plane = vtcm_src1 + i13 * vtcm_vol_stride + i12 * vtcm_plane_stride; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); - compute(d_spad, s0_spad, src1_plane, i01, ne11, &bctx->src1_dim1_div, nb11, + compute(d_spad, s0_spad, src1_plane, i01, ne11, &bctx->src1_dim1_div, vtcm_row_stride, current_block_size, bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00, ne10); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); @@ -826,7 +730,6 @@ static void binary_thread_add_id_f32(unsigned int nth, unsigned int ith, void * const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * src1 = octx->src[1]; - const struct htp_tensor * src2 = octx->src[2]; const struct htp_tensor * dst = octx->dst; const uint32_t ne00 = src0->ne[0]; @@ -852,6 +755,7 @@ static void binary_thread_add_id_f32(unsigned int nth, unsigned int ith, void * uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); const uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(const uint8_t, bctx->vtcm_base, layout->off_src1); + const int32_t * vtcm_src2 = VTCM_LAYOUT_PTR(const int32_t, bctx->vtcm_base, layout->off_src2); size_t src0_spad_half = layout->src0_spad_half_size; size_t dst_spad_half = layout->dst_spad_half_size; @@ -892,8 +796,8 @@ static void binary_thread_add_id_f32(unsigned int nth, unsigned int ith, void * htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); compute_add_id_t compute = (compute_add_id_t) bctx->compute; - compute(d_spad, s0_spad, vtcm_src1, (const char *)(uintptr_t)src2->data, - i01, i02, src2->nb[0], src2->nb[1], src1_stride, + compute(d_spad, s0_spad, vtcm_src1, vtcm_src2, + i01, i02, ne01, src1_stride, current_block_size, bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); @@ -1234,19 +1138,6 @@ static int execute_op_binary(struct htp_ops_context * octx) { const size_t src1_row_size_aligned = kparams->src1_row_size_aligned; const size_t dst_row_size_aligned = kparams->dst_row_size_aligned; - if (htp_tensor_is_extended(src1)) { - if (kparams->kernel_type != HTP_BINARY_KERNEL_SAME_SHAPE && - kparams->kernel_type != HTP_BINARY_KERNEL_ROW_BCAST && - kparams->kernel_type != HTP_BINARY_KERNEL_SCALAR_DMA && - kparams->kernel_type != HTP_BINARY_KERNEL_ADD_ID) { - return HTP_STATUS_NO_SUPPORT; - } - } - - if (octx->op == HTP_OP_ADD_ID && htp_tensor_is_extended(octx->src[2])) { - return HTP_STATUS_NO_SUPPORT; - } - struct htp_binary_context bctx; bctx.vtcm_base = (uint8_t *) octx->ctx->vtcm_base; htp_binary_vtcm_layout_build(&bctx.vtcm_layout, kparams, octx->ctx->vtcm_size); @@ -1259,12 +1150,38 @@ static int execute_op_binary(struct htp_ops_context * octx) { uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, bctx.vtcm_base, bctx.vtcm_layout.off_src1); if (kparams->kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) { dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), bctx.vtcm_layout.src1_size, 0, src1->ne[0] * elem_size, 1); - } else if (kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR_DMA) { - dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), bctx.vtcm_layout.src1_size, 0, src1->ne[1] * elem_size, 1); + } else if (kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR || + kparams->kernel_type == HTP_BINARY_KERNEL_REPEAT || + kparams->kernel_type == HTP_BINARY_KERNEL_COMPLEX) { + if (!kparams->src1_strided) { + const size_t src1_bytes = (size_t) src1->ne[0] * src1->ne[1] * src1->ne[2] * src1->ne[3] * elem_size; + dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), bctx.vtcm_layout.src1_size, 0, src1_bytes, 1); + } else { + const uint32_t ne11 = src1->ne[1]; + const uint32_t ne12 = src1->ne[2]; + const uint32_t ne13 = src1->ne[3]; + const size_t row_bytes = src1->ne[0] * elem_size; + for (uint32_t i13 = 0; i13 < ne13; i13++) { + for (uint32_t i12 = 0; i12 < ne12; i12++) { + dma_addr_t src_addr = src1->data + i13 * src1->nb[3] + i12 * src1->nb[2]; + uint8_t * dst_ptr = vtcm_src1 + (i13 * ne12 * ne11 + i12 * ne11) * row_bytes; + while (!dma_queue_push(dma_q, dma_make_data(dst_ptr, src_addr), + row_bytes, src1->nb[1], row_bytes, ne11)) { + dma_queue_pop(dma_q); + } + } + } + } } else if (kparams->kernel_type == HTP_BINARY_KERNEL_ADD_ID) { + const struct htp_tensor * src2 = octx->src[2]; + uint8_t * vtcm_src2 = VTCM_LAYOUT_PTR(uint8_t, bctx.vtcm_base, bctx.vtcm_layout.off_src2); dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), kparams->src1_row_size_aligned, src1->nb[1], src1->ne[0] * elem_size, src1->ne[1]); + const size_t src2_row_bytes = src2->ne[0] * sizeof(int32_t); + dma_queue_push(dma_q, dma_make_data(vtcm_src2, src2->data), + src2_row_bytes, src2->nb[1], + src2_row_bytes, src2->ne[1]); } bctx.octx = octx; @@ -1285,15 +1202,8 @@ static int execute_op_binary(struct htp_ops_context * octx) { bctx.src1_dim2_div = init_fastdiv_values(src1->ne[2]); bctx.src1_dim3_div = init_fastdiv_values(src1->ne[3]); - bool src0_contig_dim1 = (src0->nb[2] == src0->ne[1] * src0->nb[1]); - bool dst_contig_dim1 = (dst->nb[2] == src0->ne[1] * dst->nb[1]); - - bool src0_contig_dim2 = (src0->nb[3] == src0->ne[2] * src0->nb[2]); - bool dst_contig_dim2 = (dst->nb[3] == src0->ne[2] * dst->nb[2]); - - bctx.split_at_ne01 = (octx->op == HTP_OP_ADD_ID) || - ((src0->ne[2] > 1) && ((src1->ne[1] > 1) || (src1->ne[2] > 1) || !src0_contig_dim1 || !dst_contig_dim1)); - bctx.split_at_ne02 = (src0->ne[3] > 1) && ((src1->ne[2] > 1) || (src1->ne[3] > 1) || !src0_contig_dim2 || !dst_contig_dim2); + bctx.split_at_ne01 = kparams->split_at_ne01; + bctx.split_at_ne02 = kparams->split_at_ne02; worker_callback_t worker_func = NULL; void * compute_func = NULL; @@ -1339,26 +1249,6 @@ static int execute_op_binary(struct htp_ops_context * octx) { } } break; - case HTP_BINARY_KERNEL_SCALAR_DMA: - worker_func = binary_thread_scalar_dma; - if (src0_type == HTP_TYPE_F32) { - switch (octx->op) { - case HTP_OP_ADD: compute_func = compute_scalar_dma_add_f32; break; - case HTP_OP_SUB: compute_func = compute_scalar_dma_sub_f32; break; - case HTP_OP_MUL: compute_func = compute_scalar_dma_mul_f32; break; - case HTP_OP_DIV: compute_func = compute_scalar_dma_div_f32; break; - default: break; - } - } else if (src0_type == HTP_TYPE_F16) { - switch (octx->op) { - case HTP_OP_ADD: compute_func = compute_scalar_dma_add_f16; break; - case HTP_OP_SUB: compute_func = compute_scalar_dma_sub_f16; break; - case HTP_OP_MUL: compute_func = compute_scalar_dma_mul_f16; break; - case HTP_OP_DIV: compute_func = compute_scalar_dma_div_f16; break; - default: break; - } - } - break; case HTP_BINARY_KERNEL_SCALAR: worker_func = binary_thread_scalar; if (src0_type == HTP_TYPE_F32) { @@ -1435,9 +1325,11 @@ static int execute_op_binary(struct htp_ops_context * octx) { bctx.compute = compute_func; if (kparams->kernel_type == HTP_BINARY_KERNEL_ROW_BCAST || - kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR_DMA || + kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR || + kparams->kernel_type == HTP_BINARY_KERNEL_REPEAT || + kparams->kernel_type == HTP_BINARY_KERNEL_COMPLEX || kparams->kernel_type == HTP_BINARY_KERNEL_ADD_ID) { - dma_queue_pop(dma_q); + dma_queue_flush(dma_q); } work_queue_run(octx->ctx->work_queue, worker_func, &bctx, n_threads); @@ -1446,13 +1338,6 @@ static int execute_op_binary(struct htp_ops_context * octx) { } int op_binary(struct htp_ops_context * octx) { - - // Does not support permutations of src1 - const struct htp_tensor * src1 = octx->src[1]; - if (src1->nb[1] < src1->nb[0]) { - return HTP_STATUS_NO_SUPPORT; - } - const uint32_t src0_type = octx->src[0]->type; if ((src0_type == HTP_TYPE_F32) || (src0_type == HTP_TYPE_F16)) { return execute_op_binary(octx); diff --git a/ggml/src/ggml-hexagon/htp/binary-ops.h b/ggml/src/ggml-hexagon/htp/binary-ops.h index aa8ff4959937..db3d51489559 100644 --- a/ggml/src/ggml-hexagon/htp/binary-ops.h +++ b/ggml/src/ggml-hexagon/htp/binary-ops.h @@ -12,7 +12,6 @@ enum htp_binary_kernel_type { HTP_BINARY_KERNEL_SAME_SHAPE = 0, HTP_BINARY_KERNEL_ROW_BCAST, - HTP_BINARY_KERNEL_SCALAR_DMA, HTP_BINARY_KERNEL_SCALAR, HTP_BINARY_KERNEL_ADD_ID, HTP_BINARY_KERNEL_COMPLEX, @@ -30,15 +29,41 @@ struct htp_binary_kernel_params { uint32_t dst_row_size_aligned; uint32_t src1_size; + uint32_t src2_size; uint32_t vtcm_size; uint32_t chunk_size; uint32_t chunk_bytes; uint32_t is_scalar; + + uint8_t split_at_ne01; + uint8_t split_at_ne02; + uint8_t src1_strided; + uint8_t pad; }; #if defined(__cplusplus) static_assert(sizeof(struct htp_binary_kernel_params) <= 128, "htp_binary_kernel_params is too large for kernel_params blob"); + +struct ggml_tensor; +static inline bool htp_binary_strides_safe( + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + size_t elem_size +) { + if ((src0->ne[0] != 1 && src0->nb[0] != elem_size) || + (src1->ne[0] != 1 && src1->nb[0] != elem_size) || + (dst->ne[0] != 1 && dst->nb[0] != elem_size)) { + return false; + } + if ((src0->ne[1] > 1 && src0->nb[1] < src0->ne[0] * elem_size) || + (src1->ne[1] > 1 && src1->nb[1] < src1->ne[0] * elem_size) || + (dst->ne[1] > 1 && dst->nb[1] < dst->ne[0] * elem_size)) { + return false; + } + return true; +} #else _Static_assert(sizeof(struct htp_binary_kernel_params) <= 128, "htp_binary_kernel_params is too large for kernel_params blob"); #endif @@ -47,6 +72,7 @@ struct htp_binary_vtcm_layout { size_t total_bytes; size_t off_src0; size_t off_src1; + size_t off_src2; size_t off_dst; size_t src0_bytes_per_thread; @@ -58,6 +84,7 @@ struct htp_binary_vtcm_layout { size_t dst_spad_half_size; size_t src1_size; + size_t src2_size; uint32_t rows_per_buffer; }; @@ -111,17 +138,19 @@ static inline void htp_binary_vtcm_layout_build( ? 2 * (kparams->src0_row_size_aligned + kparams->src1_row_size_aligned + kparams->dst_row_size_aligned) : 2 * (kparams->src0_row_size_aligned + kparams->dst_row_size_aligned); - if (spad_row_total == 0 || vtcm_size < kparams->src1_size) { + const size_t static_vtcm_size = kparams->src1_size + kparams->src2_size; + if (spad_row_total == 0 || vtcm_size < static_vtcm_size) { return; } - const size_t rows_per_buffer = (vtcm_size - kparams->src1_size) / (n_threads * spad_row_total); + const size_t rows_per_buffer = (vtcm_size - static_vtcm_size) / (n_threads * spad_row_total); if (rows_per_buffer == 0) { return; } L->rows_per_buffer = (uint32_t) rows_per_buffer; L->src1_size = kparams->src1_size; + L->src2_size = kparams->src2_size; L->src0_bytes_per_thread = rows_per_buffer * 2 * kparams->src0_row_size_aligned; L->dst_bytes_per_thread = rows_per_buffer * 2 * kparams->dst_row_size_aligned; @@ -137,11 +166,13 @@ static inline void htp_binary_vtcm_layout_build( const size_t src1_total = (kparams->src1_size > 0) ? kparams->src1_size : n_threads * L->src1_bytes_per_thread; + const size_t src2_total = kparams->src2_size; const size_t dst_total = n_threads * L->dst_bytes_per_thread; size_t off = 0; VTCM_LAYOUT_ALLOC(off, off_src0, src0_total); VTCM_LAYOUT_ALLOC(off, off_src1, src1_total); + VTCM_LAYOUT_ALLOC(off, off_src2, src2_total); VTCM_LAYOUT_ALLOC(off, off_dst, dst_total); L->total_bytes = off; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index bd599e99e55d..b54abbea83f0 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -3303,6 +3303,7 @@ struct test_bin_bcast : public test_case { bool perm1; // permute src1? bool src_overlap; // src0 and src1 are overlapping views of the same buffer const ggml_type type_b; // type of src1, GGML_TYPE_COUNT -> same as src0 + bool weights; bool run_whole_graph() override { return nf > 1; } @@ -3311,6 +3312,9 @@ struct test_bin_bcast : public test_case { if (type_b != GGML_TYPE_COUNT) { s += "," + VAR_TO_STR(type_b); } + if (weights) { + s += ",weights=1"; + } return s; } @@ -3323,28 +3327,38 @@ struct test_bin_bcast : public test_case { std::array nr = {1, 2, 1, 1}, int nf = 1, bool perm1 = false, bool src_overlap = false, - ggml_type type_b = GGML_TYPE_COUNT) - : op(op), type(type), ne(ne), nr(nr), nf(nf), perm1(perm1), src_overlap(src_overlap), type_b(type_b) {} + ggml_type type_b = GGML_TYPE_COUNT, bool weights = false) + : op(op), type(type), ne(ne), nr(nr), nf(nf), perm1(perm1), src_overlap(src_overlap), type_b(type_b), weights(weights) {} + + bool use_weight_context() override { + return weights; + } ggml_tensor * build_graph(ggml_context * ctx) override { + return build_graph(ctx, ctx); + } + + ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) override { GGML_ASSERT(nf <= 16); ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne[0]*nr[0], ne[1]*nr[1], ne[2]*nr[2], ne[3]*nr[3]); ggml_set_name(a, "a"); const ggml_type tb = type_b == GGML_TYPE_COUNT ? type : type_b; + ggml_context * ctx_b = weights ? ctx_weights : ctx; + GGML_ASSERT(ctx_b); ggml_tensor * b[16]; for (int i = 0; i < nf; ++i) { if (perm1) { const int p[4] = { 1, 2, 0, 3 }; // hardcoded for now - b[i] = ggml_new_tensor_4d(ctx, tb, ne[p[0]], ne[p[1]], ne[p[2]], ne[p[3]]); - b[i] = ggml_permute(ctx, b[i], p[0], p[1], p[2], p[3]); + b[i] = ggml_new_tensor_4d(ctx_b, tb, ne[p[0]], ne[p[1]], ne[p[2]], ne[p[3]]); + b[i] = ggml_permute(ctx_b, b[i], p[0], p[1], p[2], p[3]); } else if (src_overlap) { b[i] = ggml_view_4d(ctx, a, ne[0], ne[1], ne[2], 2 * (ne[3] / 3), a->nb[1], a->nb[2], a->nb[3], (ne[3] / 3) * a->nb[3]); } else { - b[i] = ggml_new_tensor(ctx, tb, 4, ne.data()); + b[i] = ggml_new_tensor(ctx_b, tb, 4, ne.data()); } ggml_set_name(b[i], (std::string("b") + std::to_string(i)).c_str()); } @@ -10387,6 +10401,11 @@ static std::vector> make_test_cases_eval() { //add_test_bin_bcast(type, {3, 3, 2560, 1280}, {2, 1, 1, 1}); } + for (auto op : {ggml_add, ggml_mul}) { + test_cases.emplace_back(new test_bin_bcast(op, GGML_TYPE_F32, {1, 2, 2, 2}, {17, 1, 1, 1}, 1, false, false, GGML_TYPE_COUNT, true)); + } + test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {2, 2, 2, 2}, {17, 1, 1, 1}, 1, false, false, GGML_TYPE_COUNT, true)); + // single inplace tests, especially important for WebGPU backend since kernels for inplace vs. not are different test_cases.emplace_back(new test_bin_bcast(ggml_add_inplace, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 1, 1, 1}, 16)); test_cases.emplace_back(new test_bin_bcast(ggml_mul_inplace, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 1, 1, 1}, 16));