From 9fc4220ba8bb3a159b741b2d7ff7a9bd16ac39c8 Mon Sep 17 00:00:00 2001 From: Takanori Ishikawa Date: Mon, 3 Aug 2026 05:05:42 +0900 Subject: [PATCH 1/2] =?UTF-8?q?deepseek4:=20expert=5Fused=5Fcount=20?= =?UTF-8?q?=E3=81=AE=20override=20=E3=82=92=E3=83=8F=E3=83=83=E3=82=B7?= =?UTF-8?q?=E3=83=A5=E3=83=AB=E3=83=BC=E3=83=86=E3=82=A3=E3=83=B3=E3=82=B0?= =?UTF-8?q?=E5=B1=A4=E3=81=A8=E4=B8=A1=E7=AB=8B=E3=81=95=E3=81=9B=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ハッシュ層のトークン→expert 表 (ffn_gate_tid2eid) の幅は GGUF 固定で、 expert_used_count を override すると形状検証がロードで落ちる。表の実幅で ロードし、ハッシュ層の build_moe_ffn には表の幅を渡すことで、routed 層だけ 使用 expert 数を減らせるようにする。override なしの動作は変わらない --- src/models/deepseek4.cpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cpp index 2d41dace0b28..ebf39c209793 100644 --- a/src/models/deepseek4.cpp +++ b/src/models/deepseek4.cpp @@ -130,7 +130,12 @@ void llama_model_deepseek4::load_arch_tensors(llama_model_loader &) { layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); if ((uint32_t) i < hparams.dsv4_hash_layer_count) { - layer.ffn_gate_tid2eid = create_tensor(tn(LLM_TENSOR_FFN_GATE_TID2EID, "weight", i), {n_expert_used, n_vocab}, 0); + // the hash-routing table width is baked into the GGUF and must not follow an + // expert_used_count KV override (the routed layers can run with fewer experts, + // the hash layers cannot) + const ggml_tensor * tid2eid_meta = ml->get_tensor_meta(tn(LLM_TENSOR_FFN_GATE_TID2EID, "weight", i).str().c_str()); + const int64_t n_hash_used = tid2eid_meta ? tid2eid_meta->ne[0] : n_expert_used; + layer.ffn_gate_tid2eid = create_tensor(tn(LLM_TENSOR_FFN_GATE_TID2EID, "weight", i), {n_hash_used, n_vocab}, 0); } else { layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0); } @@ -1143,9 +1148,13 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p const auto & layer = model.layers[il]; ggml_tensor * selected_experts = nullptr; ggml_tensor * exp_probs_b = layer.ffn_exp_probs_b; + uint32_t n_expert_used_layer = hparams.n_expert_used; if ((uint32_t) il < hparams.dsv4_hash_layer_count) { selected_experts = ggml_get_rows(ctx0, layer.ffn_gate_tid2eid, res->t_inp_tokens); exp_probs_b = nullptr; + // hash layers always use the table's expert count, even when + // expert_used_count is overridden for the routed layers + n_expert_used_layer = (uint32_t) layer.ffn_gate_tid2eid->ne[0]; } ggml_tensor * moe_out = build_moe_ffn(cur, @@ -1154,7 +1163,7 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p layer.ffn_gate_exps, layer.ffn_down_exps, exp_probs_b, - n_expert, hparams.n_expert_used, + n_expert, n_expert_used_layer, LLM_FFN_SILU, hparams.expert_weights_norm, hparams.expert_weights_scale, (llama_expert_gating_func_type) hparams.expert_gating_func, From 87e33dc0a7d447a639d4f2e0544fbd759cf00ba7 Mon Sep 17 00:00:00 2001 From: Takanori Ishikawa Date: Mon, 3 Aug 2026 05:24:26 +0900 Subject: [PATCH 2/2] =?UTF-8?q?=E3=83=AC=E3=83=93=E3=83=A5=E3=83=BC?= =?UTF-8?q?=E6=8C=87=E6=91=98:=20=E9=9B=86=E7=B4=84=E5=B9=85=E3=81=A8=20fa?= =?UTF-8?q?il-loud=20=E5=8C=96=E3=82=92=E4=BF=AE=E6=AD=A3=E3=81=99?= =?UTF-8?q?=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - build_moe_ffn の最終集約が hparams.n_expert_used 固定で、ハッシュ層の 実計算幅 (表幅) と食い違うと一部の expert 出力が合算されない (Codex 指摘)。 warmup 時のみ hparams 幅に抑制し、通常時は引数幅で集約する - metadata-only ロード経路で表幅が読めない場合に警告を出す (Codex 指摘) - n_expert_used_layer を int64_t にする (Gemini 指摘) --- src/llama-graph.cpp | 17 ++++++++++------- src/models/deepseek4.cpp | 10 ++++++++-- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 6d1c8f4e42a8..af771bd3cd8d 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -2129,26 +2129,29 @@ ggml_tensor * llm_graph_context::build_moe_ffn( assert(n_expert_used > 0); + // aggregate over the experts actually computed (n_expert_used may differ from + // hparams.n_expert_used, e.g. for dsv4 hash layers under an expert_used_count override); + // during warmup callers pass n_expert_used == n_expert, so cap to hparams.n_expert_used + // to avoid a large number of add nodes + // ref: https://github.com/ggml-org/llama.cpp/pull/14753 + const int64_t n_expert_aggr = cparams.warmup ? std::min(n_expert_used, (int64_t) hparams.n_expert_used) : n_expert_used; + // order the views before the adds - for (uint32_t i = 0; i < hparams.n_expert_used; ++i) { + for (int64_t i = 0; i < n_expert_aggr; ++i) { cur_experts[i] = ggml_view_2d(ctx0, experts, n_embd, n_tokens, experts->nb[2], i*experts->nb[1]); ggml_build_forward_expand(gf, cur_experts[i]); } - // aggregate experts - // note: here we explicitly use hparams.n_expert_used instead of n_expert_used - // to avoid potentially a large number of add nodes during warmup - // ref: https://github.com/ggml-org/llama.cpp/pull/14753 ggml_tensor * moe_out = cur_experts[0]; - for (uint32_t i = 1; i < hparams.n_expert_used; ++i) { + for (int64_t i = 1; i < n_expert_aggr; ++i) { moe_out = ggml_add(ctx0, moe_out, cur_experts[i]); ggml_build_forward_expand(gf, moe_out); } - if (hparams.n_expert_used == 1) { + if (n_expert_aggr == 1) { // avoid returning a non-contiguous tensor moe_out = ggml_cont(ctx0, moe_out); } diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cpp index ebf39c209793..40c68a6c41c6 100644 --- a/src/models/deepseek4.cpp +++ b/src/models/deepseek4.cpp @@ -134,6 +134,12 @@ void llama_model_deepseek4::load_arch_tensors(llama_model_loader &) { // expert_used_count KV override (the routed layers can run with fewer experts, // the hash layers cannot) const ggml_tensor * tid2eid_meta = ml->get_tensor_meta(tn(LLM_TENSOR_FFN_GATE_TID2EID, "weight", i).str().c_str()); + if (tid2eid_meta == nullptr) { + // metadata-only load paths (llama_model_init_from_user) have no weights map; + // fall back to n_expert_used, which then must not be overridden + LLAMA_LOG_WARN("%s: cannot read the width of %s, assuming expert_used_count (%lld) is not overridden\n", + __func__, tn(LLM_TENSOR_FFN_GATE_TID2EID, "weight", i).str().c_str(), (long long) n_expert_used); + } const int64_t n_hash_used = tid2eid_meta ? tid2eid_meta->ne[0] : n_expert_used; layer.ffn_gate_tid2eid = create_tensor(tn(LLM_TENSOR_FFN_GATE_TID2EID, "weight", i), {n_hash_used, n_vocab}, 0); } else { @@ -1148,13 +1154,13 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p const auto & layer = model.layers[il]; ggml_tensor * selected_experts = nullptr; ggml_tensor * exp_probs_b = layer.ffn_exp_probs_b; - uint32_t n_expert_used_layer = hparams.n_expert_used; + int64_t n_expert_used_layer = hparams.n_expert_used; if ((uint32_t) il < hparams.dsv4_hash_layer_count) { selected_experts = ggml_get_rows(ctx0, layer.ffn_gate_tid2eid, res->t_inp_tokens); exp_probs_b = nullptr; // hash layers always use the table's expert count, even when // expert_used_count is overridden for the routed layers - n_expert_used_layer = (uint32_t) layer.ffn_gate_tid2eid->ne[0]; + n_expert_used_layer = layer.ffn_gate_tid2eid->ne[0]; } ggml_tensor * moe_out = build_moe_ffn(cur,