npu-pack: report which text artifacts a pack carries (info, import, container config)

docs/npu.md

@@ -505,6 +505,19 @@ contract are the model; the reader is written from the pack-side facts in
 `amd-oss/npu-re/docs/FACTS-oga-pack-weight-layout.md`, and no vendor library, wheel or compiled
 program is called or shipped.
 
+**A pack can be the weights without the text.** AMD's `_rai_1.8.0_npu_4K` repos do publish the
+model's own `tokenizer.json`, `tokenizer_config.json`, `vocab.json`, `merges.txt`,
+`chat_template.jinja`, `special_tokens_map.json`, `added_tokens.json` and `generation_config.json`,
+but they are separate files in a repo of ~70, so a pack fetched weight by weight — which is how they
+are pulled on this box — can be `model.onnx` + `model.onnx.data` + `dd_metastate_*.fconst` +
+`config.json` and nothing else (engine#1024). The importer copies whatever text is there and,
+because the command that fails without it is `1bit serve` ("not an NPU model directory"), both
+commands now say which artifacts the pack has: `npu-pack info` carries
+`text_artifacts: {present, missing, servable}` and `npu-pack import` prints the split, or a
+`WARNING:` naming `tokenizer.json` when the pack has none. The split is also recorded in the
+container's `config.json` (`q4nx_source.text_artifacts`), so a container states whether it ever had
+the text.
+
 What converts, and how faithfully:
 
 | pack tensor | where | to | fidelity |

npu/pack_import.cpp

@@ -862,6 +862,11 @@ std::string Plan::manifest() const {
                         {"down_groups_measured", mlp_down_groups},
                         {"note", mlp_note}};
     j["incomplete"] = refused();
+    const TextArtifacts text = scan_text_artifacts(dir);
+    j["text_artifacts"] = {{"present", text.present},
+                           {"missing", text.missing},
+                           // a container is servable only with tokenizer.json (docs/npu.md)
+                           {"servable", text.tokenizer}};
     return j.dump(2);
 }
 
@@ -1163,6 +1168,27 @@ double corr(const Q4& q, const uint8_t* embed, int H, int rows) {
 
 }  // namespace
 
+// The pack's text artifacts, in one place: the importer copies exactly these, and `info` reports
+// which are there before anything is written.
+static const char* const kTextFiles[] = {"tokenizer.json",
+                                         "tokenizer_config.json",
+                                         "vocab.json",
+                                         "merges.txt",
+                                         "added_tokens.json",
+                                         "special_tokens_map.json",
+                                         "chat_template.jinja",
+                                         "generation_config.json"};
+
+TextArtifacts scan_text_artifacts(const std::string& pack_dir) {
+    TextArtifacts t;
+    for (const char* f : kTextFiles) {
+        if (fs::exists(pack_dir + "/" + f)) t.present.push_back(f);
+        else t.missing.push_back(f);
+    }
+    t.tokenizer = fs::exists(pack_dir + "/tokenizer.json");
+    return t;
+}
+
 Report import_pack(const std::string& pack_dir, const std::string& out_dir, const Options& opt) {
     const Plan p = plan(pack_dir);
     Report r;
@@ -1575,18 +1601,20 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons
     {
         std::ifstream cf(pack_dir + "/config.json");
         json c = json::parse(cf);
+        const TextArtifacts ta = scan_text_artifacts(pack_dir);
         c["vocab_size"] = V;
         c["attention_bias"] = r.attn_bias_tensors > 0;  // what the container actually carries
         c["q4nx_source"] = {{"converter", "1bit npu-pack import"},
                             {"pack", p.pack},
                             {"layout", "q4nx-1"},
-                            {"incomplete", r.refused.size()}};
+                            {"incomplete", r.refused.size()},
+                            // so a container read months later states whether it ever had text
+                            {"text_artifacts", ta.present.empty() ? "none in the pack"
+                                                                  : "copied from the pack"}};
         std::ofstream out(out_dir + "/config.json");
         out << c.dump(2) << "\n";
     }
-    for (const char* f : {"tokenizer.json", "tokenizer_config.json", "vocab.json", "merges.txt",
-                          "added_tokens.json", "special_tokens_map.json", "chat_template.jinja",
-                          "generation_config.json"}) {
+    for (const char* f : kTextFiles) {
         const std::string src = pack_dir + "/" + f;
         if (fs::exists(src)) fs::copy_file(src, out_dir + "/" + f, fs::copy_options::overwrite_existing);
     }
@@ -1602,7 +1630,7 @@ int run(int argc, char** argv) {
         "  <pack dir>  a Ryzen AI OGA NPU model pack (*_rai_*_npu_*: model.onnx,\n"
         "              model.onnx.data, dd_metastate_*.fconst, config.json, tokenizer)\n"
         "  info        print what the pack states as JSON; writes nothing\n"
-        "  import      write <out dir>/model.q4nx + config.json (+ tokenizer files)\n"
+        "  import      write <out dir>/model.q4nx + config.json (+ the pack's text files, if any)\n"
         "  --allow-incomplete  write what is representable and list the rest in the\n"
         "              container's manifest instead of refusing\n";
     if (argc < 1) {
@@ -1682,6 +1710,26 @@ int run(int argc, char** argv) {
     if (!r.fconst_note.empty()) std::printf("  fconst:     %s\n", r.fconst_note.c_str());
     if (!r.refused.empty()) std::printf("  incomplete: %zu tensors, listed in %s/model.q4nx\n",
                                         r.refused.size(), out.c_str());
+    const TextArtifacts ta = scan_text_artifacts(dir);
+    if (ta.tokenizer) {
+        std::printf("  text:       %zu of %zu tokenizer/chat files copied", ta.present.size(),
+                    sizeof(kTextFiles) / sizeof(*kTextFiles));
+        if (!ta.missing.empty()) {
+            std::printf(", absent from the pack:");
+            for (const auto& f : ta.missing) std::printf(" %s", f.c_str());
+        }
+        std::printf("\n");
+    } else {
+        // Not every pack has them, and the run that fails without them is two commands later.
+        std::printf("  text:       none -- this pack ships no tokenizer.json\n");
+        std::fprintf(stderr,
+                     "WARNING: %s carries no tokenizer.json, so %s has weights and config.json\n"
+                     "  only and `1bit serve` will refuse it as not an NPU model directory. Put the\n"
+                     "  model's own text artifacts there (tokenizer.json, tokenizer_config.json,\n"
+                     "  vocab.json, merges.txt, chat_template.jinja) -- e.g. from the upstream\n"
+                     "  checkpoint the pack was built from.\n",
+                     dir.c_str(), out.c_str());
+    }
     return 0;
 }
 

npu/pack_import.h

@@ -121,6 +121,17 @@ struct Plan {
 // Throws only when the pack is structurally unreadable.
 Plan plan(const std::string& pack_dir);
 
+// The text artifacts a runnable container needs beside model.q4nx, and which of them the pack
+// actually carries. A pack fetched weight by weight can be weights-only (engine#1024; AMD's repos
+// do publish the text, and how much of it varies per pack), and an importer that says nothing about
+// it writes a container that only fails later, in another command, as "not an NPU model
+// directory" -- which reads like our bug.
+struct TextArtifacts {
+    std::vector<std::string> present, missing;  // both in the importer's own file order
+    bool tokenizer = false;                     // tokenizer.json present: what `1bit serve` opens
+};
+TextArtifacts scan_text_artifacts(const std::string& pack_dir);
+
 struct Options {
     bool allow_incomplete = false;  // write what is representable and name the rest
 };

tests/npu_pack_import_test.cpp

@@ -1073,6 +1073,41 @@ void test_post_attn_norm_header(const std::string& root) {
     fs::remove_all(dir);
 }
 
+// The text artifacts are not universal: AMD's Qwen3 packs ship none (engine#1024), so `info` has to
+// say so before an import and `import` has to say so after it. A pack with them reports the split.
+void test_text_artifacts(const std::string& root) {
+    const fs::path dir = fs::path(root) / "text-artifacts";
+    fs::remove_all(dir);
+    fs::create_directories(dir);
+    put_file((dir / "model.onnx.data").string(), std::string(4096, '\x11'));
+    put_file((dir / "config.json").string(),
+             R"({"model_type":"qwen2","hidden_size":16,"intermediate_size":32,"num_hidden_layers":1,)"
+             R"("num_attention_heads":2,"num_key_value_heads":1,"head_dim":8,"vocab_size":64,)"
+             R"("rope_theta":10000.0,"tie_word_embeddings":true})");
+    put_file((dir / "model.onnx").string(),
+             pb::model({pb::tensor("model.embed_tokens.weight", {8, 8}, "model.onnx.data", 0, 256)}, {}));
+
+    pi::TextArtifacts t = pi::scan_text_artifacts(dir.string());
+    check(t.present.empty() && t.missing.size() == 8 && !t.tokenizer,
+          "text artifacts: a pack with none reports all 8 missing and not servable");
+    check(json::parse(pi::plan(dir.string()).manifest())["text_artifacts"]["servable"] == false,
+          "text artifacts: info says the pack is not servable");
+
+    put_file((dir / "tokenizer.json").string(), "{}");
+    put_file((dir / "vocab.json").string(), "{}");
+    t = pi::scan_text_artifacts(dir.string());
+    check(t.present == std::vector<std::string>({"tokenizer.json", "vocab.json"}) && t.missing.size() == 6 &&
+              t.tokenizer,
+          "text artifacts: a pack with two reports those two present, the other six missing");
+    const json m = json::parse(pi::plan(dir.string()).manifest());
+    check(m["text_artifacts"]["servable"] == true && m["text_artifacts"]["present"].size() == 2 &&
+              m["text_artifacts"]["missing"].size() == 6,
+          "text artifacts: info lists present, missing and servable");
+    std::printf("text artifacts: %zu present, %zu missing, servable %s\n", t.present.size(),
+                t.missing.size(), t.tokenizer ? "yes" : "no");
+    fs::remove_all(dir);
+}
+
 void test_fconst_eps(const std::string& root) {    const fs::path dir = fs::path(root) / "fconst-eps";
     fs::remove_all(dir);
     fs::create_directories(dir);
@@ -1144,6 +1179,7 @@ int main(int argc, char** argv) {
         test_attn_payload_off();
         test_qwen3_attn_sections();
         test_post_attn_norm_header(golden_dir.empty() ? "." : golden_dir);
+        test_text_artifacts(golden_dir.empty() ? "." : golden_dir);
         if (pack_ran) test_pack(pack, golden_dir);
         else std::printf("skip pack: no pack directory given (the crafted and golden legs only)\n");
     } catch (const std::exception& e) {