diff --git a/openkb/agent/compiler.py b/openkb/agent/compiler.py index 8a422682..c2b92619 100644 --- a/openkb/agent/compiler.py +++ b/openkb/agent/compiler.py @@ -27,8 +27,8 @@ from pathlib import Path import litellm -import yaml +from openkb import frontmatter from openkb.config import DEFAULT_ENTITY_TYPES, get_extra_headers, resolve_entity_types from openkb.lint import list_existing_wiki_targets, strip_ghost_wikilinks from openkb.schema import INDEX_SEED, get_agents_md @@ -61,7 +61,7 @@ Write a summary page for this document in Markdown. Return a JSON object with two keys: -- "brief": A single sentence (under 100 chars) describing the document's main contribution +- "description": A single sentence (under 100 chars) describing the document's main contribution - "content": The full summary in Markdown. Include key concepts, findings, ideas, \ and [[wikilinks]] to concepts that could become cross-document concept pages @@ -142,7 +142,7 @@ {update_instruction} Return a JSON object with two keys: -- "brief": A single sentence (under 100 chars) defining this concept +- "description": A single sentence (under 100 chars) defining this concept - "content": The full concept page in Markdown. Include clear explanation, \ key details from the source document, and [[wikilinks]] to related concepts \ and [[summaries/{doc_name}]] — subject to the wikilink rules from the \ @@ -168,7 +168,7 @@ not invent new wikilink targets. Return a JSON object with two keys: -- "brief": A single sentence (under 100 chars) defining this concept (may differ from before) +- "description": A single sentence (under 100 chars) defining this concept (may differ from before) - "content": The rewritten full concept page in Markdown Return ONLY valid JSON, no fences. @@ -180,7 +180,7 @@ This entity relates to the document "{doc_name}" summarized above. Return a JSON object with three keys: -- "brief": A single sentence (under 100 chars) identifying this entity +- "description": A single sentence (under 100 chars) identifying this entity - "type": one of __ENTITY_TYPES__ - "content": The full entity page in Markdown — what this entity is, the key facts about it from this document, and [[wikilinks]] to related concepts, @@ -202,7 +202,7 @@ above for all [[wikilinks]]. Return a JSON object with three keys: -- "brief": A single sentence (under 100 chars) identifying this entity +- "description": A single sentence (under 100 chars) identifying this entity - "type": one of __ENTITY_TYPES__ - "content": The rewritten full entity page in Markdown @@ -524,12 +524,26 @@ def _read_wiki_context(wiki_dir: Path) -> tuple[str, list[str]]: return index_content, existing +def _resolve_description(fm: dict) -> str: + """Return a non-empty description string from a frontmatter dict. + + Checks ``description`` first, then the legacy ``brief`` key. Returns + an empty string when neither key holds a non-blank string value. + """ + for key in ("description", "brief"): + v = fm.get(key) + if isinstance(v, str) and v.strip(): + return v.strip() + return "" + + def _read_concept_briefs(wiki_dir: Path) -> str: """Read existing concept pages and return compact one-line summaries. - For each concept, reads the ``brief:`` field from YAML frontmatter if - present; otherwise falls back to truncating the first 150 chars of the body - (newlines collapsed to spaces). Formats each as ``- {slug}: {brief}``. + For each concept, reads the ``description:`` field (falling back to legacy + ``brief:``) from YAML frontmatter if present; otherwise falls back to + truncating the first 150 chars of the body (newlines collapsed to spaces). + Formats each as ``- {slug}: {description}``. Returns "(none yet)" if the concepts directory is missing or empty. """ @@ -544,20 +558,11 @@ def _read_concept_briefs(wiki_dir: Path) -> str: lines: list[str] = [] for path in md_files: text = path.read_text(encoding="utf-8") - brief = "" - body = text - if text.startswith("---"): - end = text.find("---", 3) - if end != -1: - fm_text = text[3:end].strip("\n") - body = text[end + 3:] - try: - fm = yaml.safe_load(fm_text) - except yaml.YAMLError: - fm = None - if isinstance(fm, dict) and isinstance(fm.get("brief"), str): - brief = fm["brief"].strip() + fm_dict = frontmatter.parse(text) + brief = _resolve_description(fm_dict) if not brief: + parts = frontmatter.split(text) + body = parts[1] if parts is not None else text brief = body.strip().replace("\n", " ")[:150] if brief: lines.append(f"- {path.stem}: {brief}") @@ -583,27 +588,13 @@ def _read_entity_briefs(wiki_dir: Path) -> str: lines: list[str] = [] for path in md_files: text = path.read_text(encoding="utf-8") - brief = "" - etype = "other" - n_sources = 0 - body = text - if text.startswith("---"): - end = text.find("---", 3) - if end != -1: - fm_text = text[3:end].strip("\n") - body = text[end + 3:] - try: - fm = yaml.safe_load(fm_text) - except yaml.YAMLError: - fm = None - if isinstance(fm, dict): - if isinstance(fm.get("brief"), str): - brief = fm["brief"].strip() - if isinstance(fm.get("type"), str): - etype = fm["type"].strip() or "other" - if isinstance(fm.get("sources"), list): - n_sources = len(fm["sources"]) + fm_dict = frontmatter.parse(text) + brief = _resolve_description(fm_dict) + etype = str(fm_dict.get("type") or "").strip().lower() or "other" + n_sources = len(fm_dict["sources"]) if isinstance(fm_dict.get("sources"), list) else 0 if not brief: + parts = frontmatter.split(text) + body = parts[1] if parts is not None else text brief = body.strip().replace("\n", " ")[:150] suffix = f" — {brief}" if brief else "" lines.append(f"- {path.stem} ({etype}, {n_sources} sources){suffix}") @@ -762,21 +753,22 @@ def _remove_section_entry(lines: list[str], heading: str, link: str) -> bool: def _write_summary(wiki_dir: Path, doc_name: str, summary: str, - doc_type: str = "short") -> None: + doc_type: str = "short", description: str = "") -> None: """Write summary page with frontmatter.""" - if summary.startswith("---"): - end = summary.find("---", 3) - if end != -1: - summary = summary[end + 3:].lstrip("\n") + parts = frontmatter.split(summary) + if parts is not None: + _, summary = parts + summary = summary.lstrip("\n") summaries_dir = wiki_dir / "summaries" summaries_dir.mkdir(parents=True, exist_ok=True) ext = "md" if doc_type == "short" else "json" - fm_lines = [ - f"doc_type: {doc_type}", - f"full_text: sources/{doc_name}.{ext}", - ] - frontmatter = "---\n" + "\n".join(fm_lines) + "\n---\n\n" - (summaries_dir / f"{doc_name}.md").write_text(frontmatter + summary, encoding="utf-8") + fm_lines = [_yaml_kv_line("type", "Summary")] + if description: + fm_lines.append(_yaml_kv_line("description", description)) + fm_lines.append(f"doc_type: {doc_type}") + fm_lines.append(_yaml_kv_line("full_text", f"sources/{doc_name}.{ext}")) + fm_block = "---\n" + "\n".join(fm_lines) + "\n---\n\n" + (summaries_dir / f"{doc_name}.md").write_text(fm_block + summary, encoding="utf-8") _SAFE_NAME_RE = re.compile(r'[^\w\-]') @@ -789,37 +781,9 @@ def _sanitize_concept_name(name: str) -> str: return sanitized or "unnamed-concept" -def _yaml_kv_line(key: str, value: str) -> str: - """Render a single ``key: value`` line that round-trips through any YAML loader. - - Uses ``json.dumps`` for the value — JSON strings are a strict subset of - YAML, always single-line, always correctly escaped (newlines, quotes, - control chars), and never auto-promoted to multi-line block scalars. - """ - return f"{key}: {json.dumps(value, ensure_ascii=False)}" - - -def _yaml_list_line(key: str, items: list[str]) -> str: - """Render ``key: [a, b, c]`` as JSON-style YAML (always single-line, always valid).""" - return f"{key}: {json.dumps(list(items), ensure_ascii=False)}" - - -def _parse_yaml_list_value(line: str) -> list[str] | None: - """Parse the right-hand side of ``key: [...]`` into a list of strings. - - Returns ``None`` when the value cannot be interpreted as a list — callers - treat that as "leave the frontmatter alone". - """ - colon = line.find(":") - if colon == -1: - return None - try: - parsed = yaml.safe_load(line[colon + 1:]) - except yaml.YAMLError: - return None - if not isinstance(parsed, list): - return None - return [str(x) for x in parsed] +_yaml_kv_line = frontmatter.kv_line +_yaml_list_line = frontmatter.list_line +_parse_yaml_list_value = frontmatter.parse_list_value def _write_concept(wiki_dir: Path, name: str, content: str, source_file: str, is_update: bool, brief: str = "") -> None: @@ -837,44 +801,58 @@ def _write_concept(wiki_dir: Path, name: str, content: str, source_file: str, is if source_file not in existing: existing = _prepend_source_to_frontmatter(existing, source_file) # Strip frontmatter from LLM content to avoid duplicate blocks - clean = content - if clean.startswith("---"): - end = clean.find("---", 3) - if end != -1: - clean = clean[end + 3:].lstrip("\n") + clean_parts = frontmatter.split(content) + clean = clean_parts[1].lstrip("\n") if clean_parts is not None else content # Replace body with LLM rewrite (prompt asks for full rewrite, not delta) - if existing.startswith("---"): - end = existing.find("---", 3) - if end != -1: - existing = existing[:end + 3] + "\n\n" + clean - else: - existing = clean + ex_parts = frontmatter.split(existing) + if ex_parts is not None: + fm_block, _ = ex_parts + existing = fm_block + "\n" + clean else: - existing = clean - if brief and existing.startswith("---"): - end = existing.find("---", 3) - if end != -1: - fm = existing[:end + 3] - body = existing[end + 3:] - brief_line = _yaml_kv_line("brief", brief) - if "brief:" in fm: - # Lambda to bypass re.sub backref interpretation in the - # replacement string (brief may contain \1, \g<…>, etc.). - fm = re.sub(r"brief:.*", lambda _m: brief_line, fm) - else: - fm = fm.replace("---\n", f"---\n{brief_line}\n", 1) - existing = fm + body + # Malformed/absent frontmatter (opening ``---`` with no closing + # delimiter, or no frontmatter at all): rebuild valid frontmatter + # rather than writing a bare body. Recover any sources already + # listed in the broken block first. + recovered: list[str] = [] + for ln in existing.split("\n"): + if ln.lstrip().startswith("sources:"): + parsed = _parse_yaml_list_value(ln) + if parsed: + recovered = parsed + break + merged = [source_file] + [s for s in recovered if s != source_file] + fm_lines = [ + _yaml_kv_line("type", "Concept"), + _yaml_list_line("sources", merged), + ] + if brief: + fm_lines.append(_yaml_kv_line("description", brief)) + existing = frontmatter.block(fm_lines) + clean + path.write_text(existing, encoding="utf-8") + return + # Guarantee type + refresh description on update; remove legacy brief:. + ex_parts2 = frontmatter.split(existing) + if ex_parts2 is not None: + fm_block, body = ex_parts2 + fm_block = _set_fm_line(fm_block, "type", "Concept") + if brief: + fm_block = _set_fm_line(fm_block, "description", brief) + # Drop legacy brief: lines (migrated to description:). + fm_block = frontmatter.drop_line(fm_block, "brief") + existing = fm_block + body path.write_text(existing, encoding="utf-8") else: - if content.startswith("---"): - end = content.find("---", 3) - if end != -1: - content = content[end + 3:].lstrip("\n") - fm_lines = [_yaml_list_line("sources", [source_file])] + clean_parts = frontmatter.split(content) + if clean_parts is not None: + content = clean_parts[1].lstrip("\n") + fm_lines = [ + _yaml_kv_line("type", "Concept"), + _yaml_list_line("sources", [source_file]), + ] if brief: - fm_lines.append(_yaml_kv_line("brief", brief)) - frontmatter = "---\n" + "\n".join(fm_lines) + "\n---\n\n" - path.write_text(frontmatter + content, encoding="utf-8") + fm_lines.append(_yaml_kv_line("description", brief)) + fm_block = "---\n" + "\n".join(fm_lines) + "\n---\n\n" + path.write_text(fm_block + content, encoding="utf-8") def _write_entity( @@ -885,8 +863,8 @@ def _write_entity( """Write or update an entity page in entities/, managing frontmatter. Frontmatter fields: ``sources`` (list), ``type`` (one of the entity - enum), ``brief`` (one-liner), and optional ``aliases`` (list, omitted - when empty). On update the new source is prepended and the body replaced + enum, capitalized on write), ``description`` (one-liner), and optional + ``aliases`` (list, omitted when empty). On update the new source is prepended and the body replaced with the LLM rewrite; ``type`` is preserved from the new write. """ entities_dir = wiki_dir / "entities" @@ -898,17 +876,14 @@ def _write_entity( return # Strip any frontmatter the LLM body may carry. - clean = content - if clean.startswith("---"): - end = clean.find("---", 3) - if end != -1: - clean = clean[end + 3:].lstrip("\n") + clean_parts = frontmatter.split(content) + clean = clean_parts[1].lstrip("\n") if clean_parts is not None else content - def _build_frontmatter(sources: list[str]) -> str: + def _build_entity_frontmatter(sources: list[str]) -> str: fm_lines = [_yaml_list_line("sources", sources)] - fm_lines.append(_yaml_kv_line("type", type_ or "other")) + fm_lines.append(_yaml_kv_line("type", (type_ or "other").title())) if brief: - fm_lines.append(_yaml_kv_line("brief", brief)) + fm_lines.append(_yaml_kv_line("description", brief)) if aliases: fm_lines.append(_yaml_list_line("aliases", aliases)) return "---\n" + "\n".join(fm_lines) + "\n---\n\n" @@ -917,12 +892,15 @@ def _build_frontmatter(sources: list[str]) -> str: existing = path.read_text(encoding="utf-8") if source_file not in existing: existing = _prepend_source_to_frontmatter(existing, source_file) - end = existing.find("---", 3) if existing.startswith("---") else -1 - if end != -1: - fm = existing[:end + 3] - fm = _set_fm_line(fm, "brief", brief) if brief else fm - fm = _set_fm_line(fm, "type", type_) if type_ else fm - existing = fm + "\n\n" + clean + ex_parts = frontmatter.split(existing) + if ex_parts is not None: + fm_block, _ = ex_parts + fm_block = _set_fm_line(fm_block, "description", brief) if brief else fm_block + fm_block = _set_fm_line(fm_block, "type", type_.title()) if type_ else fm_block + # Drop any legacy ``brief:`` key (migrated to ``description:``), + # mirroring _write_concept's update path. + fm_block = frontmatter.drop_line(fm_block, "brief") + existing = fm_block + "\n" + clean else: # Malformed/absent frontmatter (opening ``---`` with no closing # delimiter, or no frontmatter at all): rebuild valid frontmatter @@ -937,23 +915,14 @@ def _build_frontmatter(sources: list[str]) -> str: recovered = parsed break merged = [source_file] + [s for s in recovered if s != source_file] - existing = _build_frontmatter(merged) + clean + existing = _build_entity_frontmatter(merged) + clean path.write_text(existing, encoding="utf-8") return - path.write_text(_build_frontmatter([source_file]) + clean, encoding="utf-8") + path.write_text(_build_entity_frontmatter([source_file]) + clean, encoding="utf-8") -def _set_fm_line(fm: str, key: str, value: str) -> str: - """Set or replace a single scalar ``key:`` line inside a frontmatter block. - - ``fm`` includes the opening and closing ``---`` markers. Uses a lambda - replacement so values containing regex backrefs are inserted literally. - """ - line = _yaml_kv_line(key, value) - if re.search(rf"^{re.escape(key)}:", fm, flags=re.MULTILINE): - return re.sub(rf"^{re.escape(key)}:.*", lambda _m: line, fm, count=1, flags=re.MULTILINE) - return fm.replace("---\n", f"---\n{line}\n", 1) +_set_fm_line = frontmatter.set_line def _prepend_source_to_frontmatter(text: str, source_file: str) -> str: @@ -966,13 +935,17 @@ def _prepend_source_to_frontmatter(text: str, source_file: str) -> str: if not text.startswith("---"): return f"---\n{_yaml_list_line('sources', [source_file])}\n---\n\n" + text - fm_end = text.find("---", 3) - if fm_end == -1: + parts = frontmatter.split(text) + if parts is None: return text - fm_block = text[:fm_end] - body = text[fm_end:] - fm_lines = fm_block.split("\n") + fm_block, body = parts + # Split the fm_block into lines for per-line manipulation. fm_block ends + # with "\n---\n"; strip the trailing closing delimiter + newline to get + # the prefix lines (opening "---" + content lines), then re-append after. + fm_prefix, _, _ = fm_block.rpartition("\n---\n") + fm_lines = fm_prefix.split("\n") + closing = "\n---\n" for i, line in enumerate(fm_lines): if not line.lstrip().startswith("sources:"): @@ -984,10 +957,10 @@ def _prepend_source_to_frontmatter(text: str, source_file: str) -> str: return text items.insert(0, source_file) fm_lines[i] = _yaml_list_line("sources", items) - return "\n".join(fm_lines) + body + return "\n".join(fm_lines) + closing + body fm_lines.insert(1, _yaml_list_line("sources", [source_file])) - return "\n".join(fm_lines) + body + return "\n".join(fm_lines) + closing + body def _remove_source_from_frontmatter(text: str, source_file: str) -> tuple[str, bool]: @@ -1003,13 +976,14 @@ def _remove_source_from_frontmatter(text: str, source_file: str) -> tuple[str, b if not text.startswith("---"): return text, False - fm_end = text.find("---", 3) - if fm_end == -1: + parts = frontmatter.split(text) + if parts is None: return text, False - fm_block = text[:fm_end] - body = text[fm_end:] - fm_lines = fm_block.split("\n") + fm_block, body = parts + fm_prefix, _, _ = fm_block.rpartition("\n---\n") + fm_lines = fm_prefix.split("\n") + closing = "\n---\n" for i, line in enumerate(fm_lines): if not line.lstrip().startswith("sources:"): @@ -1021,7 +995,7 @@ def _remove_source_from_frontmatter(text: str, source_file: str) -> tuple[str, b return text, False items.remove(source_file) fm_lines[i] = _yaml_list_line("sources", items) - return "\n".join(fm_lines) + body, len(items) == 0 + return "\n".join(fm_lines) + closing + body, len(items) == 0 return text, False @@ -1225,17 +1199,15 @@ def scan_affected_pages(pages_dir: Path, source_file_marker: str) -> list[tuple[ return affected for path in sorted(pages_dir.glob("*.md")): text = path.read_text(encoding="utf-8") - if not text.startswith("---"): + fm_dict = frontmatter.parse(text) + if not fm_dict: continue - fm_end = text.find("---", 3) - if fm_end == -1: + sources = fm_dict.get("sources") + if not isinstance(sources, list): continue - for line in text[:fm_end].split("\n"): - if line.lstrip().startswith("sources:"): - items = _parse_yaml_list_value(line) - if items is not None and source_file_marker in items: - affected.append((path.stem, max(len(items) - 1, 0))) - break + items = [str(x) for x in sources] + if source_file_marker in items: + affected.append((path.stem, max(len(items) - 1, 0))) return affected @@ -1456,7 +1428,7 @@ def _write_v1_summary_stripped() -> None: "stripped %d ghost wikilink(s) from fallback v1 summary %s: %s", len(ghosts), doc_name, ghosts[:5], ) - _write_summary(wiki_dir, doc_name, cleaned) + _write_summary(wiki_dir, doc_name, cleaned, description=doc_brief) try: parsed = _parse_json(plan_raw) @@ -1627,7 +1599,7 @@ async def _gen_create(concept: dict) -> tuple[str, str, bool, str]: ], f"concept: {name}", response_format=_JSON_RESPONSE_FORMAT) try: parsed = _parse_json(raw) - brief = parsed.get("brief", "") + brief = parsed.get("description", "") # Parse succeeded: do NOT fall back to ``raw`` (the JSON string). # An empty/None ``content`` field yields "" so # ``_require_nonempty_content`` raises and the page is skipped, @@ -1645,11 +1617,8 @@ async def _gen_update(concept: dict) -> tuple[str, str, bool, str]: concept_path = wiki_dir / "concepts" / f"{_sanitize_concept_name(name)}.md" if concept_path.exists(): raw_text = concept_path.read_text(encoding="utf-8") - if raw_text.startswith("---"): - parts = raw_text.split("---", 2) - existing_content = parts[2].strip() if len(parts) >= 3 else raw_text - else: - existing_content = raw_text + ex_parts = frontmatter.split(raw_text) + existing_content = ex_parts[1].strip() if ex_parts is not None else raw_text else: existing_content = "(page not found — create from scratch)" async with semaphore: @@ -1665,7 +1634,7 @@ async def _gen_update(concept: dict) -> tuple[str, str, bool, str]: ], f"update: {name}", response_format=_JSON_RESPONSE_FORMAT) try: parsed = _parse_json(raw) - brief = parsed.get("brief", "") + brief = parsed.get("description", "") # Parse succeeded: do NOT fall back to ``raw`` (the JSON string). content = parsed.get("content") or "" except (json.JSONDecodeError, ValueError): @@ -1690,7 +1659,7 @@ async def _gen_entity_create(ent: dict) -> tuple[str, str, str, str]: ], f"entity: {name}", response_format=_JSON_RESPONSE_FORMAT) try: parsed = _parse_json(raw) - brief = parsed.get("brief", "") + brief = parsed.get("description", "") etype_out = parsed.get("type") if parsed.get("type") in valid_types else etype # Parse succeeded: do NOT fall back to ``raw`` (the JSON string). content = parsed.get("content") or "" @@ -1707,11 +1676,8 @@ async def _gen_entity_update(ent: dict) -> tuple[str, str, str, str]: epath = wiki_dir / "entities" / f"{_sanitize_concept_name(name)}.md" if epath.exists(): raw_text = epath.read_text(encoding="utf-8") - if raw_text.startswith("---"): - parts = raw_text.split("---", 2) - existing_content = parts[2].strip() if len(parts) >= 3 else raw_text - else: - existing_content = raw_text + ex_parts = frontmatter.split(raw_text) + existing_content = ex_parts[1].strip() if ex_parts is not None else raw_text else: existing_content = "(page not found — create from scratch)" async with semaphore: @@ -1727,7 +1693,7 @@ async def _gen_entity_update(ent: dict) -> tuple[str, str, str, str]: ], f"entity-update: {name}", response_format=_JSON_RESPONSE_FORMAT) try: parsed = _parse_json(raw) - brief = parsed.get("brief", "") + brief = parsed.get("description", "") etype_out = parsed.get("type") if parsed.get("type") in valid_types else etype # Parse succeeded: do NOT fall back to ``raw`` (the JSON string). content = parsed.get("content") or "" @@ -1875,10 +1841,9 @@ async def _gen_entity_update(ent: dict) -> tuple[str, str, str, str]: ], "summary-rewrite") candidate = rewrite_raw.strip() # Strip frontmatter if the model added one anyway. - if candidate.startswith("---"): - end = candidate.find("---", 3) - if end != -1: - candidate = candidate[end + 3:].lstrip("\n") + cand_parts = frontmatter.split(candidate) + if cand_parts is not None: + candidate = cand_parts[1].lstrip("\n") # Safety net: strip any wikilink the rewrite emitted that is # not in the whitelist. candidate, summary_ghosts = strip_ghost_wikilinks( @@ -1915,7 +1880,7 @@ async def _gen_entity_update(ent: dict) -> tuple[str, str, str, str]: "stripped %d ghost wikilink(s) from v1 fallback summary %s: %s", len(fallback_ghosts), doc_name, fallback_ghosts[:5], ) - _write_summary(wiki_dir, doc_name, final_summary) + _write_summary(wiki_dir, doc_name, final_summary, description=doc_brief) # --- Write concept pages to disk --- for name, page_content, is_update, brief in pending_writes: @@ -1997,7 +1962,7 @@ async def compile_short_doc( response_format=_JSON_RESPONSE_FORMAT) try: summary_parsed = _parse_json(summary_raw) - doc_brief = summary_parsed.get("brief", "") + doc_brief = summary_parsed.get("description", "") summary = summary_parsed.get("content", summary_raw) except (json.JSONDecodeError, ValueError): doc_brief = "" @@ -2041,6 +2006,20 @@ async def compile_long_doc( schema_md = get_agents_md(wiki_dir) summary_content = summary_path.read_text(encoding="utf-8") + # Backfill OKF fields on the indexer-written summary. Idempotent: set + # description before type so that when both keys are missing the prepends + # leave `type` first (canonical order); only rewrite when content changed. + fm_parts = frontmatter.split(summary_content) + if fm_parts is not None: + fm_block, body = fm_parts + if doc_description: + fm_block = _set_fm_line(fm_block, "description", doc_description) + fm_block = _set_fm_line(fm_block, "type", "Summary") + updated = fm_block + body + if updated != summary_content: + summary_content = updated + summary_path.write_text(summary_content, encoding="utf-8") + # Base context A. cache_control marker on the doc message creates a # cache breakpoint covering (system + doc) for every concept call. system_msg = {"role": "system", "content": _SYSTEM_TEMPLATE.format( diff --git a/openkb/frontmatter.py b/openkb/frontmatter.py new file mode 100644 index 00000000..0828496d --- /dev/null +++ b/openkb/frontmatter.py @@ -0,0 +1,110 @@ +"""Shared helpers for the YAML frontmatter blocks of OpenKB wiki pages. + +Single source of truth for building, splitting, parsing, and mutating the +``---`` frontmatter used by summaries / concepts / entities. The closing +delimiter is always matched at the start of a line (``\\n---``), so a ``---`` +that appears inside a quoted value never truncates the block — the failure +mode that ad-hoc ``text.find("---", 3)`` parsing was prone to. +""" +from __future__ import annotations + +import json +import re + +import yaml + + +def kv_line(key: str, value: str) -> str: + """Render ``key: "value"`` with the value JSON-quoted (always single-line). + + JSON strings are a strict subset of YAML: always single-line, always + correctly escaped (newlines, quotes, colons, control chars), and never + auto-promoted to a block scalar. + """ + return f"{key}: {json.dumps(value, ensure_ascii=False)}" + + +def list_line(key: str, items) -> str: + """Render ``key: ["a", "b"]`` as JSON-style YAML (always single-line).""" + return f"{key}: {json.dumps(list(items), ensure_ascii=False)}" + + +def block(lines: list[str]) -> str: + """Assemble a complete frontmatter block (with delimiters + trailing blank).""" + return "---\n" + "\n".join(lines) + "\n---\n\n" + + +def parse_list_value(line: str) -> list[str] | None: + """Parse the right-hand side of ``key: [...]`` into a list of strings. + + Returns ``None`` when the value cannot be interpreted as a list — callers + treat that as "leave the frontmatter alone". + """ + colon = line.find(":") + if colon == -1: + return None + try: + parsed = yaml.safe_load(line[colon + 1:]) + except yaml.YAMLError: + return None + if not isinstance(parsed, list): + return None + return [str(x) for x in parsed] + + +def split(text: str) -> tuple[str, str] | None: + """Split ``text`` into ``(frontmatter_block, body)``. + + ``frontmatter_block`` includes both ``---`` delimiters and the newline that + ends the closing delimiter line; ``body`` is everything after, so + ``frontmatter_block + body == text`` exactly (lossless). + + Returns ``None`` when ``text`` has no well-formed frontmatter: no leading + ``---`` or no line-anchored closing ``---``. Because the closing delimiter + must start a line (``\\n---``), a ``---`` inside a quoted value is ignored. + """ + if not text.startswith("---"): + return None + nl = text.find("\n---", 3) + if nl == -1: + return None + after = text.find("\n", nl + 1) # newline ending the closing '---' line + if after == -1: + return text, "" + return text[:after + 1], text[after + 1:] + + +def parse(text: str) -> dict: + """Return the frontmatter as a dict (``{}`` when absent or malformed).""" + parts = split(text) + if parts is None: + return {} + fm_block = parts[0] + inner = fm_block[3:] # drop opening '---' + close = inner.rfind("\n---") # drop closing '---' line + if close != -1: + inner = inner[:close] + try: + data = yaml.safe_load(inner) + except yaml.YAMLError: + return {} + return data if isinstance(data, dict) else {} + + +def set_line(fm_block: str, key: str, value: str) -> str: + """Set or insert a single scalar ``key:`` line in a frontmatter block. + + Replaces an existing line for ``key``; otherwise inserts it right after the + opening ``---``. A lambda replacement is used so values containing regex + backrefs (``\\1``, ``\\g<…>``) are inserted literally. + """ + line = kv_line(key, value) + if re.search(rf"^{re.escape(key)}:", fm_block, flags=re.MULTILINE): + return re.sub(rf"^{re.escape(key)}:.*", lambda _m: line, fm_block, + count=1, flags=re.MULTILINE) + return fm_block.replace("---\n", f"---\n{line}\n", 1) + + +def drop_line(fm_block: str, key: str) -> str: + """Remove any ``key:`` line from a frontmatter block (no-op if absent).""" + return re.sub(rf"^{re.escape(key)}:.*\n?", "", fm_block, flags=re.MULTILINE) diff --git a/openkb/indexer.py b/openkb/indexer.py index 00692319..348c7e32 100644 --- a/openkb/indexer.py +++ b/openkb/indexer.py @@ -174,7 +174,7 @@ def index_long_document( # Write wiki/summaries/ (no images, just summaries) summaries_dir = kb_dir / "wiki" / "summaries" summaries_dir.mkdir(parents=True, exist_ok=True) - summary_md = render_summary_md(tree, source_name, doc_id) + summary_md = render_summary_md(tree, source_name, doc_id, description=description) (summaries_dir / f"{source_name}.md").write_text(summary_md, encoding="utf-8") return IndexResult(doc_id=doc_id, description=description, tree=tree) diff --git a/openkb/lint.py b/openkb/lint.py index ecbfa8a5..8b7674b7 100644 --- a/openkb/lint.py +++ b/openkb/lint.py @@ -15,6 +15,7 @@ import yaml +from openkb import frontmatter from openkb.schema import PAGE_CONTENT_DIRS # Matches [[wikilink]] or [[subdir/link]] @@ -449,35 +450,66 @@ def check_index_sync(wiki: Path) -> list[str]: return sorted(issues) -def find_invalid_frontmatter(wiki: Path) -> list[str]: +def _load_wiki_pages(wiki: Path) -> dict[Path, str]: + """Read all wiki .md files once and return a ``{path: text}`` mapping. + + Applies the same scope as :func:`find_invalid_frontmatter`: walks the + whole wiki, skips names in :data:`_EXCLUDED_FILES`, and skips any path + whose first relative-to-wiki directory part is ``"reports"`` or + ``"sources"``. The result is the superset that both frontmatter checks + iterate; :func:`find_missing_okf_fields` additionally filters down to + :data:`PAGE_CONTENT_DIRS ` paths. + """ + pages: dict[Path, str] = {} + for path in wiki.rglob("*.md"): + if path.name in _EXCLUDED_FILES: + continue + rel_parts = path.relative_to(wiki).parts + if rel_parts and rel_parts[0] in ("reports", "sources"): + continue + pages[path] = _read_md(path) + return pages + + +def find_invalid_frontmatter( + wiki: Path, + pages: dict[Path, str] | None = None, +) -> list[str]: """Return wiki pages whose YAML frontmatter fails ``yaml.safe_load``. Catches the silent-write class of bug where an LLM-authored field (e.g. ``brief:``) ships unquoted and turns a colon-bearing value into invalid YAML that OpenKB itself reads with string slicing but external YAML-aware tools (VS Code, Obsidian, doc generators) reject. + + Args: + wiki: Path to the wiki root directory. + pages: Optional pre-loaded ``{path: text}`` mapping from + :func:`_load_wiki_pages`. When ``None`` (the default), the + mapping is built internally so existing callers that pass only + ``wiki`` keep working unchanged. """ issues: list[str] = [] if not wiki.exists(): return issues - for path in sorted(wiki.rglob("*.md")): - if path.name in _EXCLUDED_FILES: - continue - # Skip reports/ and sources/ — auto-generated / user-uploaded - # content, not wiki pages we manage. Matches the convention in - # find_broken_links / find_orphans. - rel_parts = path.relative_to(wiki).parts - if rel_parts and rel_parts[0] in ("reports", "sources"): - continue - text = _read_md(path) - if not text.startswith("---"): - continue - end = text.find("\n---", 3) - if end == -1: + if pages is None: + pages = _load_wiki_pages(wiki) + for path, text in sorted(pages.items()): + parts = frontmatter.split(text) + if parts is None: continue - fm = text[3:end].strip("\n") + fm_block = parts[0] + # Extract the raw YAML between the two ``---`` delimiters so that + # yaml.safe_load can surface the exact error message. The closing + # delimiter is line-anchored (frontmatter.split guarantees this), + # so we strip the opening ``---\n`` and everything from the final + # ``\n---`` onward. + inner = fm_block[4:] # drop "---\n" + close = inner.rfind("\n---") + if close != -1: + inner = inner[:close] try: - yaml.safe_load(fm) + yaml.safe_load(inner) except yaml.YAMLError as exc: rel = path.relative_to(wiki) msg = str(exc).splitlines()[0] @@ -485,6 +517,44 @@ def find_invalid_frontmatter(wiki: Path) -> list[str]: return issues +def find_missing_okf_fields( + wiki: Path, + pages: dict[Path, str] | None = None, +) -> list[str]: + """Return knowledge pages missing a non-empty ``type`` or ``description``. + + OKF v0.1 requires every non-reserved knowledge page to carry a non-empty + ``type``; ``description`` is OpenKB's required one-liner. Only summaries/, + concepts/, entities/ are checked — index.md, log.md and sources/ are exempt. + + Args: + wiki: Path to the wiki root directory. + pages: Optional pre-loaded ``{path: text}`` mapping from + :func:`_load_wiki_pages`. When ``None`` (the default), the + mapping is built internally so existing callers that pass only + ``wiki`` keep working unchanged. + """ + issues: list[str] = [] + if not wiki.exists(): + return issues + if pages is None: + pages = _load_wiki_pages(wiki) + for path, text in sorted(pages.items()): + # find_missing_okf_fields covers only PAGE_CONTENT_DIRS subsets. + rel_parts = path.relative_to(wiki).parts + if not rel_parts or rel_parts[0] not in PAGE_CONTENT_DIRS: + continue + fm = frontmatter.parse(text) + rel = path.relative_to(wiki) + type_val = fm.get("type") + if not (isinstance(type_val, str) and type_val.strip()): + issues.append(f"{rel}: missing non-empty 'type'") + desc_val = fm.get("description") + if not (isinstance(desc_val, str) and desc_val.strip()): + issues.append(f"{rel}: missing non-empty 'description'") + return issues + + def run_structural_lint(kb_dir: Path) -> str: """Run all structural lint checks and return a formatted Markdown report. @@ -497,11 +567,16 @@ def run_structural_lint(kb_dir: Path) -> str: wiki = kb_dir / "wiki" raw = kb_dir / "raw" + # Load all wiki pages once and share across both frontmatter checks + # (avoids 2N disk reads when the wiki is large). + wiki_pages = _load_wiki_pages(wiki) if wiki.exists() else {} + broken = find_broken_links(wiki) orphans = find_orphans(wiki) missing = find_missing_entries(raw, wiki, kb_dir=kb_dir) sync_issues = check_index_sync(wiki) - bad_frontmatter = find_invalid_frontmatter(wiki) + bad_frontmatter = find_invalid_frontmatter(wiki, pages=wiki_pages) + missing_okf = find_missing_okf_fields(wiki, pages=wiki_pages) lines = ["## Structural Lint Report\n"] @@ -548,5 +623,14 @@ def run_structural_lint(kb_dir: Path) -> str: lines.append(f"- {issue}") else: lines.append("All frontmatter parses as valid YAML.") + lines.append("") + + # OKF conformance + lines.append(f"### OKF Conformance ({len(missing_okf)})") + if missing_okf: + for issue in missing_okf: + lines.append(f"- {issue}") + else: + lines.append("All knowledge pages carry required OKF fields.") return "\n".join(lines) diff --git a/openkb/schema.py b/openkb/schema.py index 8a6322e6..57d2dc5e 100644 --- a/openkb/schema.py +++ b/openkb/schema.py @@ -48,7 +48,13 @@ - Use [[wikilink]] to link other wiki pages (e.g., [[concepts/attention]]) - Standard Markdown heading hierarchy - Keep each page focused on a single topic -- Do not include YAML frontmatter (---) in generated content; it is managed by code + +## Frontmatter (managed by code — do NOT emit it in generated content) +- Every summary/concept/entity page carries a non-empty `type:` — `Summary`, + `Concept`, or a capitalized entity subtype (e.g. `Organization`). This is the + one field OKF requires; consumers use it for routing/filtering/presentation. +- `description:` — a single-sentence one-liner (the field formerly named `brief`). +- Do not include YAML frontmatter (---) in generated content; it is managed by code. """ # Backward compat alias diff --git a/openkb/tree_renderer.py b/openkb/tree_renderer.py index efad9802..9bca158c 100644 --- a/openkb/tree_renderer.py +++ b/openkb/tree_renderer.py @@ -1,16 +1,17 @@ """Markdown renderers for PageIndex tree structures.""" from __future__ import annotations +from openkb import frontmatter -def _yaml_frontmatter(source_name: str, doc_id: str) -> str: - """Return a YAML frontmatter block for a PageIndex wiki page.""" - return ( - "---\n" - "doc_type: pageindex\n" - f"full_text: sources/{source_name}.json\n" - "---\n" - ) +def _yaml_frontmatter(source_name: str, doc_id: str, description: str = "") -> str: + """Return a YAML frontmatter block for a PageIndex wiki page.""" + lines = [frontmatter.kv_line("type", "Summary")] + if description: + lines.append(frontmatter.kv_line("description", description)) + lines.append("doc_type: pageindex") + lines.append(frontmatter.kv_line("full_text", f"sources/{source_name}.json")) + return "---\n" + "\n".join(lines) + "\n---\n" def _render_nodes_summary(nodes: list[dict], depth: int) -> str: @@ -24,7 +25,7 @@ def _render_nodes_summary(nodes: list[dict], depth: int) -> str: summary = node.get("summary", "") children = node.get("nodes", []) - lines.append(f"{heading_prefix} {title} (pages {start}\u2013{end})\n") + lines.append(f"{heading_prefix} {title} (pages {start}–{end})\n") if summary: lines.append(f"Summary: {summary}\n") if children: @@ -33,13 +34,15 @@ def _render_nodes_summary(nodes: list[dict], depth: int) -> str: return "\n".join(lines) - -def render_summary_md(tree: dict, source_name: str, doc_id: str) -> str: +def render_summary_md(tree: dict, source_name: str, doc_id: str, + description: str = "") -> str: """Render the summary Markdown page for a PageIndex tree. Renders each node as a heading with page range and its summary text. + Includes a YAML frontmatter block with ``type: "Summary"`` and an + optional ``description`` field. """ - frontmatter = _yaml_frontmatter(source_name, doc_id) + frontmatter = _yaml_frontmatter(source_name, doc_id, description) structure = tree.get("structure", []) body = _render_nodes_summary(structure, depth=1) return frontmatter + "\n" + body diff --git a/tests/test_compiler.py b/tests/test_compiler.py index adbae676..dfc6d6ff 100644 --- a/tests/test_compiler.py +++ b/tests/test_compiler.py @@ -200,25 +200,25 @@ def test_nfkc_normalization(self): class TestWriteSummary: - def test_writes_with_frontmatter(self, tmp_path): + def test_writes_type_and_description(self, tmp_path): wiki = tmp_path / "wiki" wiki.mkdir() - _write_summary(wiki, "my-doc", "# Summary\n\nContent here.") - path = wiki / "summaries" / "my-doc.md" - assert path.exists() - text = path.read_text() + _write_summary(wiki, "my-doc", "# Summary\n\nContent.", + description="A one-line summary.") + text = (wiki / "summaries" / "my-doc.md").read_text() + assert 'type: "Summary"' in text + assert 'description: "A one-line summary."' in text assert "doc_type: short" in text - assert "full_text: sources/my-doc.md" in text + assert 'full_text: "sources/my-doc.md"' in text assert "# Summary" in text - def test_writes_without_brief(self, tmp_path): + def test_omits_description_when_empty(self, tmp_path): wiki = tmp_path / "wiki" wiki.mkdir() - _write_summary(wiki, "my-doc", "# Summary\n\nContent here.") - path = wiki / "summaries" / "my-doc.md" - text = path.read_text() - assert "doc_type: short" in text - assert "full_text: sources/my-doc.md" in text + _write_summary(wiki, "my-doc", "# Summary\n\nContent.") + text = (wiki / "summaries" / "my-doc.md").read_text() + assert 'type: "Summary"' in text + assert "description:" not in text class TestWriteConcept: @@ -230,7 +230,7 @@ def test_new_concept_with_brief(self, tmp_path): assert path.exists() text = path.read_text() assert 'sources: ["paper.pdf"]' in text - assert 'brief: "Mechanism for selective focus"' in text + assert 'description: "Mechanism for selective focus"' in text assert "# Attention" in text def test_new_concept_without_brief(self, tmp_path): @@ -255,7 +255,7 @@ def test_update_concept_updates_brief(self, tmp_path): text = (concepts / "attention.md").read_text() assert "paper2.pdf" in text assert "paper1.pdf" in text - assert 'brief: "Updated brief"' in text + assert 'description: "Updated brief"' in text assert "Old brief" not in text def test_update_concept_appends_source(self, tmp_path): @@ -288,6 +288,38 @@ def test_update_concept_merges_into_non_canonical_sources(self, tmp_path): assert "paper2.pdf" in text assert "New info from paper2." in text + def test_new_concept_has_type_and_description(self, tmp_path): + wiki = tmp_path / "wiki" + wiki.mkdir() + _write_concept(wiki, "attention", "# Attention\n\nDetails.", "summaries/p.md", + False, brief="Mechanism for selective focus") + text = (wiki / "concepts" / "attention.md").read_text() + assert 'type: "Concept"' in text + assert 'description: "Mechanism for selective focus"' in text + assert "brief:" not in text + + def test_new_concept_without_description_still_has_type(self, tmp_path): + wiki = tmp_path / "wiki" + wiki.mkdir() + _write_concept(wiki, "attention", "# Attention\n\nDetails.", "summaries/p.md", False) + text = (wiki / "concepts" / "attention.md").read_text() + assert 'type: "Concept"' in text + assert "description:" not in text + + def test_update_concept_sets_type_and_description(self, tmp_path): + wiki = tmp_path / "wiki" + concepts = wiki / "concepts" + concepts.mkdir(parents=True) + (concepts / "attention.md").write_text( + '---\nsources: ["p1.pdf"]\ndescription: "Old"\n---\n\n# Attention\n\nOld.', + encoding="utf-8", + ) + _write_concept(wiki, "attention", "New.", "summaries/p2.md", True, brief="New one") + text = (concepts / "attention.md").read_text() + assert 'type: "Concept"' in text + assert 'description: "New one"' in text + assert "Old" not in text + class TestUpdateIndex: def test_appends_entries_with_briefs(self, tmp_path): @@ -549,6 +581,16 @@ def test_falls_back_to_body_truncation(self, tmp_path): result = _read_concept_briefs(wiki) assert "- old: Old concept without brief field." in result + def test_reads_description_field(self, tmp_path): + wiki = tmp_path / "wiki" + concepts = wiki / "concepts" + concepts.mkdir(parents=True) + (concepts / "attention.md").write_text( + '---\nsources: ["p.pdf"]\ndescription: "Selective focus"\n---\n\n# A\n', + encoding="utf-8", + ) + assert "- attention: Selective focus" in _read_concept_briefs(wiki) + class TestReadEntityBriefs: def test_none_when_missing(self, tmp_path): @@ -607,6 +649,31 @@ def test_sorted_alphabetically(self, tmp_path): assert lines[0].startswith("- alpha ") assert lines[1].startswith("- zeta ") + def test_reads_description_and_lowercases_type(self, tmp_path): + ent = tmp_path / "entities" + ent.mkdir() + (ent / "anthropic.md").write_text( + "---\n" + 'sources: ["summaries/a.md", "summaries/b.md"]\n' + 'type: "Organization"\n' + 'description: "AI lab behind Claude."\n' + "---\n\n# Anthropic\n", + encoding="utf-8", + ) + out = _read_entity_briefs(tmp_path) + assert out == "- anthropic (organization, 2 sources) — AI lab behind Claude." + + def test_reads_legacy_brief_when_no_description(self, tmp_path): + ent = tmp_path / "entities" + ent.mkdir() + (ent / "openai.md").write_text( + "---\ntype: organization\nsources: [summaries/a.md]\n" + "brief: Legacy one-liner.\n---\n\n# OpenAI\n", + encoding="utf-8", + ) + out = _read_entity_briefs(tmp_path) + assert "— Legacy one-liner." in out + class TestWriteEntity: def test_new_entity_frontmatter(self, tmp_path): @@ -617,8 +684,8 @@ def test_new_entity_frontmatter(self, tmp_path): aliases=["Anthropic PBC"], ) text = (tmp_path / "entities" / "anthropic.md").read_text(encoding="utf-8") - assert "type:" in text and "organization" in text - assert "brief:" in text and "AI lab behind Claude." in text + assert 'type: "Organization"' in text + assert 'description: "AI lab behind Claude."' in text assert "sources:" in text and "summaries/a.md" in text assert "Anthropic PBC" in text assert text.count("---") == 2 # exactly one frontmatter block @@ -638,10 +705,10 @@ def test_update_prepends_source_keeps_type(self, tmp_path): assert "summaries/b.md" in text and "summaries/a.md" in text # _yaml_list_line uses json.dumps: b prepended before a, double-quoted assert '"summaries/b.md", "summaries/a.md"' in text - assert "type:" in text and "organization" in text + assert 'type: "Organization"' in text assert "v2 richer." in text assert "v1." not in text - assert "brief:" in text and "b2" in text + assert 'description: "b2"' in text def test_update_rebuilds_frontmatter_when_no_closing_delim(self, tmp_path): """#11: malformed existing file (opening --- but no closing ---) must @@ -666,10 +733,72 @@ def test_update_rebuilds_frontmatter_when_no_closing_delim(self, tmp_path): assert "sources:" in text and "summaries/b.md" in text # The PRE-EXISTING source must be preserved, not dropped when rebuilding. assert "summaries/a.md" in text - assert "type:" in text and "organization" in text - assert "brief:" in text and "AI lab." in text + assert 'type: "Organization"' in text + assert 'description: "AI lab."' in text assert "v2 rewritten." in text + def test_new_entity_type_capitalized_and_description(self, tmp_path): + _write_entity( + tmp_path, "anthropic", "# Anthropic\n\nAI lab.", + "summaries/a.md", is_update=False, + brief="AI lab behind Claude.", type_="organization", + ) + text = (tmp_path / "entities" / "anthropic.md").read_text(encoding="utf-8") + assert 'type: "Organization"' in text # capitalized + assert 'description: "AI lab behind Claude."' in text + assert "brief:" not in text # renamed, not duplicated + + def test_update_entity_capitalizes_type_and_writes_description(self, tmp_path): + _write_entity(tmp_path, "anthropic", "# A\n\nv1.", "summaries/a.md", + is_update=False, brief="b1", type_="organization") + _write_entity(tmp_path, "anthropic", "# A\n\nv2.", "summaries/b.md", + is_update=True, brief="b2", type_="organization") + text = (tmp_path / "entities" / "anthropic.md").read_text(encoding="utf-8") + assert 'type: "Organization"' in text + assert 'description: "b2"' in text + assert "brief:" not in text + + def test_update_entity_strips_legacy_brief(self, tmp_path): + entities = tmp_path / "entities" + entities.mkdir(parents=True) + (entities / "anthropic.md").write_text( + '---\nsources: ["summaries/a.md"]\ntype: organization\n' + 'brief: Old brief.\n---\n\n# Anthropic\n\nOld.', + encoding="utf-8", + ) + _write_entity(tmp_path, "anthropic", "# Anthropic\n\nv2.", "summaries/b.md", + is_update=True, brief="New desc.", type_="organization") + text = (entities / "anthropic.md").read_text(encoding="utf-8") + assert "brief:" not in text + assert "Old brief." not in text + assert 'description: "New desc."' in text + + def test_entity_type_multiword_title_cased(self, tmp_path): + _write_entity(tmp_path, "acme", "# Acme\n\nx.", "summaries/a.md", + is_update=False, brief="b", type_="real estate") + text = (tmp_path / "entities" / "acme.md").read_text(encoding="utf-8") + assert 'type: "Real Estate"' in text + + +def test_update_keeps_single_blank_line_after_frontmatter(tmp_path): + """Regression: the update path must not accumulate a 3rd newline after ---.""" + wiki = tmp_path / "wiki" + (wiki / "concepts").mkdir(parents=True) + (wiki / "concepts" / "x.md").write_text( + '---\ntype: "Concept"\nsources: ["a"]\ndescription: "old"\n---\n\n# X\n', + encoding="utf-8") + _write_concept(wiki, "x", "# X\n\nNew.", "summaries/b.md", True, brief="new") + ctext = (wiki / "concepts" / "x.md").read_text(encoding="utf-8") + assert "---\n\n\n" not in ctext and "---\n\n" in ctext + + (wiki / "entities").mkdir(parents=True) + (wiki / "entities" / "e.md").write_text( + '---\nsources: ["a"]\ntype: "Person"\ndescription: "old"\n---\n\n# E\n', + encoding="utf-8") + _write_entity(wiki, "e", "# E\n\nNew.", "summaries/b.md", True, brief="new", type_="person") + etext = (wiki / "entities" / "e.md").read_text(encoding="utf-8") + assert "---\n\n\n" not in etext and "---\n\n" in etext + class TestBacklinkSummary: def test_adds_missing_concept_links(self, tmp_path): @@ -926,7 +1055,7 @@ async def test_full_pipeline(self, tmp_path): (tmp_path / "raw" / "test-doc.pdf").write_bytes(b"fake") summary_response = json.dumps({ - "brief": "Discusses transformers", + "description": "Discusses transformers", "content": "# Summary\n\nThis document discusses transformers.", }) concepts_list_response = json.dumps({ @@ -960,7 +1089,8 @@ async def test_full_pipeline(self, tmp_path): summary_path = wiki / "summaries" / "test-doc.md" assert summary_path.exists() summary_text = summary_path.read_text() - assert "full_text: sources/test-doc.md" in summary_text + assert 'full_text: "sources/test-doc.md"' in summary_text + assert 'type: "Summary"' in summary_text # Summary body comes from the rewrite step assert "[[concepts/transformer]]" in summary_text @@ -1612,7 +1742,7 @@ async def test_short_doc_briefs_in_index_and_frontmatter(self, tmp_path): (tmp_path / "raw" / "test-doc.pdf").write_bytes(b"fake") summary_resp = json.dumps({ - "brief": "A paper about transformers", + "description": "A paper about transformers", "content": "# Summary\n\nThis paper discusses transformers.", }) plan_resp = json.dumps({ @@ -1621,7 +1751,7 @@ async def test_short_doc_briefs_in_index_and_frontmatter(self, tmp_path): "related": [], }) concept_resp = json.dumps({ - "brief": "NN architecture using self-attention", + "description": "NN architecture using self-attention", "content": "# Transformer\n\nA neural network architecture.", }) @@ -1637,11 +1767,11 @@ async def test_short_doc_briefs_in_index_and_frontmatter(self, tmp_path): # Summary frontmatter has doc_type and full_text summary_text = (wiki / "summaries" / "test-doc.md").read_text() assert "doc_type: short" in summary_text - assert "full_text: sources/test-doc.md" in summary_text + assert 'full_text: "sources/test-doc.md"' in summary_text - # Concept frontmatter has brief + # Concept frontmatter has type and description concept_text = (wiki / "concepts" / "transformer.md").read_text() - assert 'brief: "NN architecture using self-attention"' in concept_text + assert 'description: "NN architecture using self-attention"' in concept_text # Index has briefs index_text = (wiki / "index.md").read_text() @@ -1759,7 +1889,7 @@ def fake_llm(model, messages, label, **kw): "type": "organization"}], "update": [], "related": []}, }) - return json.dumps({"brief": "b", "type": "organization", "content": "# Page\n"}) + return json.dumps({"description": "b", "type": "organization", "content": "# Page\n"}) async def fake_llm_async(model, messages, label, **kw): return fake_llm(model, messages, label, **kw) @@ -1777,9 +1907,7 @@ async def fake_llm_async(model, messages, label, **kw): assert (wiki / "concepts" / "ai-demand.md").exists() assert (wiki / "entities" / "nvidia.md").exists() ent = (wiki / "entities" / "nvidia.md").read_text(encoding="utf-8") - # Frontmatter values are JSON-quoted by _yaml_kv_line (see _write_entity, - # Task 2), matching the tolerant assertion style in TestWriteEntity. - assert "type:" in ent and "organization" in ent + assert 'type: "Organization"' in ent index = (wiki / "index.md").read_text(encoding="utf-8") assert "[[entities/nvidia]]" in index summary = (wiki / "summaries" / "doc.md").read_text(encoding="utf-8") @@ -1900,7 +2028,7 @@ def fake_llm(model, messages, label, **kw): "type": "dataset"}], "update": [], "related": []}, }) - return json.dumps({"brief": "b", "type": "dataset", "content": "# Page\n"}) + return json.dumps({"description": "b", "type": "dataset", "content": "# Page\n"}) async def fake_llm_async(model, messages, label, **kw): seen_messages.append((label, messages)) @@ -1920,7 +2048,7 @@ async def fake_llm_async(model, messages, label, **kw): ) ent = (wiki / "entities" / "imagenet.md").read_text(encoding="utf-8") - assert "type:" in ent and "dataset" in ent + assert 'type: "Dataset"' in ent # The custom type must reach the plan prompt the mock saw. plan_msgs = [m for (label, m) in seen_messages if label == "concepts-plan"] assert plan_msgs, "plan call was not made" @@ -2097,3 +2225,50 @@ async def test_llm_call_async_injects_extra_headers(self): assert out == "ok" kwargs = mock_litellm.acompletion.call_args.kwargs assert kwargs["extra_headers"] == {"Copilot-Integration-Id": "vscode-chat"} + + +class TestFrontmatterDashBoundary: + """Regression: description containing '---' must not truncate frontmatter.""" + + def test_concept_round_trip_with_dashes_in_brief(self, tmp_path): + wiki = tmp_path / "wiki" + wiki.mkdir() + # Write concept with a brief containing '---'. + brief = "--- note ---" + _write_concept(wiki, "tricky", "# Body\n\nContent.", "summaries/doc.md", + is_update=False, brief=brief) + # Round-trip: _read_concept_briefs must return the brief intact. + result = _read_concept_briefs(wiki) + assert "--- note ---" in result + # The body must not be corrupted. + text = (wiki / "concepts" / "tricky.md").read_text(encoding="utf-8") + assert "# Body" in text + assert "Content." in text + + def test_entity_round_trip_with_dashes_in_brief(self, tmp_path): + wiki = tmp_path / "wiki" + wiki.mkdir() + brief = "--- note ---" + _write_entity(wiki, "tricky-org", "# Body\n\nContent.", "summaries/doc.md", + is_update=False, brief=brief, type_="organization") + result = _read_entity_briefs(wiki) + assert "--- note ---" in result + text = (wiki / "entities" / "tricky-org.md").read_text(encoding="utf-8") + assert "# Body" in text + assert "Content." in text + + def test_concept_update_malformed_frontmatter_rebuilds(self, tmp_path): + """_write_concept(is_update=True) on a file with malformed frontmatter + must rebuild valid frontmatter, not write a bare body.""" + concepts = tmp_path / "concepts" + concepts.mkdir(parents=True) + # Opening '---' with no closing delimiter. + malformed = "---\nsources: [x]\nno close\n\nbody" + (concepts / "tricky.md").write_text(malformed, encoding="utf-8") + _write_concept(tmp_path, "tricky", "# New\n\nNew body.", "summaries/doc.md", + is_update=True, brief="brief text") + text = (concepts / "tricky.md").read_text(encoding="utf-8") + assert text.startswith("---\n") + assert 'type: "Concept"' in text + # Must have a properly closed frontmatter block (two '---' occurrences). + assert text.count("---") >= 2 diff --git a/tests/test_frontmatter.py b/tests/test_frontmatter.py new file mode 100644 index 00000000..0eb75b0d --- /dev/null +++ b/tests/test_frontmatter.py @@ -0,0 +1,104 @@ +"""Tests for openkb.frontmatter — the shared frontmatter helper module.""" +from __future__ import annotations + +import yaml + +from openkb import frontmatter as fm + + +class TestKvLine: + def test_basic(self): + assert fm.kv_line("type", "Concept") == 'type: "Concept"' + + def test_value_with_colon_and_quotes_round_trips(self): + line = fm.kv_line("description", 'a: "b" — c') + assert yaml.safe_load(line)["description"] == 'a: "b" — c' + + def test_unicode_preserved(self): + assert fm.kv_line("title", "café") == 'title: "café"' + + +class TestListLine: + def test_basic(self): + assert fm.list_line("sources", ["a", "b"]) == 'sources: ["a", "b"]' + + +class TestBlock: + def test_assembles_with_delimiters(self): + out = fm.block([fm.kv_line("type", "Concept")]) + assert out == '---\ntype: "Concept"\n---\n\n' + + +class TestSplit: + def test_basic(self): + text = '---\ntype: "Concept"\n---\n\nbody here' + block, body = fm.split(text) + assert block == '---\ntype: "Concept"\n---\n' + assert body == '\nbody here' + assert block + body == text # lossless + + def test_dashes_inside_value_do_not_truncate(self): + text = '---\ntype: "Concept"\ndescription: "--- x ---"\n---\nbody' + block, body = fm.split(text) + assert 'description: "--- x ---"' in block + assert body == 'body' + + def test_no_frontmatter(self): + assert fm.split("no frontmatter here") is None + + def test_unterminated_frontmatter(self): + assert fm.split("---\ntype: x\nbut no close") is None + + +class TestParse: + def test_basic(self): + assert fm.parse('---\ntype: "Concept"\nx: 1\n---\nbody') == {"type": "Concept", "x": 1} + + def test_dashes_inside_value(self): + d = fm.parse('---\ntype: "Concept"\ndescription: "--- x"\n---\nbody') + assert d["type"] == "Concept" + assert d["description"] == "--- x" + + def test_absent_returns_empty(self): + assert fm.parse("plain body") == {} + + def test_malformed_yaml_returns_empty(self): + # an unclosed flow sequence makes safe_load raise; we must degrade to {} + assert fm.parse("---\nfoo: [unclosed\n---\n") == {} + + +class TestSetLine: + def test_replaces_existing(self): + out = fm.set_line('---\ntype: "Old"\n---\n', "type", "New") + assert 'type: "New"' in out + assert "Old" not in out + + def test_inserts_when_absent(self): + out = fm.set_line('---\nsources: ["a"]\n---\n', "type", "Concept") + assert 'type: "Concept"' in out + assert 'sources: ["a"]' in out + + def test_value_with_regex_backref_is_literal(self): + out = fm.set_line('---\ndescription: "old"\n---\n', "description", r"a\1b") + # lambda replacement keeps the backref literal; json.dumps escapes it, + # and a YAML loader reads it back unchanged. + assert fm.parse(out)["description"] == r"a\1b" + + +class TestDropLine: + def test_removes_key(self): + out = fm.drop_line('---\ntype: "C"\nbrief: gone\n---\n', "brief") + assert "brief" not in out + assert 'type: "C"' in out + + def test_noop_when_absent(self): + block = '---\ntype: "C"\n---\n' + assert fm.drop_line(block, "brief") == block + + +class TestParseListValue: + def test_basic(self): + assert fm.parse_list_value('sources: ["a", "b"]') == ["a", "b"] + + def test_non_list_returns_none(self): + assert fm.parse_list_value("type: Concept") is None diff --git a/tests/test_lint.py b/tests/test_lint.py index 8d44b3e6..85929a53 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -5,11 +5,15 @@ from openkb.lint import ( + _EXCLUDED_FILES, + _load_wiki_pages, _normalize_target, build_norm_index, check_index_sync, find_broken_links, + find_invalid_frontmatter, find_missing_entries, + find_missing_okf_fields, find_orphans, fix_broken_links, list_existing_wiki_targets, @@ -649,3 +653,117 @@ def test_whitelist_includes_entities(tmp_path): (tmp_path / "entities" / "anthropic.md").write_text("# A", encoding="utf-8") targets = list_existing_wiki_targets(tmp_path) assert "entities/anthropic" in targets + + +def test_flags_missing_type_and_description(tmp_path): + wiki = tmp_path / "wiki" + for sub in ("summaries", "concepts", "entities"): + (wiki / sub).mkdir(parents=True) + (wiki / "concepts" / "good.md").write_text( + '---\ntype: "Concept"\ndescription: "ok"\n---\n\n# Good\n', encoding="utf-8") + (wiki / "concepts" / "no_type.md").write_text( + '---\ndescription: "x"\n---\n\n# Bad\n', encoding="utf-8") + (wiki / "summaries" / "no_desc.md").write_text( + '---\ntype: "Summary"\n---\n\n# Bad\n', encoding="utf-8") + issues = find_missing_okf_fields(wiki) + assert any("no_type.md" in i and "type" in i for i in issues) + assert any("no_desc.md" in i and "description" in i for i in issues) + assert not any("good.md" in i for i in issues) + + +def test_flags_null_type_as_missing(tmp_path): + wiki = tmp_path / "wiki" + (wiki / "concepts").mkdir(parents=True) + (wiki / "concepts" / "null_type.md").write_text( + '---\ntype: null\ndescription: "x"\n---\n\n# Bad\n', encoding="utf-8") + issues = find_missing_okf_fields(wiki) + assert any("null_type.md" in i and "type" in i for i in issues) + + +def test_flags_non_string_type_as_missing(tmp_path): + wiki = tmp_path / "wiki" + (wiki / "concepts").mkdir(parents=True) + (wiki / "concepts" / "bool_type.md").write_text( + '---\ntype: true\ndescription: "x"\n---\n\n# Bad\n', encoding="utf-8") + issues = find_missing_okf_fields(wiki) + assert any("bool_type.md" in i and "type" in i for i in issues) + + +class TestLoadWikiPages: + """Tests for :func:`_load_wiki_pages` scope and exclusion rules.""" + + def test_excludes_reports_directory(self, tmp_path): + wiki = _make_wiki(tmp_path) + (wiki / "reports" / "run.md").write_text("# Report", encoding="utf-8") + + pages = _load_wiki_pages(wiki) + + assert not any(p.parts[-2] == "reports" for p in pages) + + def test_excludes_sources_directory(self, tmp_path): + wiki = _make_wiki(tmp_path) + (wiki / "sources" / "doc.md").write_text("# Source", encoding="utf-8") + + pages = _load_wiki_pages(wiki) + + assert not any(p.parts[-2] == "sources" for p in pages) + + def test_excludes_excluded_files(self, tmp_path): + wiki = _make_wiki(tmp_path) + for name in _EXCLUDED_FILES: + (wiki / name).write_text("# excluded", encoding="utf-8") + + pages = _load_wiki_pages(wiki) + + assert not any(p.name in _EXCLUDED_FILES for p in pages) + + def test_includes_summaries_concepts_entities(self, tmp_path): + wiki = _make_wiki(tmp_path) + (wiki / "entities").mkdir(parents=True, exist_ok=True) + (wiki / "summaries" / "paper.md").write_text("x", encoding="utf-8") + (wiki / "concepts" / "idea.md").write_text("x", encoding="utf-8") + (wiki / "entities" / "person.md").write_text("x", encoding="utf-8") + + pages = _load_wiki_pages(wiki) + + paths = {p.name for p in pages} + assert "paper.md" in paths + assert "idea.md" in paths + assert "person.md" in paths + + def test_shared_pages_yields_identical_results_find_invalid_frontmatter( + self, tmp_path + ): + """Calling ``find_invalid_frontmatter`` with a pre-loaded ``pages`` + dict must produce the same issues as calling it without one.""" + wiki = _make_wiki(tmp_path) + (wiki / "concepts" / "bad.md").write_text( + "---\nkey: value: broken\n---\n\n# Bad\n", encoding="utf-8" + ) + (wiki / "concepts" / "good.md").write_text( + '---\ntype: "Concept"\n---\n\n# Good\n', encoding="utf-8" + ) + + standalone = find_invalid_frontmatter(wiki) + shared = find_invalid_frontmatter(wiki, pages=_load_wiki_pages(wiki)) + + assert standalone == shared + + def test_shared_pages_yields_identical_results_find_missing_okf_fields( + self, tmp_path + ): + """Calling ``find_missing_okf_fields`` with a pre-loaded ``pages`` + dict must produce the same issues as calling it without one.""" + wiki = _make_wiki(tmp_path) + (wiki / "concepts" / "missing_type.md").write_text( + '---\ndescription: "ok"\n---\n\n# Bad\n', encoding="utf-8" + ) + (wiki / "summaries" / "complete.md").write_text( + '---\ntype: "Summary"\ndescription: "fine"\n---\n\n# Good\n', + encoding="utf-8", + ) + + standalone = find_missing_okf_fields(wiki) + shared = find_missing_okf_fields(wiki, pages=_load_wiki_pages(wiki)) + + assert standalone == shared diff --git a/tests/test_recompile.py b/tests/test_recompile.py index 29d06137..5ea6b7d3 100644 --- a/tests/test_recompile.py +++ b/tests/test_recompile.py @@ -18,12 +18,14 @@ from __future__ import annotations +import asyncio import json from pathlib import Path from unittest.mock import AsyncMock, patch from click.testing import CliRunner +from openkb.agent import compiler from openkb.cli import cli from openkb.schema import AGENTS_MD @@ -312,3 +314,34 @@ def test_recompile_refresh_schema_noop_when_agents_missing(kb_dir): assert result.exit_code == 0, result.output assert not agents.exists() # not materialized assert not (kb_dir / "wiki" / "AGENTS.md.bak").exists() + + +# --------------------------------------------------------------------------- +# compile_long_doc backfills type + description on recompile +# --------------------------------------------------------------------------- + + +def test_compile_long_doc_backfills_summary_frontmatter(tmp_path): + wiki = tmp_path / "wiki" + (wiki / "summaries").mkdir(parents=True) + (wiki / "concepts").mkdir(parents=True) + (tmp_path / ".openkb").mkdir() + (tmp_path / ".openkb" / "config.yaml").write_text( + "model: gpt-4o-mini\nlanguage: en\n", encoding="utf-8") + summary_path = wiki / "summaries" / "long.md" + summary_path.write_text( + "---\ndoc_type: pageindex\nfull_text: sources/long.json\n---\n\n# Long\n", + encoding="utf-8", + ) + with patch.object(compiler, "_llm_call", return_value="overview"), \ + patch.object(compiler, "_compile_concepts", new=AsyncMock()), \ + patch.object(compiler, "_close_async_llm_clients", new=AsyncMock()): + asyncio.run(compiler.compile_long_doc( + "long", summary_path, "doc-1", tmp_path, "gpt-4o-mini", + doc_description="A long report.", + )) + text = summary_path.read_text(encoding="utf-8") + assert 'type: "Summary"' in text + assert 'description: "A long report."' in text + # canonical order: type before description + assert text.index('type:') < text.index('description:') diff --git a/tests/test_schema.py b/tests/test_schema.py new file mode 100644 index 00000000..7b79fd8a --- /dev/null +++ b/tests/test_schema.py @@ -0,0 +1,10 @@ +"""Tests for openkb.schema constants (wiki AGENTS_MD schema doc).""" +from openkb.schema import AGENTS_MD + + +def test_agents_md_documents_type_and_description(): + # The new OKF-aligned frontmatter fields must be documented. + assert "type:" in AGENTS_MD + assert "description:" in AGENTS_MD + # The code-manages-frontmatter contract must remain intact. + assert "managed by code" in AGENTS_MD diff --git a/tests/test_tree_renderer.py b/tests/test_tree_renderer.py index f8e34c07..d560d6d0 100644 --- a/tests/test_tree_renderer.py +++ b/tests/test_tree_renderer.py @@ -15,7 +15,7 @@ def test_has_yaml_frontmatter(self, sample_tree): output = render_summary_md(sample_tree, "Sample Document", "doc-abc") assert output.startswith("---\n") assert "doc_type: pageindex" in output - assert "full_text: sources/Sample Document.json" in output + assert 'full_text: "sources/Sample Document.json"' in output def test_top_level_nodes_are_h1(self, sample_tree): output = render_summary_md(sample_tree, "Sample Document", "doc-abc") @@ -38,3 +38,24 @@ def test_summary_included_not_text(self, sample_tree): assert "Summary: Historical context." in output # Raw text should NOT appear in summary view assert "This document introduces the core concepts of the system." not in output + + +def test_summary_md_has_type_and_description(): + tree = {"structure": [{"title": "Intro", "start_index": 1, + "end_index": 2, "summary": "x", "nodes": []}]} + md = render_summary_md(tree, "my-doc", "doc-123", description="Quarterly report.") + assert 'type: "Summary"' in md + assert 'description: "Quarterly report."' in md + assert "doc_type: pageindex" in md + assert 'full_text: "sources/my-doc.json"' in md + + +def test_summary_full_text_quoted_yaml_safe(): + import yaml + tree = {"structure": []} + md = render_summary_md(tree, "weird: name", "doc-1", description="d") + # full_text is JSON-quoted, so a source name with a colon stays valid YAML + assert 'full_text: "sources/weird: name.json"' in md + fm = yaml.safe_load(md.split("---")[1]) + assert fm["full_text"] == "sources/weird: name.json" + assert fm["type"] == "Summary"