From 8818adaaee9c57b657b6df53430c479d043ff214 Mon Sep 17 00:00:00 2001 From: Ray Date: Fri, 10 Apr 2026 10:34:15 +0800 Subject: [PATCH 1/5] fix: update existing concept briefs in index.md instead of skipping --- openkb/agent/compiler.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/openkb/agent/compiler.py b/openkb/agent/compiler.py index b8f8f982..d5d80cf4 100644 --- a/openkb/agent/compiler.py +++ b/openkb/agent/compiler.py @@ -494,10 +494,18 @@ def _update_index( for name in concept_names: concept_link = f"[[concepts/{name}]]" - if concept_link not in text: - concept_entry = f"- {concept_link}" + concept_entry = f"- {concept_link}" + if name in concept_briefs: + concept_entry += f" — {concept_briefs[name]}" + if concept_link in text: if name in concept_briefs: - concept_entry += f" — {concept_briefs[name]}" + lines = text.split("\n") + for i, line in enumerate(lines): + if concept_link in line: + lines[i] = concept_entry + break + text = "\n".join(lines) + else: if "## Concepts" in text: text = text.replace("## Concepts\n", f"## Concepts\n{concept_entry}\n", 1) From aabcf5f3c5b2eec9859cde05a3a82dde12d5d969 Mon Sep 17 00:00:00 2001 From: Ray Date: Fri, 10 Apr 2026 10:35:16 +0800 Subject: [PATCH 2/5] fix: preserve non-ASCII characters in concept name slugs --- openkb/agent/compiler.py | 4 +++- tests/test_compiler.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/openkb/agent/compiler.py b/openkb/agent/compiler.py index d5d80cf4..e59b9c57 100644 --- a/openkb/agent/compiler.py +++ b/openkb/agent/compiler.py @@ -16,6 +16,7 @@ import sys import threading import time +import unicodedata from pathlib import Path import litellm @@ -302,11 +303,12 @@ def _write_summary(wiki_dir: Path, doc_name: str, summary: str, (summaries_dir / f"{doc_name}.md").write_text(frontmatter + summary, encoding="utf-8") -_SAFE_NAME_RE = re.compile(r'[^a-zA-Z0-9_\-]') +_SAFE_NAME_RE = re.compile(r'[^\w\-]') def _sanitize_concept_name(name: str) -> str: """Sanitize a concept name for safe use as a filename.""" + name = unicodedata.normalize("NFKC", name) sanitized = _SAFE_NAME_RE.sub("-", name).strip("-") return sanitized or "unnamed-concept" diff --git a/tests/test_compiler.py b/tests/test_compiler.py index a895b793..645b5dcb 100644 --- a/tests/test_compiler.py +++ b/tests/test_compiler.py @@ -12,6 +12,7 @@ compile_short_doc, _compile_concepts, _parse_json, + _sanitize_concept_name, _write_summary, _write_concept, _update_index, @@ -74,6 +75,39 @@ def test_plain_text_fallback(self): _parse_json("Just plain markdown text without JSON") +class TestSanitizeConceptName: + def test_ascii_passthrough(self): + assert _sanitize_concept_name("hello-world") == "hello-world" + + def test_spaces_replaced(self): + assert _sanitize_concept_name("hello world") == "hello-world" + + def test_chinese(self): + result = _sanitize_concept_name("注意力机制") + assert result == "注意力机制" + + def test_japanese(self): + result = _sanitize_concept_name("トランスフォーマー") + assert result == "トランスフォーマー" + + def test_french_accents(self): + result = _sanitize_concept_name("réseau neuronal") + assert "r" in result + assert result != "r-seau-neuronal" # accented chars preserved, not stripped + + def test_distinct_chinese_names_no_collision(self): + a = _sanitize_concept_name("注意力机制") + b = _sanitize_concept_name("变压器模型") + assert a != b + + def test_empty_fallback(self): + assert _sanitize_concept_name("!!!") == "unnamed-concept" + + def test_nfkc_normalization(self): + # U+FF21 (fullwidth A) should normalize to regular A + assert _sanitize_concept_name("\uff21\uff22") == "AB" + + class TestWriteSummary: def test_writes_with_frontmatter(self, tmp_path): wiki = tmp_path / "wiki" From 9df6e6c5df9aa5378d5568e12c2a68b2c92930a6 Mon Sep 17 00:00:00 2001 From: Ray Date: Fri, 10 Apr 2026 10:44:49 +0800 Subject: [PATCH 3/5] fix: always replace concept body on update, not only when source is new --- openkb/agent/compiler.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/openkb/agent/compiler.py b/openkb/agent/compiler.py index e59b9c57..3702af01 100644 --- a/openkb/agent/compiler.py +++ b/openkb/agent/compiler.py @@ -338,21 +338,21 @@ def _write_concept(wiki_dir: Path, name: str, content: str, source_file: str, is existing = fm + body else: existing = f"---\nsources: [{source_file}]\n---\n\n" + existing - # Strip frontmatter from LLM content to avoid duplicate blocks - clean = content - if clean.startswith("---"): - end = clean.find("---", 3) - if end != -1: - clean = clean[end + 3:].lstrip("\n") - # Replace body with LLM rewrite (prompt asks for full rewrite, not delta) - if existing.startswith("---"): - end = existing.find("---", 3) - if end != -1: - existing = existing[:end + 3] + "\n\n" + clean - else: - existing = clean + # Strip frontmatter from LLM content to avoid duplicate blocks + clean = content + if clean.startswith("---"): + end = clean.find("---", 3) + if end != -1: + clean = clean[end + 3:].lstrip("\n") + # Replace body with LLM rewrite (prompt asks for full rewrite, not delta) + if existing.startswith("---"): + end = existing.find("---", 3) + if end != -1: + existing = existing[:end + 3] + "\n\n" + clean else: existing = clean + else: + existing = clean if brief and existing.startswith("---"): end = existing.find("---", 3) if end != -1: From ef235d228b8f14aa6c07de34a3e0ac2b2e6e648e Mon Sep 17 00:00:00 2001 From: Ray Date: Fri, 10 Apr 2026 11:07:44 +0800 Subject: [PATCH 4/5] Fix concept index updates by section --- openkb/agent/compiler.py | 73 ++++++++++++++++++++++++++++++++-------- tests/test_compiler.py | 48 ++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 14 deletions(-) diff --git a/openkb/agent/compiler.py b/openkb/agent/compiler.py index 3702af01..f0bd0e05 100644 --- a/openkb/agent/compiler.py +++ b/openkb/agent/compiler.py @@ -284,6 +284,55 @@ def _read_concept_briefs(wiki_dir: Path) -> str: return "\n".join(lines) or "(none yet)" +def _get_section_bounds(lines: list[str], heading: str) -> tuple[int, int] | None: + """Return the [start, end) bounds for a Markdown H2 section.""" + for i, line in enumerate(lines): + if line == heading: + start = i + 1 + end = len(lines) + for j in range(start, len(lines)): + if lines[j].startswith("## "): + end = j + break + return start, end + return None + + +def _section_contains_link(lines: list[str], heading: str, link: str) -> bool: + """Check whether a wikilink already exists inside the named section.""" + bounds = _get_section_bounds(lines, heading) + if bounds is None: + return False + + start, end = bounds + return any(link in line for line in lines[start:end]) + + +def _replace_section_entry(lines: list[str], heading: str, link: str, entry: str) -> bool: + """Replace the first matching entry within a specific section.""" + bounds = _get_section_bounds(lines, heading) + if bounds is None: + return False + + start, end = bounds + for i in range(start, end): + if link in lines[i]: + lines[i] = entry + return True + return False + + +def _insert_section_entry(lines: list[str], heading: str, entry: str) -> bool: + """Insert a new entry at the top of a specific section.""" + bounds = _get_section_bounds(lines, heading) + if bounds is None: + return False + + start, _ = bounds + lines.insert(start, entry) + return True + + def _write_summary(wiki_dir: Path, doc_name: str, summary: str, doc_type: str = "short") -> None: @@ -469,8 +518,9 @@ def _update_index( """Append document and concept entries to index.md. When ``doc_brief`` or entries in ``concept_briefs`` are provided, entries - are written as ``- [[link]] (type) — brief text``. Existing entries are - detected by the link part only and skipped to avoid duplicates. + are written as ``- [[link]] (type) — brief text``. Existing entries are + detected within their own section by the link part only and skipped to + avoid duplicates. ``doc_type`` is ``"short"`` or ``"pageindex"`` — shown in the entry so the query agent knows how to access detailed content. """ @@ -485,32 +535,27 @@ def _update_index( ) text = index_path.read_text(encoding="utf-8") + lines = text.split("\n") doc_link = f"[[summaries/{doc_name}]]" - if doc_link not in text: + if not _section_contains_link(lines, "## Documents", doc_link): doc_entry = f"- {doc_link} ({doc_type})" if doc_brief: doc_entry += f" — {doc_brief}" - if "## Documents" in text: - text = text.replace("## Documents\n", f"## Documents\n{doc_entry}\n", 1) + _insert_section_entry(lines, "## Documents", doc_entry) for name in concept_names: concept_link = f"[[concepts/{name}]]" concept_entry = f"- {concept_link}" if name in concept_briefs: concept_entry += f" — {concept_briefs[name]}" - if concept_link in text: + if _section_contains_link(lines, "## Concepts", concept_link): if name in concept_briefs: - lines = text.split("\n") - for i, line in enumerate(lines): - if concept_link in line: - lines[i] = concept_entry - break - text = "\n".join(lines) + _replace_section_entry(lines, "## Concepts", concept_link, concept_entry) else: - if "## Concepts" in text: - text = text.replace("## Concepts\n", f"## Concepts\n{concept_entry}\n", 1) + _insert_section_entry(lines, "## Concepts", concept_entry) + text = "\n".join(lines) index_path.write_text(text, encoding="utf-8") diff --git a/tests/test_compiler.py b/tests/test_compiler.py index 645b5dcb..c473155b 100644 --- a/tests/test_compiler.py +++ b/tests/test_compiler.py @@ -221,6 +221,54 @@ def test_backwards_compat_no_briefs(self, tmp_path): assert "[[summaries/my-doc]]" in text assert "[[concepts/attention]]" in text + def test_updates_concept_brief_only_inside_concepts_section(self, tmp_path): + wiki = tmp_path / "wiki" + wiki.mkdir() + (wiki / "index.md").write_text( + "# Index\n\n" + "## Documents\n" + "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here\n\n" + "## Concepts\n" + "- [[concepts/attention]] — Old brief\n\n" + "## Explorations\n", + encoding="utf-8", + ) + + _update_index( + wiki, + "my-doc", + ["attention"], + concept_briefs={"attention": "New brief"}, + ) + + text = (wiki / "index.md").read_text() + assert "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here" in text + assert "- [[concepts/attention]] — New brief" in text + assert "- [[concepts/attention]] — Old brief" not in text + + def test_adds_concept_entry_when_link_exists_outside_concepts_section(self, tmp_path): + wiki = tmp_path / "wiki" + wiki.mkdir() + (wiki / "index.md").write_text( + "# Index\n\n" + "## Documents\n" + "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here\n\n" + "## Concepts\n\n" + "## Explorations\n", + encoding="utf-8", + ) + + _update_index( + wiki, + "my-doc", + ["attention"], + concept_briefs={"attention": "New brief"}, + ) + + text = (wiki / "index.md").read_text() + assert "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here" in text + assert "- [[concepts/attention]] — New brief" in text + class TestReadWikiContext: def test_empty_wiki(self, tmp_path): From ed0d6baeaadb4b53f5a1fb38dfb9a8cf7d3896d9 Mon Sep 17 00:00:00 2001 From: Ray Date: Fri, 10 Apr 2026 11:24:18 +0800 Subject: [PATCH 5/5] Fix exact concept index row matching --- openkb/agent/compiler.py | 17 ++++++++--------- tests/test_compiler.py | 20 ++++++++++++++++++++ 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/openkb/agent/compiler.py b/openkb/agent/compiler.py index f0bd0e05..d94a5582 100644 --- a/openkb/agent/compiler.py +++ b/openkb/agent/compiler.py @@ -299,13 +299,14 @@ def _get_section_bounds(lines: list[str], heading: str) -> tuple[int, int] | Non def _section_contains_link(lines: list[str], heading: str, link: str) -> bool: - """Check whether a wikilink already exists inside the named section.""" + """Check whether an index entry already exists inside the named section.""" bounds = _get_section_bounds(lines, heading) if bounds is None: return False start, end = bounds - return any(link in line for line in lines[start:end]) + entry_prefix = f"- {link}" + return any(line.startswith(entry_prefix) for line in lines[start:end]) def _replace_section_entry(lines: list[str], heading: str, link: str, entry: str) -> bool: @@ -315,8 +316,9 @@ def _replace_section_entry(lines: list[str], heading: str, link: str, entry: str return False start, end = bounds + entry_prefix = f"- {link}" for i in range(start, end): - if link in lines[i]: + if lines[i].startswith(entry_prefix): lines[i] = entry return True return False @@ -509,7 +511,6 @@ def _backlink_concepts(wiki_dir: Path, doc_name: str, concept_slugs: list[str]) text += f"\n\n## Related Documents\n- {link}\n" path.write_text(text, encoding="utf-8") - def _update_index( wiki_dir: Path, doc_name: str, concept_names: list[str], doc_brief: str = "", concept_briefs: dict[str, str] | None = None, @@ -519,7 +520,7 @@ def _update_index( When ``doc_brief`` or entries in ``concept_briefs`` are provided, entries are written as ``- [[link]] (type) — brief text``. Existing entries are - detected within their own section by the link part only and skipped to + detected within their own section by exact entry prefix and skipped to avoid duplicates. ``doc_type`` is ``"short"`` or ``"pageindex"`` — shown in the entry so the query agent knows how to access detailed content. @@ -534,8 +535,7 @@ def _update_index( encoding="utf-8", ) - text = index_path.read_text(encoding="utf-8") - lines = text.split("\n") + lines = index_path.read_text(encoding="utf-8").split("\n") doc_link = f"[[summaries/{doc_name}]]" if not _section_contains_link(lines, "## Documents", doc_link): @@ -555,8 +555,7 @@ def _update_index( else: _insert_section_entry(lines, "## Concepts", concept_entry) - text = "\n".join(lines) - index_path.write_text(text, encoding="utf-8") + index_path.write_text("\n".join(lines), encoding="utf-8") # --------------------------------------------------------------------------- diff --git a/tests/test_compiler.py b/tests/test_compiler.py index c473155b..2a2e82dc 100644 --- a/tests/test_compiler.py +++ b/tests/test_compiler.py @@ -198,6 +198,26 @@ def test_appends_entries_with_briefs(self, tmp_path): assert "[[concepts/attention]] — Focus mechanism" in text assert "[[concepts/transformer]] — NN architecture" in text + def test_updates_only_exact_concept_row(self, tmp_path): + wiki = tmp_path / "wiki" + wiki.mkdir() + (wiki / "index.md").write_text( + "# Index\n\n## Documents\n\n## Concepts\n" + "- [[concepts/transformer]] — Uses [[concepts/attention]] internally\n" + "- [[concepts/attention]] — Old brief\n\n## Explorations\n", + encoding="utf-8", + ) + _update_index( + wiki, + "my-doc", + ["attention"], + concept_briefs={"attention": "New brief"}, + ) + text = (wiki / "index.md").read_text() + assert "- [[concepts/transformer]] — Uses [[concepts/attention]] internally" in text + assert "- [[concepts/attention]] — New brief" in text + assert text.count("[[concepts/attention]] — New brief") == 1 + def test_no_duplicates(self, tmp_path): wiki = tmp_path / "wiki" wiki.mkdir()