Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 29 additions & 8 deletions deeptutor/services/rag/pipelines/llamaindex/storage.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@
from . import ingestion, retrievers, vector_store
from .config import should_show_progress, vector_index_config_from_settings

_EMBEDDING_CHECKPOINT_SOURCE_LIMIT = 16
_EMBEDDING_CHECKPOINT_INTERVAL_SECONDS = 30.0


@dataclass(frozen=True)
class AddStoragePlan:
Expand Down Expand Up @@ -294,7 +297,7 @@ def _write_embedding_snapshot(
*,
source_records: dict[str, EmbeddingSourceRecord],
completed_keys: set[str],
document_group: list[Any],
checkpoint_documents: list[Any],
signature: EmbeddingSignature | None,
) -> None:
snapshot = Path(tempfile.mkdtemp(prefix=f".{candidate.name}.checkpoint-", dir=candidate.parent))
Expand All @@ -307,7 +310,7 @@ def _write_embedding_snapshot(
shutil.copy2(existing_catalog, snapshot / existing_catalog.name)
from deeptutor.services.rag.source_pages import persist_page_catalog

persist_page_catalog(document_group, snapshot)
persist_page_catalog(checkpoint_documents, snapshot)
atomic_write_json(
snapshot / ".embedding-checkpoint.json",
{
Expand Down Expand Up @@ -341,7 +344,7 @@ def create_resumable_index(
show_progress: bool | None = None,
checkpoint_callback: Callable[[Path, bool], None] | None = None,
) -> int:
"""Build a rebuild candidate one source at a time and snapshot each source."""
"""Embed sources individually and persist bounded batches as restart checkpoints."""
if show_progress is None:
show_progress = should_show_progress()
if (storage_dir / "meta.json").exists():
Expand All @@ -358,9 +361,11 @@ def create_resumable_index(
for key in sorted(completed_keys):
checkpoint_callback(source_records[key].path, True)

for key, document_group in grouped.items():
if key in completed_keys:
continue
remaining = [(key, group) for key, group in grouped.items() if key not in completed_keys]
checkpoint_keys: list[str] = []
checkpoint_documents: list[Any] = []
checkpointed_at = time.monotonic()
for position, (key, document_group) in enumerate(remaining, start=1):
nodes = ingestion.documents_to_nodes(document_group, show_progress=show_progress)
if index is None:
storage_context = vector_store.storage_context_for_nodes(
Expand All @@ -375,16 +380,32 @@ def create_resumable_index(
else:
index.insert_nodes(nodes)
completed_keys.add(key)
checkpoint_keys.append(key)
checkpoint_documents.extend(document_group)
# A new rebuild gets an early restart point; a resumed one already has it.
checkpoint_due = (
len(completed_keys) == 1
or position == len(remaining)
or len(checkpoint_keys) >= _EMBEDDING_CHECKPOINT_SOURCE_LIMIT
or time.monotonic() - checkpointed_at >= _EMBEDDING_CHECKPOINT_INTERVAL_SECONDS
)
if not checkpoint_due:
continue

_write_embedding_snapshot(
index,
storage_dir,
source_records=source_records,
completed_keys=completed_keys,
document_group=document_group,
checkpoint_documents=checkpoint_documents,
signature=signature,
)
if checkpoint_callback is not None:
checkpoint_callback(source_records[key].path, False)
for checkpoint_key in checkpoint_keys:
checkpoint_callback(source_records[checkpoint_key].path, False)
checkpoint_keys.clear()
checkpoint_documents.clear()
checkpointed_at = time.monotonic()
return len(documents)


Expand Down
78 changes: 78 additions & 0 deletions tests/services/rag/test_llamaindex_embedding_resume.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import hashlib
import json
from pathlib import Path
from types import SimpleNamespace

import pytest

Expand Down Expand Up @@ -33,6 +34,83 @@ def _records(kb_dir: Path, names: list[str]) -> dict[str, object]:
return records


def test_rebuild_checkpoints_batches_and_flushes_the_tail(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
from llama_index.core import Document, Settings
from llama_index.core.embeddings import MockEmbedding

from deeptutor.services.rag.pipelines.llamaindex import storage
from deeptutor.services.rag.source_pages import SOURCE_SCHEMA_VERSION

raw = tmp_path / "raw"
raw.mkdir()
paths = [raw / f"source-{number}.txt" for number in range(1, 8)]
documents = []
for path in paths:
path.write_text(f"Contents of {path.name}", encoding="utf-8")
documents.append(
Document(
text=path.read_text(encoding="utf-8"),
metadata={
"file_path": str(path),
"document_key": path.name,
"page_index": 0,
"source_schema_version": SOURCE_SCHEMA_VERSION,
},
)
)
candidate = tmp_path / "version-1"
saved_counts = []
reported = []
now = 0.0
original_snapshot = storage._write_embedding_snapshot
original_chunk = storage.ingestion.documents_to_nodes

def record_snapshot(*args, **kwargs):
original_snapshot(*args, **kwargs)
saved_counts.append(len(kwargs["completed_keys"]))

def chunk(source_documents, **kwargs):
nonlocal now
if source_documents[0].metadata["file_path"] == str(paths[4]):
now = 31.0
return original_chunk(source_documents, **kwargs)

def record_completion(path: Path, reused: bool) -> None:
assert not reused
checkpoint = json.loads(
(candidate / ".embedding-checkpoint.json").read_text(encoding="utf-8")
)
assert checkpoint["sources"][path.name]["completed"]
reported.append(path)

monkeypatch.setattr(Settings, "_embed_model", MockEmbedding(embed_dim=4))
monkeypatch.setattr(storage, "_EMBEDDING_CHECKPOINT_SOURCE_LIMIT", 3)
monkeypatch.setattr(storage, "time", SimpleNamespace(monotonic=lambda: now))
monkeypatch.setattr(storage, "_write_embedding_snapshot", record_snapshot)
monkeypatch.setattr(storage.ingestion, "documents_to_nodes", chunk)
monkeypatch.setattr(
storage.retrievers, "persist_bm25_retriever", lambda *_args, **_kwargs: False
)

count = storage.create_resumable_index(
documents,
candidate,
source_records=_records(tmp_path, [path.name for path in paths]),
signature=_signature(),
show_progress=False,
checkpoint_callback=record_completion,
)

assert count == len(documents)
assert saved_counts == [1, 4, 5, 7]
assert reported == paths
storage.verify_persisted_index(candidate)
catalog = json.loads((candidate / "source_pages.json").read_text(encoding="utf-8"))
assert [page["document_key"] for page in catalog["pages"]] == [path.name for path in paths]


@pytest.mark.asyncio
@pytest.mark.parametrize("interrupted_swap", [False, True])
async def test_rebuild_resumes_completed_documents_after_failure(
Expand Down