diff --git a/deeptutor/services/rag/pipelines/llamaindex/storage.py b/deeptutor/services/rag/pipelines/llamaindex/storage.py index 691abafc53..51fdc408f3 100644 --- a/deeptutor/services/rag/pipelines/llamaindex/storage.py +++ b/deeptutor/services/rag/pipelines/llamaindex/storage.py @@ -28,6 +28,9 @@ from . import ingestion, retrievers, vector_store from .config import should_show_progress, vector_index_config_from_settings +_EMBEDDING_CHECKPOINT_SOURCE_LIMIT = 16 +_EMBEDDING_CHECKPOINT_INTERVAL_SECONDS = 30.0 + @dataclass(frozen=True) class AddStoragePlan: @@ -294,7 +297,7 @@ def _write_embedding_snapshot( *, source_records: dict[str, EmbeddingSourceRecord], completed_keys: set[str], - document_group: list[Any], + checkpoint_documents: list[Any], signature: EmbeddingSignature | None, ) -> None: snapshot = Path(tempfile.mkdtemp(prefix=f".{candidate.name}.checkpoint-", dir=candidate.parent)) @@ -307,7 +310,7 @@ def _write_embedding_snapshot( shutil.copy2(existing_catalog, snapshot / existing_catalog.name) from deeptutor.services.rag.source_pages import persist_page_catalog - persist_page_catalog(document_group, snapshot) + persist_page_catalog(checkpoint_documents, snapshot) atomic_write_json( snapshot / ".embedding-checkpoint.json", { @@ -341,7 +344,7 @@ def create_resumable_index( show_progress: bool | None = None, checkpoint_callback: Callable[[Path, bool], None] | None = None, ) -> int: - """Build a rebuild candidate one source at a time and snapshot each source.""" + """Embed sources individually and persist bounded batches as restart checkpoints.""" if show_progress is None: show_progress = should_show_progress() if (storage_dir / "meta.json").exists(): @@ -358,9 +361,11 @@ def create_resumable_index( for key in sorted(completed_keys): checkpoint_callback(source_records[key].path, True) - for key, document_group in grouped.items(): - if key in completed_keys: - continue + remaining = [(key, group) for key, group in grouped.items() if key not in completed_keys] + checkpoint_keys: list[str] = [] + checkpoint_documents: list[Any] = [] + checkpointed_at = time.monotonic() + for position, (key, document_group) in enumerate(remaining, start=1): nodes = ingestion.documents_to_nodes(document_group, show_progress=show_progress) if index is None: storage_context = vector_store.storage_context_for_nodes( @@ -375,16 +380,32 @@ def create_resumable_index( else: index.insert_nodes(nodes) completed_keys.add(key) + checkpoint_keys.append(key) + checkpoint_documents.extend(document_group) + # A new rebuild gets an early restart point; a resumed one already has it. + checkpoint_due = ( + len(completed_keys) == 1 + or position == len(remaining) + or len(checkpoint_keys) >= _EMBEDDING_CHECKPOINT_SOURCE_LIMIT + or time.monotonic() - checkpointed_at >= _EMBEDDING_CHECKPOINT_INTERVAL_SECONDS + ) + if not checkpoint_due: + continue + _write_embedding_snapshot( index, storage_dir, source_records=source_records, completed_keys=completed_keys, - document_group=document_group, + checkpoint_documents=checkpoint_documents, signature=signature, ) if checkpoint_callback is not None: - checkpoint_callback(source_records[key].path, False) + for checkpoint_key in checkpoint_keys: + checkpoint_callback(source_records[checkpoint_key].path, False) + checkpoint_keys.clear() + checkpoint_documents.clear() + checkpointed_at = time.monotonic() return len(documents) diff --git a/tests/services/rag/test_llamaindex_embedding_resume.py b/tests/services/rag/test_llamaindex_embedding_resume.py index 0bbb41e02a..4a83d615c1 100644 --- a/tests/services/rag/test_llamaindex_embedding_resume.py +++ b/tests/services/rag/test_llamaindex_embedding_resume.py @@ -3,6 +3,7 @@ import hashlib import json from pathlib import Path +from types import SimpleNamespace import pytest @@ -33,6 +34,83 @@ def _records(kb_dir: Path, names: list[str]) -> dict[str, object]: return records +def test_rebuild_checkpoints_batches_and_flushes_the_tail( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from llama_index.core import Document, Settings + from llama_index.core.embeddings import MockEmbedding + + from deeptutor.services.rag.pipelines.llamaindex import storage + from deeptutor.services.rag.source_pages import SOURCE_SCHEMA_VERSION + + raw = tmp_path / "raw" + raw.mkdir() + paths = [raw / f"source-{number}.txt" for number in range(1, 8)] + documents = [] + for path in paths: + path.write_text(f"Contents of {path.name}", encoding="utf-8") + documents.append( + Document( + text=path.read_text(encoding="utf-8"), + metadata={ + "file_path": str(path), + "document_key": path.name, + "page_index": 0, + "source_schema_version": SOURCE_SCHEMA_VERSION, + }, + ) + ) + candidate = tmp_path / "version-1" + saved_counts = [] + reported = [] + now = 0.0 + original_snapshot = storage._write_embedding_snapshot + original_chunk = storage.ingestion.documents_to_nodes + + def record_snapshot(*args, **kwargs): + original_snapshot(*args, **kwargs) + saved_counts.append(len(kwargs["completed_keys"])) + + def chunk(source_documents, **kwargs): + nonlocal now + if source_documents[0].metadata["file_path"] == str(paths[4]): + now = 31.0 + return original_chunk(source_documents, **kwargs) + + def record_completion(path: Path, reused: bool) -> None: + assert not reused + checkpoint = json.loads( + (candidate / ".embedding-checkpoint.json").read_text(encoding="utf-8") + ) + assert checkpoint["sources"][path.name]["completed"] + reported.append(path) + + monkeypatch.setattr(Settings, "_embed_model", MockEmbedding(embed_dim=4)) + monkeypatch.setattr(storage, "_EMBEDDING_CHECKPOINT_SOURCE_LIMIT", 3) + monkeypatch.setattr(storage, "time", SimpleNamespace(monotonic=lambda: now)) + monkeypatch.setattr(storage, "_write_embedding_snapshot", record_snapshot) + monkeypatch.setattr(storage.ingestion, "documents_to_nodes", chunk) + monkeypatch.setattr( + storage.retrievers, "persist_bm25_retriever", lambda *_args, **_kwargs: False + ) + + count = storage.create_resumable_index( + documents, + candidate, + source_records=_records(tmp_path, [path.name for path in paths]), + signature=_signature(), + show_progress=False, + checkpoint_callback=record_completion, + ) + + assert count == len(documents) + assert saved_counts == [1, 4, 5, 7] + assert reported == paths + storage.verify_persisted_index(candidate) + catalog = json.loads((candidate / "source_pages.json").read_text(encoding="utf-8")) + assert [page["document_key"] for page in catalog["pages"]] == [path.name for path in paths] + + @pytest.mark.asyncio @pytest.mark.parametrize("interrupted_swap", [False, True]) async def test_rebuild_resumes_completed_documents_after_failure(