"""ProjectIndexer — write-side bootstrap orchestrator (spec §5.1, §5.3, post-#5a-2)."""
from __future__ import annotations
import asyncio
import logging
from collections.abc import Callable
from dataclasses import dataclass, field
from pathlib import Path
from pydocs_mcp.application.branch_manifest import (
BranchManifest,
BranchManifestBuilder,
NoBranchManifestBuilder,
)
from pydocs_mcp.application.indexing_service import IndexingService, IndexingStats
from pydocs_mcp.application.protocols import (
ChunkExtractor,
DependencyResolver,
MemberExtractor,
)
from pydocs_mcp.models import Package
from pydocs_mcp.storage.branch_records import BranchRecord
from pydocs_mcp.storage.protocols import UnitOfWork
log = logging.getLogger("pydocs-mcp")
def _stamped_by_this_checkout(stamped: BranchRecord | None, manifest: BranchManifest) -> bool:
"""The manifest's branch at its head, stamped by this checkout's own pass.
A row of the same name and head written from git objects (``index
--branch``, or a pass the ref watcher queued, #317) is not: it is not the
served row. A checkout onto such a branch that rewrites no in-scope file
keeps the package hash, and a skip there would leave the previous branch
served while ``HEAD`` names this one.
"""
return (
stamped is not None
and stamped.head_sha == manifest.head_sha
and stamped.worktree_path == manifest.worktree_path
)
[docs]
@dataclass(frozen=True, slots=True)
class ProjectIndexer:
"""Coordinates project + dependency indexing, returning fresh stats.
Post-#5a-2: takes its own ``uow_factory`` for the hash-cache check
(was a reach-through to ``indexing_service.package_store.get`` —
eng plan-review #4). Composition root wires the same factory to
both ``IndexingService`` and ``ProjectIndexer``.
"""
indexing_service: IndexingService
dependency_resolver: DependencyResolver
chunk_extractor: ChunkExtractor
member_extractor: MemberExtractor
uow_factory: Callable[[], UnitOfWork]
# The working-tree manifest builder (spec §6.3 step 1). The Null Object
# default keeps existing callers and tests branch-free; the composition
# root wires WorkingTreeManifestBuilder.
manifest_builder: BranchManifestBuilder = field(default_factory=NoBranchManifestBuilder)
async def index_project(
self,
project_dir: Path,
*,
force: bool = False,
include_project_source: bool = True,
include_dependencies: bool = True,
workers: int = 1,
) -> IndexingStats:
stats = IndexingStats()
if force:
await self.indexing_service.clear_all()
manifest = None
if include_project_source:
manifest = await self._index_project_source(project_dir, stats)
# WHY: per-task repository corpora (the benchmark's RepoQA path) carry
# their answer in the repo source itself, so resolving + indexing the
# repo's declared dependencies is pure noise AND the dominant ingestion
# cost. ``include_dependencies=False`` skips the whole block for those.
# Default ``True`` preserves production indexing and reference-project
# corpora (DS-1000), whose declared libraries ARE the search target.
if include_dependencies:
deps = await self.dependency_resolver.resolve(project_dir)
if workers <= 1:
for dep_name in deps:
await self._index_one_dependency(dep_name, stats)
else:
sem = asyncio.Semaphore(workers)
async def _bounded(dep_name: str) -> None:
async with sem:
await self._index_one_dependency(dep_name, stats)
await asyncio.gather(*[_bounded(d) for d in deps])
# Single post-index pass: recompute global node scores (PageRank /
# community / in-degree) over the now fully-resolved cross-package
# reference graph. No-op unless enabled on the IndexingService. Scored
# on the working tree's branch; without a manifest (a --skip-project
# pass, or no builder wired) on the branch the bundle serves.
await self.indexing_service.recompute_node_scores(
branch=manifest.name if manifest is not None else None
)
return stats
async def _index_project_source(
self,
project_dir: Path,
stats: IndexingStats,
) -> BranchManifest | None:
"""Index the project source; return the working-tree manifest (``None``
without a builder), cached pass or not — the node-score pass keys by it."""
result = await self.chunk_extractor.extract_from_project(project_dir)
pkg = result.package
# Built BEFORE the cache check because the branch stamp is part of the
# cache key (see ``_project_is_cached``). A clean tree costs no file
# reads: blob ids come from git's index and only dirty files are hashed.
# Project source only: ``discovered_paths`` is populated for dependency
# extractions too, and hashing every site-packages file (under absolute
# paths) is neither cheap nor meaningful for the branch dimension.
manifest = await self.manifest_builder.build(project_dir, result.discovered_paths)
if await self._project_is_cached(pkg, manifest):
log.info("Project: no changes (cached)")
return manifest
members = await self.member_extractor.extract_from_project(project_dir)
await self.indexing_service.reindex_package(
pkg,
result.chunks,
members,
trees=result.trees,
references=result.references,
reference_aliases=result.reference_aliases,
class_attribute_types=result.class_attribute_types,
# The project root scores staleness; the §D12 structured overlay
# rides alongside so its grounded fields land on the record.
decisions=result.decisions,
decision_structured=result.decision_structured,
project_root=project_dir,
branch_manifest=manifest,
)
stats.project_indexed = True
log.info(
"Project: %d chunks, %d symbols, %d trees",
len(result.chunks),
len(members),
len(result.trees),
)
return manifest
async def _project_is_cached(self, pkg: Package, manifest: BranchManifest | None) -> bool:
"""Cache hit only when the package hash AND the stamped branch agree.
WHY the branch half: ``packages.content_hash`` is xxh3 over
``(path, mtime)`` pairs, so a checkout (or a commit) that rewrites no
in-discovery-scope file moves no mtime and hashes identically. Skipping
on that alone would leave ``branches`` on the previous name and head
while ``stamp_metadata`` advances ``index_metadata.git_head`` at the end
of every pass — ``meta.branch`` and ``meta.indexed_git_head`` would then
describe different commits with ``index_stale`` reading False.
Re-stamping ``branches`` on the cached path is NOT the fix: the stamp
retires the previous branch by ``worktree_path``, dropping its
membership, after which the project GC would collect every project
chunk. A disagreement is a cache MISS — the full pass rebuilds both.
"""
async with self.uow_factory() as uow:
existing = await uow.packages.get(pkg.name)
stamped = None if manifest is None else await uow.branches.get_branch(manifest.name)
if existing is None or existing.content_hash != pkg.content_hash:
return False
# No manifest = the Null builder (no branch dimension wired): the
# package hash alone decides, exactly as before the branch tables.
return manifest is None or _stamped_by_this_checkout(stamped, manifest)
async def _index_one_dependency(
self,
dep_name: str,
stats: IndexingStats,
) -> None:
try:
result = await self.chunk_extractor.extract_from_dependency(dep_name)
pkg = result.package
async with self.uow_factory() as uow:
existing = await uow.packages.get(pkg.name)
if existing is not None and existing.content_hash == pkg.content_hash:
stats.cached += 1
return
members = await self.member_extractor.extract_from_dependency(dep_name)
await self.indexing_service.reindex_package(
pkg,
result.chunks,
members,
trees=result.trees,
references=result.references,
reference_aliases=result.reference_aliases,
class_attribute_types=result.class_attribute_types,
# Empty unless ``decision_capture.include_deps`` mined this
# dependency (issue #346); passed either way, so switching it
# off deletes the rows on the re-extraction the hash forces. No
# ``project_root``: staleness stays 0.0 because site-packages
# mtimes record install time, which would call every pinned
# dependency stale; the records are re-mined when its version
# changes instead.
decisions=result.decisions,
decision_structured=result.decision_structured,
)
stats.indexed += 1
log.info(
" ok %s %s (%d chunks, %d syms, %d trees)",
pkg.name,
pkg.version,
len(result.chunks),
len(members),
len(result.trees),
)
except Exception as e:
log.warning(" fail %s: %s", dep_name, e)
stats.failed += 1