fix(rag): skip hidden and junk directories when indexing (#5633)

* fix(rag): skip hidden and junk directories when indexing (#5559)

index_personal_documents walked the whole tree with no pruning, so
pointing RAG at a real-world folder silently swept in .obsidian/ plugin
JS, .git/ internals, node_modules/, and __pycache__/ — multiplying
indexing time and polluting retrieval with junk chunks.

Prune hidden directories and well-known junk directories from the walk,
and skip hidden files. The explicitly passed root is exempt, so a user
who deliberately indexes a hidden directory still gets its contents.

* fix(rag): prune hidden/junk dirs in the keyword index too, via a shared helper

The #5559 fix pruned only VectorRAG.index_personal_documents (the vector index).
The parallel keyword index built by PersonalDocsManager.refresh_index ->
load_personal_index walked the same tree unpruned, so .obsidian/, .git/,
node_modules/ etc. still swept into keyword retrieval and the file listing —
the 'end-to-end' guarantee was only half true.

Single-source the pruning policy in src/index_walk (prune_index_dirs +
is_indexable_file) and use it from both walkers so they cannot drift again.
The junk-dir match is now case-insensitive, so a Node_Modules on a
case-insensitive filesystem is pruned too.

Tests: keyword-path regressions covering hidden/junk dirs, hidden files, junk
at depth (not just top level), case-insensitive junk, and the explicit-hidden-
root exemption. The existing vector tests still pass against the shared helper.

src/index_walk.py

@@ -0,0 +1,35 @@
+"""Shared directory-walk pruning for personal-document indexing (#5559).
+
+Single source of the hidden-dir / junk-dir / hidden-file skip so the vector
+index (``rag_vector.index_personal_documents``) and the keyword index
+(``personal_docs.load_personal_index``) apply the exact same policy and cannot
+drift — the drift is what left the keyword path sweeping in `.obsidian/`,
+`.git/`, and `node_modules/` after the vector path was fixed.
+"""
+from typing import List, Set
+
+# Well-known non-hidden junk directories to skip. Matched case-insensitively so
+# a `Node_Modules` on a case-insensitive filesystem (macOS default) is still
+# pruned. Hidden directories (dot-prefixed) are pruned separately. Kept
+# deliberately small: over-pruning would silently drop a user's real content
+# (e.g. a notes directory legitimately named "build").
+EXCLUDED_DIR_NAMES: Set[str] = {'node_modules', '__pycache__', 'venv'}
+
+
+def prune_index_dirs(dirs: List[str]) -> None:
+    """In-place ``os.walk`` (topdown) directory prune: drop hidden and known
+    junk directories so the walk never descends into them.
+
+    The explicitly-targeted walk root is never a member of ``dirs`` (it is the
+    ``dirpath`` argument), so it stays exempt — a user who deliberately points
+    indexing at a hidden directory gets its contents, minus nested junk.
+    """
+    dirs[:] = [
+        d for d in dirs
+        if not d.startswith('.') and d.lower() not in EXCLUDED_DIR_NAMES
+    ]
+
+
+def is_indexable_file(name: str) -> bool:
+    """A file is indexable only if it is not hidden (dot-prefixed)."""
+    return not name.startswith('.')

src/personal_docs.py

@@ -6,6 +6,8 @@ import logging
 from typing import List, Dict, Set, Any, Tuple
 from dataclasses import dataclass
 
+from src.index_walk import prune_index_dirs, is_indexable_file
+
 from src.markitdown_runtime import MARKITDOWN_EXTS
 
 logger = logging.getLogger(__name__)
@@ -94,13 +96,22 @@ def tokenize(s: str) -> Set[str]:
     return set(t for t in tokens if t not in config.STOP_WORDS and len(t) > 1)
 
 def load_personal_index(
-    personal_dir: str, 
+    personal_dir: str,
     extensions: Tuple[str, ...] = config.DEFAULT_EXTENSIONS
 ) -> List[Dict[str, Any]]:
-    """Load and index personal documents."""
+    """Load and index personal documents.
+
+    Skips hidden and junk directories and hidden files via the shared
+    ``index_walk`` policy, so the keyword index matches the vector index and a
+    real vault/repo does not sweep in ``.obsidian/`` / ``.git/`` /
+    ``node_modules/`` content (#5559).
+    """
     files = []
-    for root, _, names in os.walk(personal_dir):
+    for root, dirs, names in os.walk(personal_dir):
+        prune_index_dirs(dirs)
         for name in sorted(names):
+            if not is_indexable_file(name):
+                continue
             p = os.path.join(root, name)
             if not os.path.isfile(p):
                 continue

src/rag_vector.py

@@ -14,6 +14,7 @@ import numpy as np
 from typing import List, Dict, Any, Optional, Set
 
 from src.constants import CHROMA_DIR
+from src.index_walk import prune_index_dirs, is_indexable_file
 from pathlib import Path
 
 from src.embedding_lanes import (
@@ -34,6 +35,10 @@ DEFAULT_FILE_EXTENSIONS: Set[str] = {
     '.csv', '.html', '.css', '.js', '.pdf'
 }
 
+# Tool-internal directories that match DEFAULT_FILE_EXTENSIONS but are never
+# Directory-walk pruning is single-sourced in src.index_walk so the vector and
+# keyword indexers apply the same hidden/junk policy and cannot drift (#5559).
+
 VECTOR_WEIGHT = 0.7
 KEYWORD_WEIGHT = 0.3
 
@@ -497,8 +502,15 @@ class VectorRAG:
         failed = 0
 
         try:
-            for root, _, files in os.walk(directory):
+            for root, dirs, files in os.walk(directory):
+                # Prune in place so os.walk never descends into hidden or junk
+                # directories (#5559), via the shared index_walk policy. The
+                # passed-in root is exempt: a user who deliberately targets a
+                # hidden directory gets it.
+                prune_index_dirs(dirs)
                 for fname in files:
+                    if not is_indexable_file(fname):
+                        continue
                     fpath = os.path.join(root, fname)
                     ext = Path(fname).suffix.lower()
                     if ext not in file_extensions:

tests/test_personal_index_hidden_dirs.py

@@ -0,0 +1,63 @@
+"""Regression guard for #5559 — the KEYWORD index (load_personal_index, which
+PersonalDocsManager.refresh_index builds from) must skip hidden dirs, hidden
+files, and junk dirs at ANY depth, the same as the vector index. Both walkers
+share one pruning helper (src/index_walk) so they cannot drift again.
+"""
+import os
+
+os.environ.setdefault("DATABASE_URL", "sqlite:///:memory:")
+
+from src.personal_docs import load_personal_index
+
+
+def _write(path, content="real content"):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(content, encoding="utf-8")
+
+
+def _indexed(root):
+    return {rec["name"] for rec in load_personal_index(str(root))}
+
+
+def test_keyword_index_skips_hidden_and_junk_dirs(tmp_path):
+    _write(tmp_path / "note.md")
+    _write(tmp_path / "sub" / "deeper.md")
+    _write(tmp_path / ".obsidian" / "workspace.json")
+    _write(tmp_path / ".git" / "hooks.md")
+    _write(tmp_path / "node_modules" / "lib" / "readme.md")
+    _write(tmp_path / "__pycache__" / "cached.txt")
+    _write(tmp_path / "venv" / "lib" / "site.txt")
+    assert _indexed(tmp_path) == {"note.md", os.path.join("sub", "deeper.md")}
+
+
+def test_keyword_index_skips_hidden_files(tmp_path):
+    _write(tmp_path / "visible.md")
+    _write(tmp_path / ".hidden.md")
+    _write(tmp_path / "sub" / ".secret.txt")
+    assert _indexed(tmp_path) == {"visible.md"}
+
+
+def test_keyword_index_prunes_junk_at_depth(tmp_path):
+    """Pruning must apply at every level, not just the first (the vector test's
+    fixtures only nested one level under the root)."""
+    _write(tmp_path / "a" / "b" / "keep.md")
+    _write(tmp_path / "a" / "b" / "node_modules" / "dep.md")
+    _write(tmp_path / "a" / ".obsidian" / "deep.json")
+    assert _indexed(tmp_path) == {os.path.join("a", "b", "keep.md")}
+
+
+def test_keyword_index_junk_match_is_case_insensitive(tmp_path):
+    """A case-variant junk dir must still be pruned (macOS default FS is
+    case-insensitive, so `Node_Modules` and `node_modules` are the same dir)."""
+    _write(tmp_path / "keep.md")
+    _write(tmp_path / "Node_Modules" / "dep.md")
+    assert _indexed(tmp_path) == {"keep.md"}
+
+
+def test_keyword_index_explicit_hidden_root_still_indexed(tmp_path):
+    """Children-only pruning: pointing indexing at a hidden dir gets its
+    contents, minus nested hidden/junk."""
+    root = tmp_path / ".notes"
+    _write(root / "idea.md")
+    _write(root / ".obsidian" / "plugin.json")
+    assert {rec["name"] for rec in load_personal_index(str(root))} == {"idea.md"}

tests/test_rag_index_hidden_dirs.py

@@ -0,0 +1,78 @@
+"""Regression guard for #5559 — directory indexing must skip hidden directories,
+hidden files, and well-known junk directories.
+
+VectorRAG.index_personal_documents walked the whole tree with no pruning, so
+pointing RAG at a real-world folder (an Obsidian vault, a git repo) swept in
+`.obsidian/` plugin JavaScript, `.git/` internals, `node_modules/`, etc. The
+junk multiplied indexing time and polluted retrieval.
+
+These tests are hermetic — no chromadb; VectorRAG is created via __new__ (skip
+Chroma connect) with add_document stubbed to record which files get indexed.
+"""
+import os
+
+os.environ.setdefault("DATABASE_URL", "sqlite:///:memory:")
+
+import src.rag_vector as rag_vector
+
+
+def _make_rag(recorded_sources):
+    rag = rag_vector.VectorRAG.__new__(rag_vector.VectorRAG)  # skip Chroma connect
+
+    def _record(text, metadata):
+        recorded_sources.add(metadata["source"])
+        return True
+
+    rag.add_document = _record
+    return rag
+
+
+def _write(path, content="some real content"):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(content, encoding="utf-8")
+
+
+def test_index_skips_hidden_and_junk_directories(tmp_path):
+    _write(tmp_path / "note.md")
+    _write(tmp_path / "sub" / "deeper.md")
+    _write(tmp_path / ".obsidian" / "plugins" / "plugin.js")
+    _write(tmp_path / ".git" / "hooks.js")
+    _write(tmp_path / "node_modules" / "lib" / "index.js")
+    _write(tmp_path / "__pycache__" / "cached.py")
+    _write(tmp_path / "venv" / "lib" / "site.py")
+
+    recorded = set()
+    rag = _make_rag(recorded)
+    result = rag.index_personal_documents(str(tmp_path))
+
+    assert result["success"] is True
+    indexed = {os.path.relpath(p, str(tmp_path)) for p in recorded}
+    assert indexed == {"note.md", os.path.join("sub", "deeper.md")}
+
+
+def test_index_skips_hidden_files(tmp_path):
+    _write(tmp_path / "visible.md")
+    _write(tmp_path / ".hidden.md")
+    _write(tmp_path / "sub" / ".secret.txt")
+
+    recorded = set()
+    rag = _make_rag(recorded)
+    rag.index_personal_documents(str(tmp_path))
+
+    indexed = {os.path.relpath(p, str(tmp_path)) for p in recorded}
+    assert indexed == {"visible.md"}
+
+
+def test_explicitly_passed_hidden_root_is_still_indexed(tmp_path):
+    """Pruning applies to children only — a user who deliberately points RAG at
+    a hidden directory gets its contents, minus nested hidden/junk dirs."""
+    root = tmp_path / ".notes"
+    _write(root / "idea.md")
+    _write(root / ".obsidian" / "plugin.js")
+
+    recorded = set()
+    rag = _make_rag(recorded)
+    rag.index_personal_documents(str(root))
+
+    indexed = {os.path.relpath(p, str(root)) for p in recorded}
+    assert indexed == {"idea.md"}