fix(personal): serialize add/remove/reload on an async job lock

The #5558 fix took the job lock INSIDE the threadpool worker and only on the
add path, so (1) remove_directory and /reload mutated PersonalDocsManager's
unsynchronized list/index concurrently with an in-flight add — the inconsistent
state the PR claimed to prevent — and (2) a queued add blocked on the lock while
holding an AnyIO threadpool token, starving the shared pool.

Move the lock to an asyncio.Lock acquired in the async handler BEFORE offloading,
and route add, remove and reload through it. A waiting request now parks on the
event loop instead of pinning a worker, and all three mutators are serialized so
the 'add/remove are serialized and cannot leave inconsistent state' guarantee
holds. remove and reload also run their blocking work off the event loop. The
lock is per-router so each app binds it to its own loop; single-process scope.

Tests: add-vs-remove and add-vs-reload serialization regressions (async via
ASGITransport, since asyncio.Lock deadlocks starlette TestClient's portal); the
existing add-vs-add test converted to the same driver.

tests/test_add_directory_event_loop.py

@@ -11,14 +11,43 @@ triggers ``refresh_index()``, which re-extracts text across tracked dirs.
 These tests build the real router with fake managers and compare the thread
 the indexing work runs on against the event loop's thread.
 """
+import asyncio
 import os
 import threading
 
 os.environ.setdefault("DATABASE_URL", "sqlite:///:memory:")
 
+import httpx
 from fastapi import FastAPI
 from fastapi.testclient import TestClient
 
+
+def _serialization_probe():
+    """Shared counter proving two critical sections never overlap."""
+    state = {"active": 0, "max_active": 0}
+    lock = threading.Lock()
+
+    def enter():
+        with lock:
+            state["active"] += 1
+            state["max_active"] = max(state["max_active"], state["active"])
+
+    def leave():
+        with lock:
+            state["active"] -= 1
+
+    return state, enter, leave
+
+
+# Concurrency tests are `async def` (pyproject asyncio_mode="auto") and drive the
+# ASGI app through httpx.ASGITransport + AsyncClient + asyncio.gather, NOT starlette
+# TestClient + ThreadPoolExecutor: the job lock is an asyncio.Lock acquired in the
+# async handler, and TestClient's portal-thread dispatch deadlocks against it (same
+# reason test_notes_fail_closed_auth.py uses ASGITransport). asyncio.gather runs both
+# requests on the test's own loop.
+def _async_client(app):
+    return httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://t")
+
 import routes.personal_routes as personal_routes
 from core.middleware import require_admin
 from src.auth_helpers import require_user
@@ -104,43 +133,30 @@ def test_response_and_bookkeeping_unchanged(tmp_path, monkeypatch):
     assert record["bookkeeping_index_flag"] is False
 
 
-def test_concurrent_add_directory_requests_serialize_indexing(tmp_path, monkeypatch):
+async def test_concurrent_add_directory_requests_serialize_indexing(tmp_path, monkeypatch):
     """Off-loop execution must not mean parallel index jobs: concurrent
     requests would race PersonalDocsManager's unsynchronized list mutations
     and file writes (save_directories/_save_excluded are plain open('w'))."""
     import time
-    from concurrent.futures import ThreadPoolExecutor
 
-    record = {}
-    state = {"active": 0, "max_active": 0}
-    state_lock = threading.Lock()
+    state, enter, leave = _serialization_probe()
 
     def _slow_index(self, directory, owner=None):
-        with state_lock:
-            state["active"] += 1
-            state["max_active"] = max(state["max_active"], state["active"])
-        time.sleep(0.2)
-        with state_lock:
-            state["active"] -= 1
+        enter(); time.sleep(0.2); leave()
         return {"success": True, "indexed_count": 1, "failed_count": 0}
 
     monkeypatch.setattr(_FakeRag, "index_personal_documents", _slow_index)
 
+    record = {}
     app = _build_app(tmp_path, monkeypatch, record)
     for name in ("docs_a", "docs_b"):
         (tmp_path / name).mkdir()
 
-    client = TestClient(app)
-    with ThreadPoolExecutor(max_workers=2) as pool:
-        futures = [
-            pool.submit(
-                client.post,
-                "/api/personal/add_directory",
-                json={"directory": str(tmp_path / name)},
-            )
-            for name in ("docs_a", "docs_b")
-        ]
-        results = [f.result() for f in futures]
+    async with _async_client(app) as ac:
+        results = await asyncio.gather(
+            ac.post("/api/personal/add_directory", json={"directory": str(tmp_path / "docs_a")}),
+            ac.post("/api/personal/add_directory", json={"directory": str(tmp_path / "docs_b")}),
+        )
 
     assert all(r.status_code == 200 for r in results)
     assert state["max_active"] == 1, (
@@ -164,3 +180,74 @@ def test_failed_indexing_still_returns_500(tmp_path, monkeypatch):
     resp = client.post("/api/personal/add_directory", json={"directory": str(target)})
     assert resp.status_code == 500
     assert "boom" in resp.json()["detail"]
+
+
+async def test_add_and_remove_serialize(tmp_path, monkeypatch):
+    """#5634: remove must hold the SAME job lock as add. Otherwise a remove
+    running while an add job is in flight races PersonalDocsManager's
+    unsynchronized list/index mutations — the inconsistent state the PR's
+    'add/remove are serialized' guarantee claims to prevent."""
+    import time
+
+    state, enter, leave = _serialization_probe()
+
+    def _slow_index(self, directory, owner=None):
+        enter(); time.sleep(0.25); leave()
+        return {"success": True, "indexed_count": 1, "failed_count": 0}
+
+    def _slow_remove(self, directory):
+        enter(); time.sleep(0.25); leave()
+
+    monkeypatch.setattr(_FakeRag, "index_personal_documents", _slow_index)
+    monkeypatch.setattr(_FakeDocsManager, "remove_directory", _slow_remove, raising=False)
+
+    record = {}
+    app = _build_app(tmp_path, monkeypatch, record)
+    (tmp_path / "docs_a").mkdir()
+    (tmp_path / "docs_b").mkdir()
+
+    async with _async_client(app) as ac:
+        results = await asyncio.gather(
+            ac.post("/api/personal/add_directory", json={"directory": str(tmp_path / "docs_a")}),
+            ac.delete("/api/personal/remove_directory", params={"directory": str(tmp_path / "docs_b")}),
+        )
+
+    assert all(r.status_code == 200 for r in results)
+    assert state["max_active"] == 1, (
+        f"{state['max_active']} add/remove critical sections overlapped — "
+        "remove must hold the same index job lock as add"
+    )
+
+
+async def test_reload_serializes_with_add(tmp_path, monkeypatch):
+    """#5634: POST /reload rebuilds the index via refresh_index(); it must hold
+    the same job lock so it cannot race an in-flight add job."""
+    import time
+
+    state, enter, leave = _serialization_probe()
+
+    def _slow_index(self, directory, owner=None):
+        enter(); time.sleep(0.25); leave()
+        return {"success": True, "indexed_count": 1, "failed_count": 0}
+
+    def _slow_refresh(self):
+        enter(); time.sleep(0.25); leave()
+
+    monkeypatch.setattr(_FakeRag, "index_personal_documents", _slow_index)
+    monkeypatch.setattr(_FakeDocsManager, "refresh_index", _slow_refresh, raising=False)
+
+    record = {}
+    app = _build_app(tmp_path, monkeypatch, record)
+    (tmp_path / "docs_a").mkdir()
+
+    async with _async_client(app) as ac:
+        results = await asyncio.gather(
+            ac.post("/api/personal/add_directory", json={"directory": str(tmp_path / "docs_a")}),
+            ac.post("/api/personal/reload"),
+        )
+
+    assert all(r.status_code == 200 for r in results)
+    assert state["max_active"] == 1, (
+        f"{state['max_active']} add/reload critical sections overlapped — "
+        "reload must hold the same index job lock as add"
+    )