mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-08-07 05:58:37 -04:00
Ashvin
c8a012d4d2
* fix(memory): don't let an unreadable store get overwritten with an empty one
load_all() answered a failed read the same way it answered an empty store:
with []. Every mutation path is a read-modify-write (load the whole file,
change it, save it back), so a failed read became
load_all() -> [] -> [].append(new) -> save([new])
and save() is atomic, so the replacement stuck.
The case that actually destroys data is a store that is READABLE but not
parseable - a truncated file, or one holding {} instead of []. Nothing
obstructs the write, so adding a memory returns HTTP 200 and every memory
already stored is gone. Verified end-to-end against a running instance: on the
current code a truncated memory.json plus one add leaves the file holding only
the new entry. Truncation is reachable - core/database.py rewrites memory.json
during migration with a plain open(.., "w") + json.dump, which is not atomic.
A live exclusive lock is not the dangerous case: it blocks the read and the
os.replace alike, so the save fails too and the store survives. That path
currently 500s and loses nothing.
_read_entries() now returns [] only when the file genuinely does not exist and
raises MemoryStoreUnreadable for every other failure, including a store that
parses but is not a JSON array. load_all() keeps the old lenient behaviour so
display, search and context injection still degrade quietly instead of
breaking chat. The read-modify-write callers switch to load_all_for_update(),
which propagates the error: the memory routes turn it into a 503 and change
nothing, backup import refuses rather than saving only the incoming rows, and
auto-extraction and the audit merge skip the write. The audit merge mattered
most - it rebuilds the whole file from one owner's slice plus everyone else's
rows, so an empty read there dropped every other tenant's memories.
The corrupt-JSON path still gets its one shot at the legacy memory.txt
migration before raising, so that recovery is unchanged.
The two updated fakes gained load_all_for_update because the real class has it;
MagicMock would otherwise hand the import path a Mock instead of the seeded list.
Fixes #5673
* fix(memory): fail closed on the remaining read-modify-write add paths
The strict loader landed with the routes, the backup import and the extractor
converted, but three read-modify-write sinks still called load_all(), which
degrades an unreadable store to []. Two of them are the paths users actually
reach, so the data loss in #5673 stayed reproducible:
- src/ai_interaction.py do_manage_memory, action "add" — reached from ordinary
chat via src/tool_execution.py:793 -> dispatch_ai_tool. "Remember that I
prefer X" against an unreadable store wrote a one-entry file over it and
reported success.
- mcp_servers/memory_server.py, action "add" — the same shape through
_scope_entries(), registered as a built-in in src/builtin_mcp.py.
- src/memory_provider.py NativeMemoryProvider.remember and .delete — wired
into app state in src/app_initializer.py but not consumed outside tests yet,
converted here so the pattern is uniform before it goes live.
The MCP server takes _scope_entries(for_update=True) so list keeps the lenient
read. The edit and delete branches on both tool paths were already fail-closed
by accident — an empty view matches nothing and returns before the save — so
they are left alone.
The three new tests drive the real entry points rather than replaying the
shape, and use a truncated store, which is the case that reads back fine so
nothing stops the save. Each asserts memory.json is byte-identical afterwards;
all three fail on the previous commit with the store overwritten.
222 lines
9.1 KiB
Python
222 lines
9.1 KiB
Python
"""Backup routes — export/import user data (memories, presets, settings, skills, preferences)."""
|
|
|
|
import json
|
|
import logging
|
|
from datetime import datetime
|
|
|
|
from fastapi import APIRouter, HTTPException, Request, Response
|
|
from core.middleware import require_admin
|
|
from services.memory import MemoryStoreUnreadable
|
|
from src.auth_helpers import get_current_user
|
|
from src.settings import load_settings, save_settings, load_features, save_features
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def setup_backup_routes(memory_manager, preset_manager, skills_manager) -> APIRouter:
|
|
router = APIRouter(tags=["backup"])
|
|
|
|
@router.get("/api/export")
|
|
async def export_data(request: Request):
|
|
"""Export all user data as a downloadable JSON file."""
|
|
require_admin(request)
|
|
user = get_current_user(request)
|
|
|
|
# Memories (filtered by owner when auth is enabled)
|
|
memories = memory_manager.load(owner=user)
|
|
|
|
# Presets (shared across users — export all)
|
|
presets = preset_manager.get_all()
|
|
|
|
# Skills (filtered by owner when auth is enabled)
|
|
skills = skills_manager.load(owner=user)
|
|
|
|
# Settings
|
|
settings = load_settings()
|
|
|
|
# Feature flags
|
|
features = load_features()
|
|
|
|
# User preferences
|
|
from routes.prefs_routes import _load_for_user
|
|
preferences = _load_for_user(user)
|
|
|
|
export_data = {
|
|
"version": 1,
|
|
"exported_at": datetime.now().isoformat(),
|
|
"exported_by": user,
|
|
"memories": memories,
|
|
"presets": presets,
|
|
"skills": skills,
|
|
"settings": settings,
|
|
"features": features,
|
|
"preferences": preferences,
|
|
}
|
|
|
|
filename = f"odysseus_backup_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
|
|
return Response(
|
|
content=json.dumps(export_data, indent=2, ensure_ascii=False),
|
|
media_type="application/json",
|
|
headers={"Content-Disposition": f"attachment; filename={filename}"},
|
|
)
|
|
|
|
@router.post("/api/import")
|
|
async def import_data(request: Request):
|
|
"""Import user data from a previously exported JSON file. Merges with existing data."""
|
|
require_admin(request)
|
|
user = get_current_user(request)
|
|
try:
|
|
body = await request.json()
|
|
except Exception:
|
|
raise HTTPException(400, "Invalid JSON")
|
|
|
|
if not isinstance(body, dict):
|
|
raise HTTPException(400, "Expected a JSON object")
|
|
|
|
imported = []
|
|
|
|
# ── Memories ──
|
|
if "memories" in body and isinstance(body["memories"], list):
|
|
# Strict load: importing on top of an unreadable store would write
|
|
# only the incoming rows back and drop everything already saved.
|
|
try:
|
|
existing = memory_manager.load_all_for_update()
|
|
except MemoryStoreUnreadable as e:
|
|
logger.error("Refusing to import memories: %s", e)
|
|
raise HTTPException(
|
|
503, "Memory store is temporarily unreadable — nothing was imported."
|
|
)
|
|
# Dedup against THIS user's own memories only. Using every tenant's
|
|
# rows (load_all) meant a memory whose text matched any other
|
|
# user's was silently skipped, so the importing user lost their own
|
|
# data. The full store is still saved back below.
|
|
existing_texts = {e.get("text", "").strip().lower()
|
|
for e in existing if e.get("owner") == user}
|
|
added = 0
|
|
for mem in body["memories"]:
|
|
if not isinstance(mem, dict) or not mem.get("text"):
|
|
continue
|
|
if mem["text"].strip().lower() in existing_texts:
|
|
continue # skip duplicates
|
|
# Assign owner when auth is enabled
|
|
if user and not mem.get("owner"):
|
|
mem["owner"] = user
|
|
existing.append(mem)
|
|
existing_texts.add(mem["text"].strip().lower())
|
|
added += 1
|
|
memory_manager.save(existing)
|
|
imported.append(f"{added} memories")
|
|
|
|
# ── Skills ──
|
|
if "skills" in body and isinstance(body["skills"], list):
|
|
existing = skills_manager.load_all()
|
|
# Dedup against THIS user's own skills only. Using every tenant's
|
|
# rows (load_all) meant a skill whose id/name/title matched any
|
|
# other user's was silently skipped, so the importing user lost
|
|
# their own data — same cross-tenant bug fixed for memories above.
|
|
# The full store is still saved back below.
|
|
own = [s for s in existing if s.get("owner") == user]
|
|
existing_names = {s.get("name") for s in own if s.get("name")}
|
|
existing_ids = {s.get("id") for s in own if s.get("id")}
|
|
existing_titles = {
|
|
(s.get("title") or s.get("description") or "").strip().lower()
|
|
for s in own
|
|
}
|
|
added = 0
|
|
for skill in body["skills"]:
|
|
if not isinstance(skill, dict):
|
|
continue
|
|
title = (
|
|
skill.get("title") or skill.get("description")
|
|
or skill.get("name") or ""
|
|
).strip()
|
|
if not title:
|
|
continue
|
|
sid = skill.get("id") or skill.get("name")
|
|
if sid and sid in existing_ids:
|
|
continue
|
|
nm = skill.get("name")
|
|
if nm and nm in existing_names:
|
|
continue
|
|
if title.lower() in existing_titles:
|
|
continue
|
|
owner = skill.get("owner")
|
|
if user and not owner:
|
|
owner = user
|
|
# Skills live on disk as SKILL.md files; the old JSON-era
|
|
# skills_manager.save() no longer exists. Write each new skill
|
|
# via add_skill (source="user" skips auto-dedup — this is an
|
|
# explicit backup restore).
|
|
result = skills_manager.add_skill(
|
|
title=title,
|
|
name=skill.get("name"),
|
|
description=skill.get("description"),
|
|
problem=skill.get("problem", ""),
|
|
solution=skill.get("solution", ""),
|
|
steps=skill.get("steps"),
|
|
tags=skill.get("tags"),
|
|
source="user",
|
|
teacher_model=skill.get("teacher_model"),
|
|
confidence=skill.get("confidence", 0.8),
|
|
owner=owner,
|
|
category=skill.get("category", "general"),
|
|
when_to_use=skill.get("when_to_use"),
|
|
procedure=skill.get("procedure"),
|
|
pitfalls=skill.get("pitfalls"),
|
|
verification=skill.get("verification"),
|
|
platforms=skill.get("platforms"),
|
|
requires_toolsets=skill.get("requires_toolsets"),
|
|
fallback_for_toolsets=skill.get("fallback_for_toolsets"),
|
|
status=skill.get("status", "draft"),
|
|
version=skill.get("version", "1.0.0"),
|
|
)
|
|
if result.get("_deduped"):
|
|
continue
|
|
if result.get("name"):
|
|
existing_names.add(result["name"])
|
|
if result.get("id"):
|
|
existing_ids.add(result["id"])
|
|
existing_titles.add(title.lower())
|
|
added += 1
|
|
imported.append(f"{added} skills")
|
|
|
|
# ── Presets ──
|
|
if "presets" in body and isinstance(body["presets"], dict):
|
|
current = preset_manager.get_all()
|
|
for key, value in body["presets"].items():
|
|
if isinstance(value, dict):
|
|
current[key] = value
|
|
elif isinstance(value, list):
|
|
current[key] = value
|
|
preset_manager.save(current)
|
|
imported.append("presets")
|
|
|
|
# ── Settings ──
|
|
if "settings" in body and isinstance(body["settings"], dict):
|
|
current = load_settings()
|
|
current.update(body["settings"])
|
|
save_settings(current)
|
|
imported.append("settings")
|
|
|
|
# ── Features ──
|
|
if "features" in body and isinstance(body["features"], dict):
|
|
current = load_features()
|
|
current.update(body["features"])
|
|
save_features(current)
|
|
imported.append("features")
|
|
|
|
# ── Preferences ──
|
|
if "preferences" in body and isinstance(body["preferences"], dict):
|
|
from routes.prefs_routes import _load_for_user, _save_for_user
|
|
current = _load_for_user(user)
|
|
current.update(body["preferences"])
|
|
_save_for_user(user, current)
|
|
imported.append("preferences")
|
|
|
|
if not imported:
|
|
return {"ok": False, "message": "No recognized data found in the file"}
|
|
|
|
return {"ok": True, "imported": imported, "message": f"Imported: {', '.join(imported)}"}
|
|
|
|
return router
|