mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-06-20 11:45:24 -04:00
refactor(tools): migrate execution logic to src/agent_tools/ package with handler registry (#3435)
* refactor(tools): implement strict cohesive class coordinator pattern per #2917 * test: update edit_file tests to use EditFileTool class * fix(tools): restore tool_policy param and security backstop in coordinator * refactor(tools): migrate domain tools to agent_tools package per #2917 * test: update test imports for new agent_tools package * fix: resolve circular import between tool_execution and agent_tools * fix: remove leftover git conflict markers * fix(tools): resolve pytest failure and document _apply method * fix(tools): clean up whitespace and remove dead _tool_python helper --------- Co-authored-by: Alexandre Teixeira <111787685+alteixeira20@users.noreply.github.com>
This commit is contained in:
Maanas
@@ -0,0 +1,131 @@
|
||||
"""
|
||||
agent_tools.py — Facade module.
|
||||
|
||||
Re-exports tool parsing, schemas, execution, and implementations
|
||||
for backward compatibility. All importers continue to work unchanged.
|
||||
|
||||
Sub-modules:
|
||||
- tool_parsing.py: regex patterns, parse/strip functions
|
||||
- tool_schemas.py: FUNCTION_TOOL_SCHEMAS, function_call_to_tool_block
|
||||
- tool_execution.py: execute_tool_block, format_tool_result, MCP helpers
|
||||
- tool_implementations.py: all do_* tool functions
|
||||
"""
|
||||
|
||||
import logging
|
||||
from collections import namedtuple
|
||||
|
||||
from src.tool_utils import _truncate, get_mcp_manager, set_mcp_manager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
from .subprocess_tools import BashTool, PythonTool
|
||||
from .web_tools import WebSearchTool, WebFetchTool
|
||||
from .filesystem_tools import ReadFileTool, WriteFileTool, EditFileTool, LsTool, GlobTool, GrepTool
|
||||
|
||||
TOOL_HANDLERS = {
|
||||
"bash": BashTool().execute,
|
||||
"python": PythonTool().execute,
|
||||
"web_search": WebSearchTool().execute,
|
||||
"web_fetch": WebFetchTool().execute,
|
||||
"read_file": ReadFileTool().execute,
|
||||
"write_file": WriteFileTool().execute,
|
||||
"edit_file": EditFileTool().execute,
|
||||
"ls": LsTool().execute,
|
||||
"glob": GlobTool().execute,
|
||||
"grep": GrepTool().execute,
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants (re-exported for backward compatibility — single source of truth
|
||||
# is src.constants; always prefer importing from there for new code)
|
||||
# ---------------------------------------------------------------------------
|
||||
MAX_AGENT_ROUNDS = 50
|
||||
SHELL_TIMEOUT = 60
|
||||
PYTHON_TIMEOUT = 30
|
||||
|
||||
# Tool types that trigger execution
|
||||
TOOL_TAGS = {"bash", "python", "web_search", "web_fetch", "read_file", "write_file", "edit_file",
|
||||
"grep", "glob", "ls",
|
||||
"create_document", "update_document", "edit_document",
|
||||
"search_chats",
|
||||
"chat_with_model", "create_session", "list_sessions",
|
||||
"send_to_session",
|
||||
"pipeline",
|
||||
"manage_session", "manage_memory", "list_models",
|
||||
"ui_control", "generate_image", "ask_user", "update_plan",
|
||||
"manage_tasks", "api_call", "ask_teacher", "manage_skills",
|
||||
"suggest_document",
|
||||
"manage_endpoints", "manage_mcp", "manage_webhooks",
|
||||
"manage_tokens", "manage_documents", "manage_settings",
|
||||
"manage_notes", "manage_calendar",
|
||||
"resolve_contact", "manage_contact", "list_email_accounts", "send_email", "list_emails",
|
||||
"read_email", "reply_to_email", "bulk_email", "archive_email",
|
||||
"delete_email", "mark_email_read",
|
||||
# Cookbook tools (LLM serving + downloads). Without these
|
||||
# entries, native function calls to e.g. list_served_models
|
||||
# are rejected as "Unknown function call" before reaching
|
||||
# the dispatcher — silent failure for the whole cookbook
|
||||
# surface.
|
||||
"download_model", "serve_model",
|
||||
"list_served_models", "stop_served_model",
|
||||
"list_downloads", "cancel_download",
|
||||
"search_hf_models", "list_cached_models",
|
||||
"list_serve_presets", "serve_preset", "adopt_served_model",
|
||||
"list_cookbook_servers",
|
||||
# Other tools the agent reaches for that were also missing.
|
||||
"edit_image", "trigger_research", "manage_research",
|
||||
# Generic loopback to any UI-button endpoint (cookbook,
|
||||
# gallery, email folders, etc.) — agent uses this when
|
||||
# there's no named tool wrapper for the action.
|
||||
"app_api"}
|
||||
|
||||
ToolBlock = namedtuple("ToolBlock", ["tool_type", "content"])
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Re-exports from sub-modules
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Parsing
|
||||
from src.tool_parsing import ( # noqa: E402, F401
|
||||
parse_tool_blocks,
|
||||
strip_tool_blocks,
|
||||
_TOOL_NAME_MAP,
|
||||
_TOOL_BLOCK_RE,
|
||||
_TOOL_CALL_RE,
|
||||
_XML_TOOL_CALL_RE,
|
||||
_XML_INVOKE_RE,
|
||||
_XML_PARAM_RE,
|
||||
)
|
||||
|
||||
# Schemas
|
||||
from src.tool_schemas import ( # noqa: E402, F401
|
||||
FUNCTION_TOOL_SCHEMAS,
|
||||
function_call_to_tool_block,
|
||||
)
|
||||
|
||||
# Execution
|
||||
from src.tool_execution import ( # noqa: E402, F401
|
||||
execute_tool_block,
|
||||
format_tool_result,
|
||||
)
|
||||
|
||||
# Implementations
|
||||
from src.tool_implementations import ( # noqa: E402, F401
|
||||
set_active_document,
|
||||
set_active_model,
|
||||
get_active_document,
|
||||
do_create_document,
|
||||
do_update_document,
|
||||
do_edit_document,
|
||||
do_suggest_document,
|
||||
do_search_chats,
|
||||
do_manage_skills,
|
||||
do_manage_tasks,
|
||||
do_manage_endpoints,
|
||||
do_manage_mcp,
|
||||
do_manage_webhooks,
|
||||
do_manage_tokens,
|
||||
do_manage_documents,
|
||||
do_manage_settings,
|
||||
do_api_call,
|
||||
)
|
||||
@@ -0,0 +1,419 @@
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import difflib
|
||||
import fnmatch
|
||||
import shutil
|
||||
from typing import Optional, Dict, Any, Tuple
|
||||
|
||||
from src.constants import MAX_READ_CHARS, MAX_DIFF_LINES, MAX_OUTPUT_CHARS
|
||||
|
||||
_CODENAV_SKIP_DIRS = frozenset({
|
||||
".git", ".hg", ".svn", "node_modules", "venv", ".venv", "__pycache__",
|
||||
".mypy_cache", ".pytest_cache", ".ruff_cache", "dist", "build",
|
||||
".next", ".cache", "site-packages", ".idea", ".tox",
|
||||
})
|
||||
_CODENAV_MAX_HITS = 200
|
||||
_CODENAV_MAX_LINE = 400
|
||||
|
||||
def _unified_diff(old: str, new: str, path: str) -> Optional[Dict[str, Any]]:
|
||||
if old == new:
|
||||
return None
|
||||
old_lines = old.splitlines()
|
||||
new_lines = new.splitlines()
|
||||
label = path or "file"
|
||||
diff_lines = list(difflib.unified_diff(
|
||||
old_lines, new_lines,
|
||||
fromfile=f"a/{label}", tofile=f"b/{label}",
|
||||
lineterm="",
|
||||
))
|
||||
added = sum(1 for line in diff_lines if line.startswith("+") and not line.startswith("+++"))
|
||||
removed = sum(1 for line in diff_lines if line.startswith("-") and not line.startswith("---"))
|
||||
truncated = False
|
||||
if len(diff_lines) > MAX_DIFF_LINES:
|
||||
diff_lines = diff_lines[:MAX_DIFF_LINES]
|
||||
truncated = True
|
||||
text = "\n".join(diff_lines)
|
||||
if truncated:
|
||||
text += f"\n… diff truncated at {MAX_DIFF_LINES} lines"
|
||||
return {
|
||||
"text": text,
|
||||
"added": added,
|
||||
"removed": removed,
|
||||
"new_file": old == "",
|
||||
"file": os.path.basename(path) or (path or "file"),
|
||||
}
|
||||
|
||||
class EditFileTool:
|
||||
async def execute(self, content: str, ctx: dict) -> dict:
|
||||
from src.tool_execution import (
|
||||
_resolve_tool_path,
|
||||
_resolve_tool_path_in_workspace,
|
||||
_resolve_search_root,
|
||||
_truncate
|
||||
)
|
||||
workspace = ctx.get("workspace")
|
||||
try:
|
||||
args = json.loads(content) if content.strip().startswith("{") else {}
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
args = {}
|
||||
raw_path = (args.get("path") or "").strip()
|
||||
old = args.get("old_string", "")
|
||||
new = args.get("new_string", "")
|
||||
replace_all = bool(args.get("replace_all", False))
|
||||
if not raw_path:
|
||||
return {"error": "edit_file: path required", "exit_code": 1}
|
||||
try:
|
||||
path = (_resolve_tool_path_in_workspace(workspace, raw_path)
|
||||
if workspace else _resolve_tool_path(raw_path))
|
||||
except ValueError as e:
|
||||
return {"error": f"edit_file: {e}", "exit_code": 1}
|
||||
if old == "":
|
||||
return {"error": "edit_file: old_string required (use write_file to create a file)", "exit_code": 1}
|
||||
if old == new:
|
||||
return {"error": "edit_file: old_string and new_string are identical", "exit_code": 1}
|
||||
|
||||
def _apply():
|
||||
"""Helper function that performs the actual string replacement and file writing logic."""
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
original = f.read()
|
||||
count = original.count(old)
|
||||
if count == 0:
|
||||
return original, None, "not_found"
|
||||
if count > 1 and not replace_all:
|
||||
return original, None, f"not_unique:{count}"
|
||||
updated = original.replace(old, new) if replace_all else original.replace(old, new, 1)
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
f.write(updated)
|
||||
return original, updated, "ok"
|
||||
|
||||
try:
|
||||
original, updated, status = await asyncio.to_thread(_apply)
|
||||
except FileNotFoundError:
|
||||
return {"error": f"edit_file: {path}: not found (use write_file to create it)", "exit_code": 1}
|
||||
except (IsADirectoryError, UnicodeDecodeError):
|
||||
return {"error": f"edit_file: {path}: not an editable text file", "exit_code": 1}
|
||||
except PermissionError:
|
||||
return {"error": f"edit_file: {path}: permission denied", "exit_code": 1}
|
||||
except OSError as e:
|
||||
return {"error": f"edit_file: {path}: {e}", "exit_code": 1}
|
||||
|
||||
if status == "not_found":
|
||||
return {"error": f"edit_file: old_string not found in {path}. Read the file and match it exactly.", "exit_code": 1}
|
||||
if status.startswith("not_unique"):
|
||||
n = status.split(":", 1)[1]
|
||||
return {"error": f"edit_file: old_string is not unique in {path} ({n} matches). Add surrounding context or set replace_all=true.", "exit_code": 1}
|
||||
|
||||
n = original.count(old)
|
||||
result = {"output": f"Edited {path} ({n} replacement{'s' if n != 1 else ''})", "exit_code": 0}
|
||||
diff = _unified_diff(original, updated, path)
|
||||
if diff:
|
||||
result["diff"] = diff
|
||||
return result
|
||||
|
||||
class ReadFileTool:
|
||||
async def execute(self, content: str, ctx: dict) -> dict:
|
||||
from src.tool_execution import (
|
||||
_resolve_tool_path,
|
||||
_resolve_tool_path_in_workspace,
|
||||
_resolve_search_root,
|
||||
_truncate
|
||||
)
|
||||
workspace = ctx.get("workspace")
|
||||
raw_path, offset, limit = content.split("\n", 1)[0].strip(), 0, 0
|
||||
_stripped = content.strip()
|
||||
if _stripped.startswith("{"):
|
||||
try:
|
||||
_a = json.loads(_stripped)
|
||||
raw_path = str(_a.get("path", "")).strip()
|
||||
offset = int(_a.get("offset") or 0)
|
||||
limit = int(_a.get("limit") or 0)
|
||||
except (json.JSONDecodeError, TypeError, ValueError):
|
||||
pass
|
||||
try:
|
||||
path = (_resolve_tool_path_in_workspace(workspace, raw_path)
|
||||
if workspace else _resolve_tool_path(raw_path))
|
||||
except ValueError as e:
|
||||
return {"error": f"read_file: {e}", "exit_code": 1}
|
||||
try:
|
||||
def _read():
|
||||
if offset > 0 or limit > 0:
|
||||
start = max(offset, 1)
|
||||
out, n, budget = [], 0, MAX_READ_CHARS
|
||||
with open(path, "r", encoding="utf-8", errors="replace") as f:
|
||||
for i, line in enumerate(f, 1):
|
||||
if i < start:
|
||||
continue
|
||||
if limit > 0 and n >= limit:
|
||||
break
|
||||
out.append(line)
|
||||
n += 1
|
||||
budget -= len(line)
|
||||
if budget <= 0:
|
||||
out.append(f"\n... [truncated at {MAX_READ_CHARS} chars]")
|
||||
break
|
||||
return "".join(out)
|
||||
with open(path, "r", encoding="utf-8", errors="replace") as f:
|
||||
return f.read(MAX_READ_CHARS + 1)
|
||||
data = await asyncio.to_thread(_read)
|
||||
except FileNotFoundError:
|
||||
return {"error": f"read_file: {path}: not found", "exit_code": 1}
|
||||
except PermissionError:
|
||||
return {"error": f"read_file: {path}: permission denied", "exit_code": 1}
|
||||
except IsADirectoryError:
|
||||
return {"error": f"read_file: {path}: is a directory (use ls)", "exit_code": 1}
|
||||
except OSError as e:
|
||||
return {"error": f"read_file: {path}: {e}", "exit_code": 1}
|
||||
if not (offset > 0 or limit > 0) and len(data) > MAX_READ_CHARS:
|
||||
data = data[:MAX_READ_CHARS] + f"\n... [truncated at {MAX_READ_CHARS} chars]"
|
||||
return {"output": data, "exit_code": 0}
|
||||
|
||||
class WriteFileTool:
|
||||
async def execute(self, content: str, ctx: dict) -> dict:
|
||||
from src.tool_execution import (
|
||||
_resolve_tool_path,
|
||||
_resolve_tool_path_in_workspace,
|
||||
_resolve_search_root,
|
||||
_truncate
|
||||
)
|
||||
workspace = ctx.get("workspace")
|
||||
lines = content.split("\n", 1)
|
||||
raw_path = lines[0].strip()
|
||||
body = lines[1] if len(lines) > 1 else ""
|
||||
try:
|
||||
path = (_resolve_tool_path_in_workspace(workspace, raw_path)
|
||||
if workspace else _resolve_tool_path(raw_path))
|
||||
except ValueError as e:
|
||||
return {"error": f"write_file: {e}", "exit_code": 1}
|
||||
try:
|
||||
def _write():
|
||||
old = ""
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
old = f.read()
|
||||
except (FileNotFoundError, IsADirectoryError, UnicodeDecodeError, OSError):
|
||||
old = ""
|
||||
d = os.path.dirname(path)
|
||||
if d:
|
||||
os.makedirs(d, exist_ok=True)
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
f.write(body)
|
||||
return old, len(body)
|
||||
old_content, size = await asyncio.to_thread(_write)
|
||||
except PermissionError:
|
||||
return {"error": f"write_file: {path}: permission denied", "exit_code": 1}
|
||||
except OSError as e:
|
||||
return {"error": f"write_file: {path}: {e}", "exit_code": 1}
|
||||
diff = _unified_diff(old_content, body, path)
|
||||
result = {"output": f"Wrote {size} bytes to {path}", "exit_code": 0}
|
||||
if diff:
|
||||
result["diff"] = diff
|
||||
return result
|
||||
|
||||
class LsTool:
|
||||
async def execute(self, content: str, ctx: dict) -> dict:
|
||||
from src.tool_execution import (
|
||||
_resolve_tool_path,
|
||||
_resolve_tool_path_in_workspace,
|
||||
_resolve_search_root,
|
||||
_truncate
|
||||
)
|
||||
workspace = ctx.get("workspace")
|
||||
raw_path = ""
|
||||
_s = (content or "").strip()
|
||||
if _s.startswith("{"):
|
||||
try:
|
||||
raw_path = str(json.loads(_s).get("path", "")).strip()
|
||||
except json.JSONDecodeError:
|
||||
raw_path = ""
|
||||
else:
|
||||
raw_path = _s.split("\n", 1)[0].strip()
|
||||
try:
|
||||
root = _resolve_search_root(raw_path)
|
||||
except ValueError as e:
|
||||
return {"error": f"ls: {e}", "exit_code": 1}
|
||||
|
||||
def _ls():
|
||||
if not os.path.isdir(root):
|
||||
return None, f"ls: {root}: not a directory"
|
||||
rows = []
|
||||
try:
|
||||
with os.scandir(root) as it:
|
||||
for entry in it:
|
||||
if entry.name.startswith("."):
|
||||
continue
|
||||
try:
|
||||
is_dir = entry.is_dir(follow_symlinks=False)
|
||||
size = entry.stat(follow_symlinks=False).st_size if not is_dir else 0
|
||||
except OSError:
|
||||
continue
|
||||
rows.append((is_dir, entry.name, size))
|
||||
except (PermissionError, OSError) as _e:
|
||||
return None, f"ls: {_e}"
|
||||
rows.sort(key=lambda r: (not r[0], r[1].lower()))
|
||||
lines = [f"{root}:"]
|
||||
for is_dir, name, size in rows[:_CODENAV_MAX_HITS]:
|
||||
lines.append(f" {name}/" if is_dir else f" {name} ({size} B)")
|
||||
if len(rows) > _CODENAV_MAX_HITS:
|
||||
lines.append(f" ... [{len(rows) - _CODENAV_MAX_HITS} more]")
|
||||
if not rows:
|
||||
lines.append(" (empty)")
|
||||
return "\n".join(lines), None
|
||||
|
||||
out, err = await asyncio.to_thread(_ls)
|
||||
if err:
|
||||
return {"error": err, "exit_code": 1}
|
||||
return {"output": _truncate(out), "exit_code": 0}
|
||||
|
||||
class GlobTool:
|
||||
async def execute(self, content: str, ctx: dict) -> dict:
|
||||
from src.tool_execution import (
|
||||
_resolve_tool_path,
|
||||
_resolve_tool_path_in_workspace,
|
||||
_resolve_search_root,
|
||||
_truncate
|
||||
)
|
||||
workspace = ctx.get("workspace")
|
||||
args = {}
|
||||
_s = (content or "").strip()
|
||||
if _s.startswith("{"):
|
||||
try:
|
||||
args = json.loads(_s)
|
||||
except json.JSONDecodeError:
|
||||
args = {}
|
||||
else:
|
||||
args = {"pattern": _s}
|
||||
pattern = str(args.get("pattern", "")).strip()
|
||||
if not pattern:
|
||||
return {"error": "glob: pattern is required", "exit_code": 1}
|
||||
try:
|
||||
root = _resolve_search_root(str(args.get("path", "")))
|
||||
except ValueError as e:
|
||||
return {"error": f"glob: {e}", "exit_code": 1}
|
||||
|
||||
def _glob():
|
||||
from pathlib import Path
|
||||
base = Path(root)
|
||||
if not base.is_dir():
|
||||
return None, f"glob: {root}: not a directory"
|
||||
matched = []
|
||||
try:
|
||||
for p in base.rglob(pattern):
|
||||
if set(p.relative_to(base).parts) & _CODENAV_SKIP_DIRS:
|
||||
continue
|
||||
try:
|
||||
mtime = p.stat().st_mtime
|
||||
except OSError:
|
||||
mtime = 0
|
||||
matched.append((mtime, str(p)))
|
||||
if len(matched) > _CODENAV_MAX_HITS * 5:
|
||||
break
|
||||
except (OSError, ValueError) as _e:
|
||||
return None, f"glob: {_e}"
|
||||
matched.sort(key=lambda t: t[0], reverse=True)
|
||||
return [pth for _, pth in matched[:_CODENAV_MAX_HITS]], None
|
||||
|
||||
paths, err = await asyncio.to_thread(_glob)
|
||||
if err:
|
||||
return {"error": err, "exit_code": 1}
|
||||
if not paths:
|
||||
return {"output": f"No files matching {pattern!r} under {root}", "exit_code": 0}
|
||||
out = "\n".join(paths)
|
||||
if len(paths) >= _CODENAV_MAX_HITS:
|
||||
out += f"\n... [capped at {_CODENAV_MAX_HITS} files]"
|
||||
return {"output": _truncate(out), "exit_code": 0}
|
||||
|
||||
class GrepTool:
|
||||
async def execute(self, content: str, ctx: dict) -> dict:
|
||||
from src.tool_execution import (
|
||||
_resolve_tool_path,
|
||||
_resolve_tool_path_in_workspace,
|
||||
_resolve_search_root,
|
||||
_truncate
|
||||
)
|
||||
workspace = ctx.get("workspace")
|
||||
args: Dict[str, Any] = {}
|
||||
_s = (content or "").strip()
|
||||
if _s.startswith("{"):
|
||||
try:
|
||||
args = json.loads(_s)
|
||||
except json.JSONDecodeError:
|
||||
args = {}
|
||||
else:
|
||||
args = {"pattern": _s}
|
||||
pattern = str(args.get("pattern", "")).strip()
|
||||
if not pattern:
|
||||
return {"error": "grep: pattern is required", "exit_code": 1}
|
||||
ignore_case = bool(args.get("ignore_case"))
|
||||
glob_pat = str(args.get("glob", "") or "").strip()
|
||||
try:
|
||||
max_hits = int(args.get("max_results") or _CODENAV_MAX_HITS)
|
||||
except (TypeError, ValueError):
|
||||
max_hits = _CODENAV_MAX_HITS
|
||||
max_hits = max(1, min(max_hits, _CODENAV_MAX_HITS))
|
||||
try:
|
||||
root = _resolve_search_root(str(args.get("path", "")))
|
||||
except ValueError as e:
|
||||
return {"error": f"grep: {e}", "exit_code": 1}
|
||||
|
||||
def _grep():
|
||||
import re as _re
|
||||
import shutil
|
||||
rg = shutil.which("rg")
|
||||
if rg:
|
||||
cmd = [rg, "--line-number", "--no-heading", "--color=never",
|
||||
"--max-count", str(max_hits)]
|
||||
if ignore_case:
|
||||
cmd.append("--ignore-case")
|
||||
if glob_pat:
|
||||
cmd += ["--glob", glob_pat]
|
||||
for _d in _CODENAV_SKIP_DIRS:
|
||||
cmd += ["--glob", f"!**/{_d}/**"]
|
||||
cmd += ["--regexp", pattern, root]
|
||||
try:
|
||||
import subprocess
|
||||
p = subprocess.run(cmd, capture_output=True, text=True, timeout=20)
|
||||
lines = [ln for ln in (p.stdout or "").splitlines() if ln][:max_hits]
|
||||
return lines, None
|
||||
except subprocess.TimeoutExpired:
|
||||
return None, "grep: timed out"
|
||||
except Exception as _e:
|
||||
return None, f"grep: {_e}"
|
||||
try:
|
||||
rx = _re.compile(pattern, _re.IGNORECASE if ignore_case else 0)
|
||||
except _re.error as _e:
|
||||
return None, f"grep: bad pattern: {_e}"
|
||||
hits = []
|
||||
if os.path.isfile(root):
|
||||
file_iter = [root]
|
||||
else:
|
||||
file_iter = []
|
||||
for dp, dns, fns in os.walk(root):
|
||||
dns[:] = [d for d in dns if d not in _CODENAV_SKIP_DIRS]
|
||||
for fn in fns:
|
||||
if glob_pat and not fnmatch.fnmatch(fn, glob_pat):
|
||||
continue
|
||||
file_iter.append(os.path.join(dp, fn))
|
||||
for fp in file_iter:
|
||||
if len(hits) >= max_hits:
|
||||
break
|
||||
try:
|
||||
with open(fp, "r", encoding="utf-8", errors="strict") as f:
|
||||
for i, line in enumerate(f, 1):
|
||||
if rx.search(line):
|
||||
hits.append(f"{fp}:{i}:{line.rstrip()[:_CODENAV_MAX_LINE]}")
|
||||
if len(hits) >= max_hits:
|
||||
break
|
||||
except (UnicodeDecodeError, OSError):
|
||||
continue
|
||||
return hits, None
|
||||
|
||||
lines, err = await asyncio.to_thread(_grep)
|
||||
if err:
|
||||
return {"error": err, "exit_code": 1}
|
||||
if not lines:
|
||||
return {"output": f"No matches for {pattern!r} under {root}", "exit_code": 0}
|
||||
out = "\n".join(ln[:_CODENAV_MAX_LINE] for ln in lines)
|
||||
if len(lines) >= max_hits:
|
||||
out += f"\n... [capped at {max_hits} matches]"
|
||||
return {"output": _truncate(out), "exit_code": 0}
|
||||
@@ -0,0 +1,155 @@
|
||||
import asyncio
|
||||
import sys
|
||||
import time
|
||||
import collections
|
||||
from typing import Optional, Callable, Awaitable, Tuple, Dict
|
||||
from src.constants import MAX_OUTPUT_CHARS
|
||||
|
||||
DEFAULT_BASH_TIMEOUT = 60 * 60 # 1 hour
|
||||
DEFAULT_PYTHON_TIMEOUT = 60 * 60
|
||||
|
||||
PROGRESS_INTERVAL_S = 2.0
|
||||
PROGRESS_TAIL_LINES = 12
|
||||
|
||||
async def _run_subprocess_streaming(
|
||||
proc: asyncio.subprocess.Process,
|
||||
*,
|
||||
timeout: float,
|
||||
progress_cb: Optional[Callable[[Dict], Awaitable[None]]] = None,
|
||||
) -> Tuple[str, str, Optional[int], bool]:
|
||||
started = time.time()
|
||||
stdout_full: list[str] = []
|
||||
stderr_full: list[str] = []
|
||||
tail = collections.deque(maxlen=PROGRESS_TAIL_LINES)
|
||||
|
||||
async def _reader(stream, full_buf, label: str):
|
||||
if stream is None:
|
||||
return
|
||||
while True:
|
||||
line = await stream.readline()
|
||||
if not line:
|
||||
break
|
||||
decoded = line.decode("utf-8", errors="replace").rstrip("\n")
|
||||
full_buf.append(decoded)
|
||||
if label == "err":
|
||||
tail.append(f"! {decoded}")
|
||||
else:
|
||||
tail.append(decoded)
|
||||
|
||||
async def _progress_emitter():
|
||||
await asyncio.sleep(PROGRESS_INTERVAL_S)
|
||||
while True:
|
||||
if progress_cb:
|
||||
try:
|
||||
await progress_cb({
|
||||
"elapsed_s": round(time.time() - started, 1),
|
||||
"tail": "\n".join(list(tail)),
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
await asyncio.sleep(PROGRESS_INTERVAL_S)
|
||||
|
||||
rd_out = asyncio.create_task(_reader(proc.stdout, stdout_full, "out"))
|
||||
rd_err = asyncio.create_task(_reader(proc.stderr, stderr_full, "err"))
|
||||
prog_task = asyncio.create_task(_progress_emitter()) if progress_cb else None
|
||||
|
||||
timed_out = False
|
||||
try:
|
||||
await asyncio.wait_for(proc.wait(), timeout=timeout)
|
||||
except asyncio.TimeoutError:
|
||||
timed_out = True
|
||||
try:
|
||||
proc.kill()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
await asyncio.wait_for(proc.wait(), timeout=2)
|
||||
except Exception:
|
||||
pass
|
||||
except asyncio.CancelledError:
|
||||
try:
|
||||
proc.kill()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
await asyncio.wait_for(proc.wait(), timeout=2)
|
||||
except Exception:
|
||||
pass
|
||||
for t in (rd_out, rd_err):
|
||||
t.cancel()
|
||||
if prog_task is not None:
|
||||
prog_task.cancel()
|
||||
raise
|
||||
finally:
|
||||
if prog_task is not None and not prog_task.done():
|
||||
prog_task.cancel()
|
||||
try:
|
||||
await prog_task
|
||||
except (asyncio.CancelledError, Exception):
|
||||
pass
|
||||
for t in (rd_out, rd_err):
|
||||
try:
|
||||
await asyncio.wait_for(t, timeout=1)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return (
|
||||
"\n".join(stdout_full),
|
||||
"\n".join(stderr_full),
|
||||
proc.returncode,
|
||||
timed_out,
|
||||
)
|
||||
|
||||
class BashTool:
|
||||
async def execute(self, content: str, ctx: dict) -> dict:
|
||||
from src.tool_execution import _AGENT_WORKDIR, _truncate
|
||||
progress_cb = ctx.get("progress_cb")
|
||||
workspace = ctx.get("workspace")
|
||||
_subproc_env = ctx.get("subproc_env")
|
||||
proc = await asyncio.create_subprocess_shell(
|
||||
content,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
env=_subproc_env,
|
||||
cwd=workspace or _AGENT_WORKDIR,
|
||||
)
|
||||
stdout, stderr, rc, timed_out = await _run_subprocess_streaming(
|
||||
proc,
|
||||
timeout=DEFAULT_BASH_TIMEOUT,
|
||||
progress_cb=progress_cb,
|
||||
)
|
||||
if timed_out:
|
||||
return {"error": f"bash: timed out after {DEFAULT_BASH_TIMEOUT}s — process killed", "exit_code": 124, "stdout": _truncate(stdout, MAX_OUTPUT_CHARS), "stderr": _truncate(stderr, MAX_OUTPUT_CHARS)}
|
||||
output = stdout.rstrip()
|
||||
err = stderr.rstrip()
|
||||
if err:
|
||||
output = (output + "\nSTDERR: " + err).strip() if output else "STDERR: " + err
|
||||
output = _truncate(output, MAX_OUTPUT_CHARS)
|
||||
return {"output": output or "(no output)", "exit_code": rc or 0}
|
||||
|
||||
class PythonTool:
|
||||
async def execute(self, content: str, ctx: dict) -> dict:
|
||||
from src.tool_execution import _AGENT_WORKDIR, _truncate
|
||||
progress_cb = ctx.get("progress_cb")
|
||||
workspace = ctx.get("workspace")
|
||||
_subproc_env = ctx.get("subproc_env")
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
(sys.executable or "python"), "-I", "-c", content,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
env=_subproc_env,
|
||||
cwd=workspace or _AGENT_WORKDIR,
|
||||
)
|
||||
stdout, stderr, rc, timed_out = await _run_subprocess_streaming(
|
||||
proc,
|
||||
timeout=DEFAULT_PYTHON_TIMEOUT,
|
||||
progress_cb=progress_cb,
|
||||
)
|
||||
if timed_out:
|
||||
return {"error": f"python: timed out after {DEFAULT_PYTHON_TIMEOUT}s — process killed", "exit_code": 124, "stdout": _truncate(stdout, MAX_OUTPUT_CHARS), "stderr": _truncate(stderr, MAX_OUTPUT_CHARS)}
|
||||
output = stdout.rstrip()
|
||||
err = stderr.rstrip()
|
||||
if err:
|
||||
output = (output + "\nSTDERR: " + err).strip() if output else "STDERR: " + err
|
||||
output = _truncate(output, MAX_OUTPUT_CHARS)
|
||||
return {"output": output or "(no output)", "exit_code": rc or 0}
|
||||
@@ -0,0 +1,101 @@
|
||||
import asyncio
|
||||
import json
|
||||
from typing import Dict, Any
|
||||
|
||||
from src.constants import MAX_OUTPUT_CHARS
|
||||
|
||||
class WebSearchTool:
|
||||
async def execute(self, content: str, ctx: dict) -> dict:
|
||||
from src.search import comprehensive_web_search
|
||||
raw = content.strip()
|
||||
query = raw
|
||||
time_filter = None
|
||||
max_pages = 5
|
||||
if raw.startswith("{"):
|
||||
try:
|
||||
parsed = json.loads(raw)
|
||||
if isinstance(parsed, dict) and "query" in parsed:
|
||||
query = str(parsed.get("query", "")).strip()
|
||||
tf = parsed.get("time_filter") or parsed.get("freshness")
|
||||
if isinstance(tf, str) and tf.lower() in ("day", "week", "month", "year"):
|
||||
time_filter = tf.lower()
|
||||
mp = parsed.get("max_pages")
|
||||
if isinstance(mp, int) and 1 <= mp <= 10:
|
||||
max_pages = mp
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
if not query:
|
||||
query = raw.split("\n")[0].strip()
|
||||
if time_filter is None:
|
||||
q_lc = query.lower()
|
||||
if any(kw in q_lc for kw in ("today", "latest", "breaking", "this morning", "right now", "currently")):
|
||||
time_filter = "day"
|
||||
elif any(kw in q_lc for kw in ("this week", "past week", "recent news", "last few days")):
|
||||
time_filter = "week"
|
||||
elif any(kw in q_lc for kw in ("this month", "past month")):
|
||||
time_filter = "month"
|
||||
elif " news" in q_lc or q_lc.startswith("news ") or q_lc.endswith(" news"):
|
||||
time_filter = "week"
|
||||
loop = asyncio.get_running_loop()
|
||||
text, sources = await asyncio.wait_for(
|
||||
loop.run_in_executor(
|
||||
None,
|
||||
lambda: comprehensive_web_search(
|
||||
query,
|
||||
max_pages=max_pages,
|
||||
time_filter=time_filter,
|
||||
return_sources=True,
|
||||
),
|
||||
),
|
||||
timeout=30,
|
||||
)
|
||||
output = text[:MAX_OUTPUT_CHARS] if len(text) > MAX_OUTPUT_CHARS else text
|
||||
if sources:
|
||||
output += "\n\n<!-- SOURCES:" + json.dumps(sources) + " -->"
|
||||
return {"output": output, "exit_code": 0}
|
||||
|
||||
class WebFetchTool:
|
||||
async def execute(self, content: str, ctx: dict) -> dict:
|
||||
from src.search.content import fetch_webpage_content
|
||||
raw = content.strip()
|
||||
url = ""
|
||||
if raw.startswith("{"):
|
||||
try:
|
||||
parsed = json.loads(raw)
|
||||
if isinstance(parsed, dict):
|
||||
url = str(parsed.get("url") or "").strip()
|
||||
except json.JSONDecodeError:
|
||||
url = ""
|
||||
if not url:
|
||||
url = raw.split("\n")[0].strip()
|
||||
if not url or url.startswith("{") or any(c in url for c in (" ", "\t", "\n")):
|
||||
return {"error": "web_fetch: provide a single URL or domain, e.g. example.com", "exit_code": 1}
|
||||
low = url.lower()
|
||||
if "://" in low and not low.startswith(("http://", "https://")):
|
||||
return {"error": f"web_fetch: unsupported URL scheme (only http/https): {url[:80]}", "exit_code": 1}
|
||||
if not low.startswith(("http://", "https://")):
|
||||
url = "https://" + url
|
||||
loop = asyncio.get_running_loop()
|
||||
try:
|
||||
result = await asyncio.wait_for(
|
||||
loop.run_in_executor(None, lambda: fetch_webpage_content(url, timeout=10)),
|
||||
timeout=30,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
return {"error": f"web_fetch: timed out fetching {url}", "exit_code": 1}
|
||||
except Exception as e:
|
||||
return {"error": f"web_fetch: {url}: {e}", "exit_code": 1}
|
||||
err = result.get("error")
|
||||
text = (result.get("content") or "").strip()
|
||||
title = result.get("title") or ""
|
||||
|
||||
if not text:
|
||||
if err:
|
||||
return {"error": f"web_fetch: {url}: {err}", "exit_code": 1}
|
||||
return {"error": f"web_fetch: {url}: no readable text content (not HTML, or the page needs JS/login)", "exit_code": 1}
|
||||
|
||||
header = (f"# {title}\n" if title else "") + f"Source: {url}\n\n"
|
||||
output = header + text
|
||||
if len(output) > MAX_OUTPUT_CHARS:
|
||||
output = output[:MAX_OUTPUT_CHARS] + "\n\n[...truncated]"
|
||||
return {"output": output, "exit_code": 0}
|
||||
Reference in New Issue
Block a user