mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-08-07 22:18:50 -04:00
Husam
f1e96d102e
The `end` branch of _QWEN_BARE_MARKER_RE had both pipes optional (`\|?end\|?`), so it also matched a bare `end` between whitespace and replaced it with a space. Messages containing Ruby, Lua or shell code that closes a block with a lone `end` had those lines deleted, and ordinary prose lost the word too. Require at least one pipe so only real turn markers match; `|end`, `end|`, `|end|` and `/|end|` strip exactly as before. Applied to the duplicated pattern in static/js/chatRenderer.js as well. Fixes #5547
97 lines
3.6 KiB
Python
97 lines
3.6 KiB
Python
"""Regression: the Qwen bare-marker scrub must not eat a lone `end` (#5547).
|
|
|
|
`_QWEN_BARE_MARKER_RE` cleans Qwen turn markers that leak into content. Its
|
|
`end` branch was `\\|?end\\|?` — both pipes optional — so it also matched a bare
|
|
`end` surrounded by whitespace and replaced it with a space. Any message
|
|
containing Ruby, Lua or shell code that closes a block with a lone `end` had
|
|
those lines silently deleted, in the stored text and in the rendered message.
|
|
|
|
Requiring at least one pipe keeps every real marker (`|end`, `end|`, `|end|`,
|
|
`/|end|`) stripping as before. The same pattern is duplicated in
|
|
static/js/chatRenderer.js, so the JS copy is checked here too — the two must
|
|
not drift.
|
|
"""
|
|
import json
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
import src.agent_tools # noqa: F401 (break agent_tools<->tool_parsing import cycle)
|
|
from src.tool_parsing import strip_tool_blocks
|
|
|
|
_REPO = Path(__file__).resolve().parent.parent
|
|
_CHAT_RENDERER = _REPO / "static" / "js" / "chatRenderer.js"
|
|
|
|
# Inputs that must survive untouched, and the substring that proves they did.
|
|
KEPT = [
|
|
("loop do\n puts \"yo\"\nend\n", "\nend"), # the reported Ruby case
|
|
("if x then\nend", "\nend"),
|
|
("function f()\nend\n", "\nend"),
|
|
("a end b", "a end b"),
|
|
("append end", "append end"),
|
|
("END", "END"),
|
|
("\nEnd\n", "End"),
|
|
]
|
|
|
|
# Real markers — at least one pipe, plus the role word — with the exact output
|
|
# they must still produce. Asserted as equality rather than "marker not in out"
|
|
# so narrowing the pattern can't pass by deleting more than it should.
|
|
STRIPPED = [
|
|
("a |end| b", "a b"),
|
|
("a /|end| b", "a b"),
|
|
("a |end b", "a b"),
|
|
("a end| b", "a b"),
|
|
("x assistant y", "x y"),
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize("text,kept", KEPT)
|
|
def test_bare_end_survives_stripping(text, kept):
|
|
assert kept in strip_tool_blocks(text)
|
|
|
|
|
|
@pytest.mark.parametrize("text,expected", STRIPPED)
|
|
def test_piped_end_markers_are_still_stripped(text, expected):
|
|
assert strip_tool_blocks(text) == expected
|
|
|
|
|
|
def test_bare_end_inside_a_fenced_block_survives():
|
|
"""The scrub runs over the whole message, fenced regions included."""
|
|
out = strip_tool_blocks("Here:\n```ruby\nloop do\n puts 1\nend\n```\nDone.")
|
|
assert "\nend\n" in out
|
|
|
|
|
|
def _js_bare_marker_regex_source():
|
|
src = _CHAT_RENDERER.read_text(encoding="utf-8")
|
|
m = re.search(r"^const QWEN_BARE_MARKER_RE = (/.*/[gimsuy]*);$", src, re.MULTILINE)
|
|
assert m, "QWEN_BARE_MARKER_RE literal not found in chatRenderer.js"
|
|
return m.group(1)
|
|
|
|
|
|
def test_js_copy_of_the_pattern_matches_the_python_one():
|
|
"""Guard the duplication: the JS branch must require a pipe too."""
|
|
if shutil.which("node") is None:
|
|
pytest.skip("node binary not on PATH")
|
|
|
|
cases = [text for text, _ in KEPT] + [text for text, _ in STRIPPED]
|
|
script = (
|
|
"const RE = %s;\n"
|
|
"const cases = JSON.parse(process.argv[1]);\n"
|
|
"console.log(JSON.stringify(cases.map(c => c.replace(RE, ' '))));"
|
|
% _js_bare_marker_regex_source()
|
|
)
|
|
result = subprocess.run(
|
|
["node", "--input-type=module", "-e", script, json.dumps(cases)],
|
|
cwd=_REPO, capture_output=True, timeout=15, text=True,
|
|
)
|
|
assert result.returncode == 0, f"node failed:\n{result.stderr}"
|
|
got = json.loads(result.stdout.splitlines()[-1])
|
|
|
|
for (text, kept), out in zip(KEPT, got):
|
|
assert kept in out, f"JS regex dropped {kept!r} from {text!r}"
|
|
for (text, expected), out in zip(STRIPPED, got[len(KEPT):]):
|
|
assert out == expected, f"JS regex: {text!r} -> {out!r}, expected {expected!r}"
|