Salastil/odysseus
1
0
Fork 0
mirror of https://github.com/pewdiepie-archdaemon/odysseus.git synced 2026-06-17 02:05:22 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat: select cached gguf artifacts for serve (#891)

Browse Source
This commit is contained in:
spooky
2026-06-02 13:32:40 +10:00
committed by GitHub
parent 8455b88643
commit 8b3c0d8ad4
4 changed files with 131 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
routes/cookbook_helpers.py
+39 -8
View File
@@ -191,6 +191,38 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None) -> str:
" for root, dirs, fns in os.walk(top, followlinks=False):",
" dirs[:] = [d for d in dirs if not os.path.islink(os.path.join(root, d)) and safe_path(os.path.join(root, d))]",
" yield root, dirs, fns",
"def gguf_role(name):",
" n = name.lower()",
" if n.startswith('mmproj') or 'mmproj' in n: return 'projector'",
" return 'model'",
"def gguf_quant(name):",
" m = re.search(r'(?i)(UD-)?(IQ[0-9]_[A-Z0-9_]+|Q[0-9](?:_[A-Z0-9]+)+|BF16|F16|FP16|F32|Q8_0)', name)",
" return m.group(0).upper() if m else ''",
"def collect_ggufs(base):",
" files = []",
" split_groups = {}",
" if not os.path.isdir(base) or not safe_path(base): return files",
" for root, dirs, fns in safe_walk(base):",
" for fn in sorted(fns):",
" if not fn.lower().endswith('.gguf'): continue",
" fp = os.path.join(root, fn)",
" try: size = os.path.getsize(fp)",
" except Exception: size = 0",
" try: rel = os.path.relpath(fp, base).replace(os.sep, '/')",
" except Exception: rel = fn",
" sm = re.match(r'(?i)^(.+)-(\\d+)-of-(\\d+)\\.gguf$', fn)",
" if sm:",
" prefix, part_s, total_s = sm.group(1), sm.group(2), sm.group(3)",
" key = (root, prefix, total_s)",
" g = split_groups.setdefault(key, {'name':fn,'rel_path':rel,'size_bytes':0,'role':gguf_role(fn),'quant':gguf_quant(fn),'parts':int(total_s),'split':True})",
" g['size_bytes'] += size",
" if int(part_s) == 1:",
" g.update({'name':fn,'rel_path':rel,'role':gguf_role(fn),'quant':gguf_quant(fn)})",
" continue",
" files.append({'name':fn,'rel_path':rel,'size_bytes':size,'role':gguf_role(fn),'quant':gguf_quant(fn)})",
" files.extend(split_groups.values())",
" files.sort(key=lambda f: (f.get('role') != 'model', f.get('rel_path', '')))",
" return files",
"def scan_hf(cache):",
" if not os.path.isdir(cache): return",
" for d in sorted(os.listdir(cache)):",
@@ -205,16 +237,14 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None) -> str:
" if f.is_file(): nf += 1; sz += f.stat().st_size",
" if f.name.endswith('.incomplete'): ic = True",
" snap = os.path.join(cache, d, 'snapshots')",
" is_diffusion = False; is_gguf = False",
" is_diffusion = False; gguf_files = []",
" if os.path.isdir(snap):",
" for sd in os.listdir(snap):",
" sf = os.path.join(snap, sd)",
" if not os.path.isdir(sf): continue",
" if os.path.exists(os.path.join(sf, 'model_index.json')): is_diffusion = True",
" try:",
" if any(x.endswith('.gguf') for x in os.listdir(sf)): is_gguf = True",
" except Exception: pass",
" models.append({'repo_id':rid,'size_bytes':sz,'nb_files':nf,'has_incomplete':ic,'path':cache,'is_diffusion':is_diffusion,'is_gguf':is_gguf})",
" for f in collect_ggufs(sf): f['rel_path'] = sd + '/' + f['rel_path']; gguf_files.append(f)",
" models.append({'repo_id':rid,'size_bytes':sz,'nb_files':nf,'has_incomplete':ic,'path':cache,'is_diffusion':is_diffusion,'is_gguf':bool(gguf_files),'gguf_files':gguf_files})",
"def scan_dir(p):",
" if not os.path.isdir(p) or not safe_path(p): return",
" for d in sorted(os.listdir(p)):",
@@ -223,13 +253,14 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None) -> str:
" fp = os.path.join(p, d)",
" if not os.path.isdir(fp) or os.path.islink(fp) or not safe_path(fp): continue",
" if d in seen: continue",
" is_model = False; is_gguf = False",
" is_model = False; gguf_files = []",
" for root, dirs, fns in safe_walk(fp):",
" for fn in fns:",
" if fn.endswith('.gguf'): is_gguf = True; is_model = True",
" if fn.lower().endswith('.gguf'): is_model = True",
" elif fn == 'config.json' or fn.endswith('.safetensors') or fn.endswith('.bin'): is_model = True",
" if is_model: break",
" if not is_model: continue",
" gguf_files = collect_ggufs(fp)",
" seen.add(d)",
" sz, nf = 0, 0",
" for dp, _, fns in safe_walk(fp):",
@@ -237,7 +268,7 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None) -> str:
" try: nf += 1; sz += os.path.getsize(os.path.join(dp, fn))",
" except Exception: pass",
" is_diff = os.path.exists(os.path.join(fp, 'model_index.json'))",
" models.append({'repo_id':d,'size_bytes':sz,'nb_files':nf,'has_incomplete':False,'path':p,'is_local_dir':True,'is_diffusion':is_diff,'is_gguf':is_gguf})",
" models.append({'repo_id':d,'size_bytes':sz,'nb_files':nf,'has_incomplete':False,'path':p,'is_local_dir':True,'is_diffusion':is_diff,'is_gguf':bool(gguf_files),'gguf_files':gguf_files})",
"def parse_size(num, unit):",
" try: n = float(num)",
" except Exception: return 0",
routes/cookbook_routes.py
+2
View File
@@ -731,6 +731,8 @@ def setup_cookbook_routes() -> APIRouter:
entry["backend"] = m.get("backend")
if m.get("is_ollama"):
entry["is_ollama"] = True
if isinstance(m.get("gguf_files"), list):
entry["gguf_files"] = m["gguf_files"]
models.append(entry)
except Exception as e:
logger.warning(f"Failed to parse cached models: {e}")