SalastilandClaude Opus 5.5
1387ed92fb
reaper (the game process subreaper) and the gameoverlayui launchers were missed by the original >8 KB inventory. Proton 9.0-4f and Experimental 11.0-20261001 are pinned for how Steam launches Windows games and for lsteamclient's Steamworks headers (88 SDK versions). Co-Authored-By: Claude Opus 5.5 <[email protected]>
343 lines
17 KiB
Python
Executable File
343 lines
17 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Pin and fetch the original source code of every open-source binary Steam ships.
|
|
|
|
python3 -I tools/sources.py resolve -> SOURCES.tsv, BINARY_SOURCES.tsv (committed)
|
|
python3 -I tools/sources.py fetch [--only ID-substring] [--chromium]
|
|
-> sources/<id>/ (gitignored)
|
|
|
|
Runtime binaries (decomp/runtime/PROVENANCE.tsv, matched by SHA-256 against their
|
|
.deb) map to the exact Debian/Valve source package (.dsc) of that build.
|
|
Steam-bundled third-party libraries are Valve's own builds, so they can't be
|
|
checksum-matched; they map to the upstream git tag/commit identified from version
|
|
strings or, where marked "inferred", from ABI/build evidence.
|
|
Anything without public source (Steam's own code, RAD Miles, ...) has no entry
|
|
here and stays decompiled under decomp/.
|
|
|
|
Downloads are untrusted data: .dsc files are checked against their SHA-256
|
|
listings and unpacked with dpkg-source; nothing fetched is executed.
|
|
"""
|
|
import argparse
|
|
import concurrent.futures as cf
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
SRC = ROOT / "sources"
|
|
DL = SRC / "_downloads"
|
|
CACHE = ROOT / "client" / "debs" / "srcmap.json"
|
|
UA = {"User-Agent": "steam-source-pinning/1 (personal research)"}
|
|
|
|
# Steam-bundled third-party libraries (Valve builds). id, name, version, git URL,
|
|
# ref, evidence, binaries (glob-ish suffixes under client/).
|
|
CLIENT = [
|
|
("git:ffmpeg-8.1.2", "FFmpeg", "8.1.2", "https://git.ffmpeg.org/ffmpeg.git", "n8.1.2",
|
|
"version string 'FFmpeg version 8.1.2', build prefix ffmpeg-8.1.2",
|
|
["libavcodec.so.62", "libavformat.so.62", "libavutil.so.60", "libavfilter.so.11",
|
|
"libswscale.so.9", "libswresample.so.6"]),
|
|
("git:ffmpeg-4.3.4", "FFmpeg", "4.3.4", "https://git.ffmpeg.org/ffmpeg.git", "n4.3.4",
|
|
"version string 'FFmpeg version 4.3.4' (Lavc58.91.100)",
|
|
["libavcodec.so.58", "libavformat.so.58", "libavutil.so.56", "libavfilter.so.7",
|
|
"libswscale.so.5", "libavresample.so.4"]),
|
|
("git:sdl-3.4.0", "SDL", "3.4.0", "https://github.com/libsdl-org/SDL.git", "release-3.4.0",
|
|
"version string 'release-3.4.0'", ["libSDL3.so", "libSDL3.so.0"]),
|
|
("git:sdl_image-3.2.0", "SDL_image", "3.2.0", "https://github.com/libsdl-org/SDL_image.git",
|
|
"release-3.2.0", "IMG_Version() returns 3002000", ["libSDL3_image.so.0"]),
|
|
("git:sdl_ttf-3.1.0", "SDL_ttf", "3.1.0", "https://github.com/libsdl-org/SDL_ttf.git",
|
|
"preview-3.1.0", "TTF_Version() returns 3001000", ["libSDL3_ttf.so.0"]),
|
|
("git:libvpx-338013712", "libvpx", "v1.10.0-78-g338013712",
|
|
"https://chromium.googlesource.com/webm/libvpx", "338013712",
|
|
"version string 'v1.10.0-78-g338013712' (exact commit)", ["libvpx.so.6"]),
|
|
("git:aom-3.6.1", "libaom", "3.6.1", "https://aomedia.googlesource.com/aom", "v3.6.1",
|
|
"aom_codec_version_str() 'v3.6.1', aom_codec_version() 0x30601", ["libaom.so.3"]),
|
|
("git:dav1d-1.0.0", "dav1d", "1.0.0", "https://code.videolan.org/videolan/dav1d.git", "1.0.0",
|
|
"inferred: soname libdav1d.so.6 without dav1d_version_api (added in 1.1.0); version string empty",
|
|
["libdav1d.so.6"]),
|
|
("git:vorbis-1.3.7", "libvorbis", "1.3.7", "https://github.com/xiph/vorbis.git", "v1.3.7",
|
|
"version string 'Xiph.Org libVorbis 1.3.7'",
|
|
["libvorbis.so.0", "libvorbisenc.so.2", "libvorbisfile.so.3"]),
|
|
("git:libavif-1.0.4", "libavif", "1.0.4", "https://github.com/AOMediaCodec/libavif.git", "v1.0.4",
|
|
"avifVersion() '1.0.4'", ["steamrt64/libavif.so.16"]),
|
|
("git:libavif-1.0.3", "libavif", "1.0.3", "https://github.com/AOMediaCodec/libavif.git", "v1.0.3",
|
|
"avifVersion() '1.0.3'", ["steamrt32/libavif.so.16"]),
|
|
("git:chromium-126.0.6478.183", "Chromium (for Valve's CEF build)", "126.0.6478.183",
|
|
"https://chromium.googlesource.com/chromium/src.git", "126.0.6478.183",
|
|
"UA string 'Chrome/126.0.6478.183'; Valve's own CEF build (does not match Spotify CEF 126.2.18 by checksum), "
|
|
"so Valve patches may be missing", ["libcef.so"]),
|
|
("git:cef-126.2.18", "CEF", "126.2.18+g3647d39", "https://github.com/chromiumembedded/cef.git", "3647d39",
|
|
"CEF release for Chromium 126.0.6478.183; Valve's exact CEF commit/patches unknown", ["libcef.so"]),
|
|
("git:angle-5d4df51d", "ANGLE", "chromium 126.0.6478.183 DEPS",
|
|
"https://chromium.googlesource.com/angle/angle.git", "5d4df51d1d7d6a290d54111527a4798f10c7ca3c",
|
|
"inferred: ships with libcef.so; pinned by Chromium 126.0.6478.183 DEPS", ["libEGL.so", "libGLESv2.so"]),
|
|
("git:swiftshader-da334852", "SwiftShader", "chromium 126.0.6478.183 DEPS",
|
|
"https://swiftshader.googlesource.com/SwiftShader.git", "da334852e70510d259bfa8cbaa7c5412966b2f41",
|
|
"inferred: ships with libcef.so; pinned by Chromium 126.0.6478.183 DEPS", ["libvk_swiftshader.so"]),
|
|
("git:vulkan-validation-944660e3", "Vulkan-ValidationLayers", "chromium 126.0.6478.183 vulkan-deps",
|
|
"https://github.com/KhronosGroup/Vulkan-ValidationLayers.git", "944660e342cfafb6c318d11731751d9a291434d4",
|
|
"inferred: ships with libcef.so; pinned by Chromium 126.0.6478.183 -> vulkan-deps f1dcf238 DEPS",
|
|
["libVkLayer_khronos_validation.so"]),
|
|
# Reference only (not shipped by the client): how Steam launches Windows games and
|
|
# lsteamclient, the game <-> steamclient.so bridge with every Steamworks SDK header.
|
|
("git:proton-9.0-4f", "Proton", "9.0-4f", "https://github.com/ValveSoftware/Proton.git", "proton-9.0-4f",
|
|
"installed 'Proton 9.0 (Beta)' version file", []),
|
|
("git:proton-experimental-11.0-20261001", "Proton Experimental", "11.0-20261001",
|
|
"https://github.com/ValveSoftware/Proton.git", "experimental-11.0-20261001",
|
|
"installed 'Proton - Experimental' version file", []),
|
|
]
|
|
STR_TOOLS = "https://gitlab.steamos.cloud/steamrt/steam-runtime-tools.git"
|
|
|
|
|
|
def get(url, binary=False, tries=3):
|
|
for i in range(tries):
|
|
try:
|
|
with urllib.request.urlopen(urllib.request.Request(url, headers=UA), timeout=120) as r:
|
|
data = r.read()
|
|
return data if binary else data.decode("utf-8", "replace")
|
|
except urllib.error.HTTPError as e:
|
|
if e.code in (403, 404, 410):
|
|
raise
|
|
except (urllib.error.URLError, TimeoutError):
|
|
pass
|
|
time.sleep(3 * (i + 1))
|
|
raise RuntimeError("failed: " + url)
|
|
|
|
|
|
def dsc_files(text):
|
|
"""[(sha256, size, name)] from a .dsc's Checksums-Sha256 section."""
|
|
sec = text.split("Checksums-Sha256:", 1)[-1].split("\n\n")[0]
|
|
return re.findall(r"^ ([0-9a-f]{64}) (\d+) (\S+)$", sec, re.M)
|
|
|
|
|
|
def resolve_runtime():
|
|
"""Map each matched runtime binary to its exact source package."""
|
|
cache = json.loads(CACHE.read_text()) if CACHE.exists() else {}
|
|
prov = [l.rstrip("\n").split("\t") for l in open(ROOT / "decomp/runtime/PROVENANCE.tsv")][1:]
|
|
groups = {} # (pool dir, source) -> {binary versions}
|
|
for r in prov:
|
|
if r[2] != "match":
|
|
continue
|
|
url, src, ver = r[9], r[7], r[4]
|
|
if "/pool/" in url or "snapshot.debian.org" in url:
|
|
key = (url.rsplit("/", 1)[0] + "/" if "/pool/" in url else "snapshot", src)
|
|
groups.setdefault(key, set()).add(ver) # epoch kept: snapshot.debian.org needs it
|
|
|
|
def work(item):
|
|
(base, src), vers = item
|
|
out = {}
|
|
if base == "snapshot":
|
|
for full in vers:
|
|
v = full.split(":", 1)[-1]
|
|
for sv in (full, re.sub(r"\+b\d+$", "", full)):
|
|
k = f"snapshot|{src}|{sv}"
|
|
if k in cache:
|
|
out[v] = cache[k]
|
|
break
|
|
try:
|
|
j = json.loads(get(f"https://snapshot.debian.org/mr/package/{urllib.parse.quote(src)}/"
|
|
f"{urllib.parse.quote(sv)}/srcfiles?fileinfo=1"))
|
|
except Exception:
|
|
time.sleep(1)
|
|
continue
|
|
dsc = [h for h, info in j["fileinfo"].items() if info[0]["name"].endswith(".dsc")][0]
|
|
cache[k] = out[v] = {"version": sv.split(":", 1)[-1], "dsc": "https://snapshot.debian.org/file/" + dsc,
|
|
"dsc_name": j["fileinfo"][dsc][0]["name"]}
|
|
break
|
|
return src, out
|
|
k = f"pool|{base}"
|
|
if k not in cache:
|
|
idx = get(base)
|
|
cache[k] = [urllib.parse.unquote(h) for h in re.findall(r'href="([^"/]+\.dsc)"', idx)]
|
|
for v in {x.split(":", 1)[-1] for x in vers}:
|
|
c = [d for d in cache[k] if d.startswith(src + "_") and v.startswith(d[len(src) + 1:-4])]
|
|
if c:
|
|
d = max(c, key=len)
|
|
out[v] = {"version": d[len(src) + 1:-4], "dsc": base + urllib.parse.quote(d), "dsc_name": d}
|
|
return src, out
|
|
|
|
resolved = {} # (src, binary version) -> info
|
|
with cf.ThreadPoolExecutor(8) as ex:
|
|
for src, out in ex.map(work, groups.items()):
|
|
for v, info in out.items():
|
|
resolved[(src, v)] = info
|
|
CACHE.write_text(json.dumps(cache))
|
|
return prov, resolved
|
|
|
|
|
|
def main_resolve():
|
|
prov, resolved = resolve_runtime()
|
|
sources = {} # id -> row
|
|
binmap = [] # (binary, source id, evidence)
|
|
pv_versions = set()
|
|
for r in prov:
|
|
path, h, status, pkg, ver, arch, member, src, repo, url = r
|
|
if status != "match":
|
|
continue
|
|
rel = "client/runtime/" + path
|
|
if repo == "pressure-vessel":
|
|
sid = f"pv:{ver}"
|
|
pv_versions.add(ver)
|
|
binmap.append((rel, sid, f"sha256 matches {member} in {url.rsplit('/', 1)[-1]}"))
|
|
binmap.append((rel, f"git:steam-runtime-tools-{ver}", "pressure-vessel release of this version"))
|
|
continue
|
|
if repo == "steamrt-images-scout":
|
|
# steam-runtime-tools binaries from an archived scout image
|
|
sid = "git:steam-runtime-tools-0.20260618.0"
|
|
binmap.append((rel, sid, f"sha256 matches {member} in scout image {ver}"))
|
|
continue
|
|
info = resolved.get((src, ver.split(":", 1)[-1]))
|
|
if not info:
|
|
binmap.append((rel, "", f"sha256 matches {pkg} {ver}; source package not resolved"))
|
|
continue
|
|
sid = f"dsc:{src}_{info['version']}"
|
|
sources.setdefault(sid, ["dsc", src, info["version"], info["dsc"], "-",
|
|
"binary sha256 matches the package built from this source"])
|
|
binmap.append((rel, sid, f"sha256 matches {pkg}_{ver}_{arch} {member}"))
|
|
for v in sorted(pv_versions):
|
|
sources[f"pv:{v}"] = ["dsc-dir", "pressure-vessel bundled libraries", v,
|
|
f"https://repo.steampowered.com/pressure-vessel/snapshots/{v}/sources/", "-",
|
|
"source packages Valve publishes with this pressure-vessel release"]
|
|
for v in sorted(pv_versions | {"0.20260618.0"}):
|
|
sources[f"git:steam-runtime-tools-{v}"] = ["git", "steam-runtime-tools", v, STR_TOOLS, f"v{v}",
|
|
"release tag matching the runtime's tool version"]
|
|
client = ROOT / "client"
|
|
for sid, name, ver, url, ref, evidence, bins in CLIENT:
|
|
sources[sid] = ["git", name, ver, url, ref, evidence]
|
|
for b in bins:
|
|
for p in sorted(client.glob("*/" + b)) if "/" not in b else [client / b]:
|
|
if p.exists():
|
|
binmap.append(("client/" + str(p.relative_to(client)), sid, evidence))
|
|
with open(ROOT / "SOURCES.tsv", "w") as f:
|
|
f.write("id\tkind\tname\tversion\tlocation\tref\tevidence\n")
|
|
for sid in sorted(sources):
|
|
f.write(sid + "\t" + "\t".join(sources[sid]) + "\n")
|
|
with open(ROOT / "BINARY_SOURCES.tsv", "w") as f:
|
|
f.write("binary\tsource_id\tevidence\n")
|
|
for row in sorted(binmap):
|
|
f.write("\t".join(row) + "\n")
|
|
unresolved = sum(1 for b in binmap if not b[1])
|
|
print(f"{len(sources)} sources, {len(binmap)} binary mappings, {unresolved} unresolved")
|
|
|
|
|
|
# ---------------------------------------------------------------- fetch
|
|
|
|
def fetch_file(url, dest, sha256=None):
|
|
if dest.exists() and (not sha256 or hashlib.sha256(dest.read_bytes()).hexdigest() == sha256):
|
|
return
|
|
data = get(url, binary=True)
|
|
if sha256 and hashlib.sha256(data).hexdigest() != sha256:
|
|
raise RuntimeError(f"sha256 mismatch for {url}")
|
|
# unique temp name: two source versions can share one .orig tarball
|
|
tmp = dest.with_name(f"{dest.name}.{threading.get_ident()}.part")
|
|
tmp.write_bytes(data)
|
|
tmp.rename(dest)
|
|
|
|
|
|
def fetch_dsc(dsc_url, dsc_name, outdir):
|
|
d = DL / "dsc"
|
|
d.mkdir(parents=True, exist_ok=True)
|
|
dsc = d / dsc_name
|
|
fetch_file(dsc_url, dsc)
|
|
text = dsc.read_text(errors="replace")
|
|
if "snapshot.debian.org" in dsc_url:
|
|
# snapshot.debian.org serves files by SHA-1, which the .dsc lists
|
|
sec = text.split("Checksums-Sha1:", 1)[-1].split("Checksums-Sha256:", 1)[0]
|
|
byname = {n: "https://snapshot.debian.org/file/" + h
|
|
for h, n in re.findall(r"^ ([0-9a-f]{40}) \d+ (\S+)$", sec, re.M)}
|
|
else:
|
|
base = dsc_url.rsplit("/", 1)[0] + "/"
|
|
byname = None
|
|
for sha, size, name in dsc_files(text):
|
|
url = byname[name] if byname else base + urllib.parse.quote(name)
|
|
fetch_file(url, d / name, sha)
|
|
if outdir.exists():
|
|
return
|
|
tmp = outdir.with_name(outdir.name + ".tmp")
|
|
shutil.rmtree(tmp, ignore_errors=True)
|
|
subprocess.run(["dpkg-source", "--no-check", "-x", str(dsc), str(tmp)], check=True,
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
|
|
tmp.rename(outdir)
|
|
for f in outdir.parent.iterdir(): # dpkg-source copies the tarballs beside the tree
|
|
if f.is_file() and f.name.startswith(dsc_name.split("_", 1)[0] + "_"):
|
|
f.unlink()
|
|
|
|
|
|
def fetch_git(url, ref, outdir):
|
|
if outdir.exists():
|
|
return
|
|
tmp = outdir.with_name(outdir.name + ".tmp")
|
|
shutil.rmtree(tmp, ignore_errors=True)
|
|
if re.fullmatch(r"[0-9a-f]{40}", ref):
|
|
subprocess.run(["git", "init", "-q", str(tmp)], check=True)
|
|
subprocess.run(["git", "-C", str(tmp), "fetch", "-q", "--depth", "1", url, ref], check=True)
|
|
subprocess.run(["git", "-C", str(tmp), "checkout", "-q", "FETCH_HEAD"], check=True)
|
|
elif re.fullmatch(r"[0-9a-f]{7,39}", ref): # abbreviated commit: needs history
|
|
subprocess.run(["git", "clone", "-q", "--filter=blob:none", url, str(tmp)], check=True)
|
|
subprocess.run(["git", "-C", str(tmp), "checkout", "-q", ref], check=True)
|
|
else:
|
|
subprocess.run(["git", "clone", "-q", "--depth", "1", "--branch", ref, url, str(tmp)], check=True)
|
|
tmp.rename(outdir)
|
|
|
|
|
|
def main_fetch(only, chromium):
|
|
rows = [l.rstrip("\n").split("\t") for l in open(ROOT / "SOURCES.tsv")][1:]
|
|
SRC.mkdir(exist_ok=True)
|
|
jobs = []
|
|
for sid, kind, name, ver, loc, ref, ev in rows:
|
|
if only and only not in sid:
|
|
continue
|
|
if sid.startswith("git:chromium") and not chromium:
|
|
continue # ~30 GB with deps; opt in with --chromium
|
|
jobs.append((sid, kind, loc, ref))
|
|
|
|
def work(job):
|
|
sid, kind, loc, ref = job
|
|
out = SRC / sid.replace(":", "/", 1)
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
if kind == "dsc":
|
|
src_, ver_ = sid.split(":", 1)[1].rsplit("_", 1)
|
|
fetch_dsc(loc, f"{src_}_{ver_.split(':', 1)[-1]}.dsc", out)
|
|
elif kind == "dsc-dir":
|
|
idx = get(loc)
|
|
out.mkdir(exist_ok=True)
|
|
for d in re.findall(r'href="([^"/]+\.dsc)"', idx):
|
|
name = urllib.parse.unquote(d)
|
|
fetch_dsc(loc + d, name, out / name[:-4])
|
|
elif kind == "git":
|
|
fetch_git(loc, ref, out)
|
|
return sid, None
|
|
except Exception as e:
|
|
return sid, str(e)[-300:]
|
|
|
|
failed = []
|
|
with cf.ThreadPoolExecutor(6) as ex:
|
|
for n, (sid, err) in enumerate(ex.map(work, jobs), 1):
|
|
if err:
|
|
failed.append((sid, err))
|
|
if n % 50 == 0:
|
|
print(f" {n}/{len(jobs)} ({len(failed)} failed)", flush=True)
|
|
with open(SRC / "FETCH_ERRORS.tsv", "w") as f:
|
|
for sid, err in failed:
|
|
f.write(f"{sid}\t{err!r}\n")
|
|
print(f"fetched {len(jobs) - len(failed)}/{len(jobs)}; failures in sources/FETCH_ERRORS.tsv")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
ap = argparse.ArgumentParser()
|
|
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
sub.add_parser("resolve")
|
|
f = sub.add_parser("fetch")
|
|
f.add_argument("--only")
|
|
f.add_argument("--chromium", action="store_true")
|
|
a = ap.parse_args()
|
|
sys.exit(main_resolve() if a.cmd == "resolve" else main_fetch(a.only, a.chromium))
|