330 lines
17 KiB
Python
330 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""Publish the staged WebGL Art and hot-update packages over Windows OpenSSH.
|
|
|
|
No password handling, remote deletion, or shell-built SCP commands. The remote
|
|
transaction verifies every file before exposing cdn/version.json.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import gzip
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import urllib.request
|
|
import uuid
|
|
import zipfile
|
|
import zlib
|
|
|
|
REPO = Path(__file__).resolve().parents[1]
|
|
DEFAULT_CONFIG = REPO / "Tools/cdn_publish.local.json"
|
|
OUTPUT = REPO / "Tools/output/cdn_publish"
|
|
REMOTE_SCRIPT = REPO / "Tools/Publish-CdnRelease.ps1"
|
|
HASH32 = re.compile(r"[a-f0-9]{32}\Z")
|
|
RESOURCE_PATH = re.compile(r"yoo/(Art|DefaultPackage)/[A-Za-z0-9_.-]+\Z")
|
|
STARTUP_PATH = re.compile(r"[a-f0-9]{16}\.webgl\.data\.unityweb\.bin\.txt\Z")
|
|
|
|
|
|
def read_json(path: Path) -> dict:
|
|
return json.loads(path.read_text(encoding="utf-8-sig"))
|
|
|
|
|
|
def write_json(path: Path, value: dict) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
|
|
|
|
def digest(path: Path, algorithm="sha256") -> str:
|
|
result = hashlib.new(algorithm)
|
|
with path.open("rb") as stream:
|
|
for block in iter(lambda: stream.read(1024 * 1024), b""):
|
|
result.update(block)
|
|
return result.hexdigest()
|
|
|
|
|
|
def validate_path(path: str) -> None:
|
|
if path != "cdn/version.json" and not RESOURCE_PATH.fullmatch(path) and not STARTUP_PATH.fullmatch(path):
|
|
raise ValueError(f"Unsupported release path: {path}")
|
|
if any(part in (".", "..") for part in path.split("/")):
|
|
raise ValueError(f"Unsafe release path: {path}")
|
|
|
|
|
|
def make_plan(stage: Path, destination: Path, startup_export: Path | None = None) -> dict:
|
|
version_file = stage / "version.prod.json"
|
|
version = read_json(version_file)
|
|
if version.get("cdnRoot") != "https://www.xworld.ren/game/Friends/webgl/yoo/DefaultPackage":
|
|
raise ValueError("Staged production version has an unexpected CDN root")
|
|
packages = version.get("packages")
|
|
if not isinstance(packages, list) or "DefaultPackage" not in packages or len(packages) != len(set(packages)):
|
|
raise ValueError("Invalid package list")
|
|
app = version.get("appVersionMax", "")
|
|
if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", app):
|
|
raise ValueError("Invalid appVersionMax")
|
|
files = []
|
|
|
|
def add(source: Path, relative: str):
|
|
validate_path(relative)
|
|
if not source.is_file():
|
|
raise ValueError(f"Missing staged file: {source}")
|
|
files.append({"path": relative, "source": str(source.resolve()),
|
|
"size": source.stat().st_size, "sha256": digest(source)})
|
|
|
|
for package in packages:
|
|
field = {"Art": "resVersion", "DefaultPackage": "packVersion"}.get(package)
|
|
if field is None or not HASH32.fullmatch(version.get(field, "")):
|
|
raise ValueError(f"Invalid package/version: {package}")
|
|
package_version = version[field]
|
|
directory = stage / app / package
|
|
index = directory / f"{package}_{package_version}.bytes"
|
|
if not index.is_file() or digest(index, "md5") != package_version:
|
|
raise ValueError(f"Manifest MD5 mismatch or missing index: {index}")
|
|
marker = directory / f"{package}.version"
|
|
if marker.read_text(encoding="utf-8-sig").strip() != package_version:
|
|
raise ValueError(f"Package version marker mismatch: {marker}")
|
|
checksum = directory / f"{package}_{package_version}.hash"
|
|
expected_crc = zlib.crc32(index.read_bytes()).to_bytes(4, "little").hex()
|
|
if checksum.read_text(encoding="utf-8-sig").strip().lower() != expected_crc:
|
|
raise ValueError(f"Package hash marker mismatch: {checksum}")
|
|
selected = [marker, index, checksum]
|
|
selected.extend(sorted(p for p in directory.iterdir() if p.suffix in (".bundle", ".rawfile")))
|
|
if len(selected) <= 3:
|
|
raise ValueError(f"No payload files in {directory}")
|
|
for source in selected:
|
|
add(source, f"yoo/{package}/{source.name}")
|
|
if startup_export is not None:
|
|
game = (startup_export / "minigame/game.js").read_text(encoding="utf-8-sig")
|
|
hashes = re.findall(r"\bDATA_FILE_MD5\s*:\s*['\"]([a-f0-9]{16})['\"]", game)
|
|
sizes = re.findall(r"\bDATA_FILE_SIZE\s*:\s*['\"]([0-9]+)['\"]", game)
|
|
roots = re.findall(r"\bDATA_CDN\s*:\s*['\"]([^'\"]+)['\"]", game)
|
|
if len(hashes) != 1 or len(sizes) != 1 or roots != ["https://www.xworld.ren/game/Friends/webgl/"]:
|
|
raise ValueError("Cannot identify the exported startup data/CDN")
|
|
name = hashes[0] + ".webgl.data.unityweb.bin.txt"
|
|
source = startup_export / "webgl" / name
|
|
# WX UnityUtil.BuildFileMd5 uses the middle 16 characters of MD5.
|
|
if not source.is_file() or source.stat().st_size != int(sizes[0]) or digest(source, "md5")[8:24] != hashes[0]:
|
|
raise ValueError("Exported startup data size or MD5 mismatch")
|
|
add(source, name)
|
|
add(version_file, "cdn/version.json")
|
|
plan = {"schema": 1, "version": version, "files": files,
|
|
"totalBytes": sum(f["size"] for f in files)}
|
|
write_json(destination, plan)
|
|
return plan
|
|
|
|
|
|
def load_config(path: Path) -> dict:
|
|
config = read_json(path)
|
|
if not re.fullmatch(r"[A-Za-z0-9_.-]+", config.get("host", "")) or config["host"].startswith("-"):
|
|
raise ValueError("Invalid SSH host")
|
|
if not re.fullmatch(r"[A-Za-z0-9_.-]+", config.get("user", "")):
|
|
raise ValueError("Invalid SSH user")
|
|
if not isinstance(config.get("port"), int) or not 1 <= config["port"] <= 65535:
|
|
raise ValueError("Invalid SSH port")
|
|
key = Path(config["identity_file"]).expanduser()
|
|
config["identity_file"] = str(key if key.is_absolute() else REPO / key)
|
|
if not config.get("remote_root") or not config.get("public_base", "").startswith("https://"):
|
|
raise ValueError("Missing remote_root or HTTPS public_base")
|
|
return config
|
|
|
|
|
|
def ssh_options(config: dict, scp=False) -> list[str]:
|
|
return ["-P" if scp else "-p", str(config["port"]),
|
|
"-o", "BatchMode=yes", "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=yes",
|
|
"-o", "IdentitiesOnly=yes", "-i", config["identity_file"]]
|
|
|
|
|
|
def remote(config: dict, action: str, request: dict) -> dict:
|
|
payload = base64.b64encode(json.dumps(request, separators=(",", ":")).encode()).decode()
|
|
script = REMOTE_SCRIPT.read_text(encoding="utf-8-sig")
|
|
code = f"& {{\n{script}\n}} -Action {action} -RequestBase64 '{payload}'"
|
|
# Feed the (potentially large) request over stdin; keep the Windows CMD command under its limit.
|
|
# Windows OpenSSH does not reliably deliver stdin EOF for larger redirected
|
|
# payloads. An explicit final line lets the server finish without waiting for it.
|
|
bootstrap = ("$lines=[Collections.Generic.List[string]]::new(); "
|
|
"while($null -ne ($line=[Console]::In.ReadLine())){if($line -eq '.'){break};$lines.Add($line)}; "
|
|
"$data=[Convert]::FromBase64String(($lines -join '')); "
|
|
"$stream=[IO.MemoryStream]::new($data); "
|
|
"$zip=[IO.Compression.GZipStream]::new($stream,[IO.Compression.CompressionMode]::Decompress); "
|
|
"$reader=[IO.StreamReader]::new($zip,[Text.Encoding]::UTF8); "
|
|
"& ([ScriptBlock]::Create($reader.ReadToEnd()))")
|
|
bootstrap64 = base64.b64encode(bootstrap.encode("utf-16le")).decode()
|
|
wire = base64.encodebytes(gzip.compress(code.encode("utf-8"))).decode("ascii") + ".\n"
|
|
try:
|
|
result = subprocess.run(["ssh", *ssh_options(config), f"{config['user']}@{config['host']}",
|
|
"powershell.exe", "-NoLogo", "-NoProfile", "-NonInteractive",
|
|
"-EncodedCommand", bootstrap64], input=wire, text=True,
|
|
encoding="utf-8", errors="replace", capture_output=True,
|
|
timeout=900 if action in ("Publish", "Rollback") else 120)
|
|
except subprocess.TimeoutExpired as error:
|
|
raise RuntimeError(f"SSH {action} timed out; inspect the server state before retrying a publication") from error
|
|
if result.returncode:
|
|
raise RuntimeError((result.stderr or result.stdout).strip())
|
|
try:
|
|
return json.loads(result.stdout.strip())
|
|
except json.JSONDecodeError as error:
|
|
raise RuntimeError(f"Unexpected SSH response: {result.stdout[:500]}") from error
|
|
|
|
|
|
def request_for(config: dict, plan: dict, release: str) -> dict:
|
|
return {"root": config["remote_root"], "releaseId": release,
|
|
"version": plan["version"],
|
|
"files": [{k: f[k] for k in ("path", "size", "sha256")} for f in plan["files"]]}
|
|
|
|
|
|
def make_archive(plan: dict, changed: list[str], path: Path) -> None:
|
|
wanted = set(changed)
|
|
by_path = {f["path"]: f for f in plan["files"]}
|
|
if not wanted.issubset(by_path):
|
|
raise ValueError("Server requested a path outside the release")
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_STORED, allowZip64=True) as archive:
|
|
for name in sorted(wanted):
|
|
entry = by_path[name]
|
|
source = Path(entry["source"])
|
|
if source.stat().st_size != entry["size"] or digest(source) != entry["sha256"]:
|
|
raise ValueError(f"Staged file changed since planning: {name}")
|
|
archive.write(source, name)
|
|
|
|
|
|
def verify_public(config: dict, version: dict, release: str, plan: dict | None = None) -> None:
|
|
root = config["public_base"].rstrip("/")
|
|
def get(relative):
|
|
req = urllib.request.Request(f"{root}/{relative}?publish={release}", headers={"Cache-Control": "no-cache"})
|
|
with urllib.request.urlopen(req, timeout=30) as response:
|
|
return response.read()
|
|
actual = json.loads(get("cdn/version.json").decode("utf-8-sig"))
|
|
if actual != version:
|
|
raise RuntimeError("Public version.json does not match the published release")
|
|
for package in version["packages"]:
|
|
expected = version["resVersion" if package == "Art" else "packVersion"]
|
|
data = get(f"yoo/{package}/{package}_{expected}.bytes")
|
|
if hashlib.md5(data).hexdigest() != expected:
|
|
raise RuntimeError(f"Public {package} manifest MD5 mismatch")
|
|
for entry in (plan or {}).get("files", []):
|
|
if STARTUP_PATH.fullmatch(entry["path"]):
|
|
data = get(entry["path"])
|
|
if len(data) != entry["size"] or hashlib.sha256(data).hexdigest() != entry["sha256"]:
|
|
raise RuntimeError("Public startup data SHA256 mismatch")
|
|
|
|
|
|
def init_key(config_path: Path) -> None:
|
|
if not config_path.exists():
|
|
shutil.copyfile(REPO / "Tools/cdn_publish.example.json", config_path)
|
|
config = load_config(config_path)
|
|
key = Path(config["identity_file"])
|
|
key.parent.mkdir(parents=True, exist_ok=True)
|
|
if not key.exists():
|
|
subprocess.run(["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-C", "aigame-cdn-deploy", "-f", str(key)], check=True)
|
|
if os.name == "nt":
|
|
path64 = base64.b64encode(str(key).encode()).decode()
|
|
code = "$p=[Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('" + path64 + "')); " + \
|
|
"$sid=[Security.Principal.WindowsIdentity]::GetCurrent().User; " + \
|
|
"$acl=New-Object Security.AccessControl.FileSecurity; $acl.SetOwner($sid); " + \
|
|
"$acl.SetAccessRuleProtection($true,$false); " + \
|
|
"$acl.AddAccessRule((New-Object Security.AccessControl.FileSystemAccessRule($sid,'FullControl','Allow'))); " + \
|
|
"$acl.AddAccessRule((New-Object Security.AccessControl.FileSystemAccessRule((New-Object Security.Principal.SecurityIdentifier('S-1-5-18')),'FullControl','Allow'))); " + \
|
|
"Set-Acl -LiteralPath $p -AclObject $acl -ErrorAction Stop"
|
|
subprocess.run(["powershell.exe", "-NoProfile", "-NonInteractive", "-EncodedCommand",
|
|
base64.b64encode(code.encode("utf-16le")).decode()], check=True)
|
|
if not key.with_suffix(".pub").is_file():
|
|
raise ValueError("Existing private key has no matching .pub file")
|
|
public_key = key.with_suffix(".pub").read_text(encoding="ascii").strip()
|
|
if not re.fullmatch(r"ssh-ed25519 [A-Za-z0-9+/=]+(?: .*)?", public_key):
|
|
raise ValueError("Unexpected public-key format")
|
|
setup = REPO / "Tools/.cdn-deploy/Install-PublicKey-OnServer.ps1"
|
|
setup.write_text("""# Run this file in an elevated PowerShell ON THE SERVER.
|
|
$ErrorActionPreference = 'Stop'
|
|
$path = Join-Path $env:ProgramData 'ssh/administrators_authorized_keys'
|
|
New-Item -ItemType Directory -Path (Split-Path -Parent $path) -Force | Out-Null
|
|
$key = 'restrict """ + public_key + """'
|
|
$existing = if (Test-Path -LiteralPath $path) { [IO.File]::ReadAllText($path) } else { '' }
|
|
if (-not $existing.Contains($key)) {
|
|
if ($existing -and -not $existing.EndsWith("`n")) { $existing += "`r`n" }
|
|
[IO.File]::WriteAllText($path, $existing + $key + "`r`n", (New-Object Text.UTF8Encoding($false)))
|
|
}
|
|
& icacls.exe $path /inheritance:r /grant:r '*S-1-5-32-544:F' '*S-1-5-18:F'
|
|
if ($LASTEXITCODE -ne 0) { throw 'Failed to set authorized_keys permissions' }
|
|
Write-Output 'AIGame CDN public key installed.'
|
|
""", encoding="utf-8-sig")
|
|
print(f"Dedicated key ready: {key}\nServer setup script (PUBLIC KEY ONLY): {setup}")
|
|
|
|
|
|
def main(argv=None) -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("command", choices=("init-key", "plan", "check", "upload", "rollback"))
|
|
parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG)
|
|
parser.add_argument("--stage", type=Path, default=REPO / "Tools/cdn_root/wechat")
|
|
parser.add_argument("--compare", action="store_true", help="Compare staged files with the server without publishing")
|
|
parser.add_argument("--startup-export", type=Path, help="Also publish startup data selected by EXPORT/minigame/game.js")
|
|
parser.add_argument("--release", help="Release ID to roll back")
|
|
args = parser.parse_args(argv)
|
|
if args.command == "init-key":
|
|
init_key(args.config)
|
|
return 0
|
|
config = load_config(args.config)
|
|
if args.command == "check":
|
|
print(json.dumps(remote(config, "Check", {"root": config["remote_root"]}), ensure_ascii=False, indent=2))
|
|
return 0
|
|
if args.command == "rollback":
|
|
if not args.release or not re.fullmatch(r"[a-f0-9]{32}", args.release):
|
|
raise ValueError("rollback requires --release with a 32-character release ID")
|
|
result = remote(config, "Rollback", {"root": config["remote_root"], "releaseId": args.release})
|
|
write_json(OUTPUT / f"rollback-{args.release}.json", result)
|
|
verify_public(config, result["version"], uuid.uuid4().hex)
|
|
print(json.dumps(result, ensure_ascii=False, indent=2))
|
|
return 0
|
|
OUTPUT.mkdir(parents=True, exist_ok=True)
|
|
plan = make_plan(args.stage, OUTPUT / "plan.json", args.startup_export)
|
|
print(f"Staged: {len(plan['files'])} files, {plan['totalBytes'] / 1048576:.2f} MiB; pack={plan['version']['packVersion']}; art={plan['version'].get('resVersion', '')}")
|
|
if args.command == "plan" and not args.compare:
|
|
print(f"File list: {OUTPUT / 'plan.json'}")
|
|
return 0
|
|
release = uuid.uuid4().hex
|
|
request = request_for(config, plan, release)
|
|
comparison = remote(config, "Plan", request)
|
|
changed = comparison["changed"]
|
|
write_json(OUTPUT / "comparison.json", comparison)
|
|
total = sum(f["size"] for f in plan["files"] if f["path"] in changed)
|
|
print(f"Remote difference: {len(changed)} files, {total / 1048576:.2f} MiB")
|
|
for name in changed:
|
|
print(f" {name}")
|
|
if args.command == "plan":
|
|
return 0
|
|
request["expectedVersionSha256"] = comparison["currentVersionSha256"]
|
|
if not changed:
|
|
verify_public(config, plan["version"], release, plan)
|
|
print("Already published; public version and package manifests verified.")
|
|
return 0
|
|
archive = OUTPUT / f"{release}.zip"
|
|
make_archive(plan, changed, archive)
|
|
prepared = remote(config, "Prepare", request)
|
|
remote_zip = prepared["archive"].replace("\\", "/")
|
|
if not re.fullmatch(r"[A-Za-z]:/[A-Za-z0-9_./ -]+", remote_zip):
|
|
raise ValueError("Unexpected remote staging path")
|
|
subprocess.run(["scp", *ssh_options(config, scp=True), str(archive),
|
|
f"{config['user']}@{config['host']}:{remote_zip}"], check=True, timeout=1800)
|
|
result = remote(config, "Publish", request)
|
|
# Persist the server commit before checking public HTTP. A failed HTTP check must never replay a commit.
|
|
receipt = OUTPUT / f"release-{release}.json"
|
|
write_json(receipt, result)
|
|
print(f"Published release={release}; receipt={receipt}")
|
|
try:
|
|
verify_public(config, plan["version"], release, plan)
|
|
except Exception as error:
|
|
raise RuntimeError(f"SERVER ALREADY PUBLISHED. Public HTTP verification failed: {error}. Receipt: {receipt}") from error
|
|
print("Verified public version.json, package manifest MD5s and any selected startup data SHA256.")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
raise SystemExit(main())
|
|
except (OSError, ValueError, RuntimeError, subprocess.SubprocessError) as error:
|
|
print(f"ERROR: {error}", file=sys.stderr)
|
|
raise SystemExit(1)
|