1481 lines
58 KiB
Python
1481 lines
58 KiB
Python
"""sortof FastAPI service.
|
|
|
|
Loopback-only (bound 127.0.0.1) HTTP API that accepts a Steam Workshop input
|
|
blob, looks up cached mod metadata in Postgres, runs mlos_sort over the
|
|
hit subset, and returns SORTOF_DATA-shaped JSON. Cache misses are queued
|
|
into download_jobs for the worker to fulfill.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import os
|
|
import time
|
|
from contextlib import asynccontextmanager
|
|
from pathlib import Path
|
|
from typing import Any, Dict, List, Optional
|
|
from uuid import UUID
|
|
|
|
import httpx
|
|
from dotenv import load_dotenv
|
|
from fastapi import FastAPI, HTTPException, Request
|
|
from fastapi.middleware.cors import CORSMiddleware
|
|
from fastapi.staticfiles import StaticFiles
|
|
from pydantic import BaseModel, Field
|
|
|
|
import adapters
|
|
import db
|
|
import expansion
|
|
import jobs
|
|
import steam
|
|
from mlos_sort import ModInfo, parse_sorting_rules, sort_mods
|
|
from parse import parse_with_collections, parse_workshop_input
|
|
|
|
# Load .env before reading SORTOF_CORS_ORIGINS so dev/manual runs (no systemd
|
|
# EnvironmentFile) still pick it up. systemd-injected env wins by default
|
|
# because load_dotenv does not override pre-set variables.
|
|
load_dotenv(Path(__file__).resolve().parent.parent / ".env")
|
|
|
|
_cors_raw = os.getenv("SORTOF_CORS_ORIGINS", "").strip()
|
|
CORS_ORIGINS = [o.strip() for o in _cors_raw.split(",") if o.strip()] or [
|
|
"http://127.0.0.1:8801",
|
|
]
|
|
|
|
MAX_IDS = 500
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s %(levelname)s %(name)s %(message)s",
|
|
)
|
|
log = logging.getLogger("sortof.api")
|
|
|
|
|
|
# Path to the human-edited PZ version-label file. Checked into the repo so
|
|
# updates are visible in git history (vs. a hidden .env). The API loads it
|
|
# at startup; bumping the labels requires a `systemctl restart sortof-api`.
|
|
_PZ_VERSIONS_PATH = Path(__file__).resolve().parent.parent / "data" / "pz_versions.json"
|
|
|
|
|
|
def _load_pz_versions() -> Dict[str, str]:
|
|
"""Read pz_versions.json and return {branch_key: human_label}. Drops
|
|
any keys starting with '_' (those are inline documentation)."""
|
|
import json
|
|
try:
|
|
raw = json.loads(_PZ_VERSIONS_PATH.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError) as e:
|
|
log.warning("pz_versions.json load failed: %s", e)
|
|
return {}
|
|
return {k: v for k, v in raw.items() if not k.startswith("_") and isinstance(v, str)}
|
|
|
|
|
|
@asynccontextmanager
|
|
async def lifespan(app: FastAPI):
|
|
pool = await db.create_pool()
|
|
http = httpx.AsyncClient(timeout=30.0)
|
|
app.state.db = pool
|
|
app.state.http = http
|
|
app.state.pending_tasks = set()
|
|
app.state.pz_versions = _load_pz_versions()
|
|
async with pool.acquire() as conn:
|
|
n_reaped = await jobs.sweep_stale_expansions(conn)
|
|
if n_reaped:
|
|
log.info("lifespan startup: reaped %d stale expansion job(s)", n_reaped)
|
|
log.info(
|
|
"sortof.api started (pz_versions: %s)",
|
|
", ".join(f"{k}={v}" for k, v in app.state.pz_versions.items()) or "(none)",
|
|
)
|
|
try:
|
|
yield
|
|
finally:
|
|
await http.aclose()
|
|
await pool.close()
|
|
log.info("sortof.api stopped")
|
|
|
|
|
|
app = FastAPI(title="sortof API", lifespan=lifespan)
|
|
|
|
app.add_middleware(
|
|
CORSMiddleware,
|
|
allow_origins=CORS_ORIGINS,
|
|
allow_credentials=False,
|
|
allow_methods=["POST", "GET", "OPTIONS"],
|
|
allow_headers=["Content-Type"],
|
|
)
|
|
|
|
|
|
class SortRequest(BaseModel):
|
|
input: str = Field(default="")
|
|
rules: Optional[str] = Field(default=None)
|
|
# Spec C §2: PZ build context. "B41" or "B42"; default "B42" when missing
|
|
# or unrecognized. Drives Rule A (build-aware default) in adapters.
|
|
pz_build: Optional[str] = Field(default=None)
|
|
|
|
|
|
class ResortRequest(BaseModel):
|
|
selected_mod_ids: List[str] = Field(default_factory=list)
|
|
# Same semantics as SortRequest.pz_build. Forwarded so resort can emit
|
|
# build-mismatch warnings (the user's pzBuild state on the frontend
|
|
# doesn't change between /api/sort and /api/resort, but resort runs in
|
|
# isolation server-side without the original job's pz_build).
|
|
pz_build: Optional[str] = Field(default=None)
|
|
# Original /api/sort input wsids (the user's "subscription set").
|
|
# Resort needs them for wsid-keyed warnings — build-mismatch et al.
|
|
# are per-wsid, but resort only knows mod_ids, and mod_id ↔ wsid
|
|
# isn't 1:1 when a mod_id has been evicted to a different wsid in the
|
|
# cache (common for B41 ↔ B42 pairs that share a mod_id).
|
|
input_wsids: List[str] = Field(default_factory=list)
|
|
|
|
|
|
def _strip_path_prefix(deps) -> List[str]:
|
|
"""Defensively normalize dep names. Strips leading backslashes (B42 path
|
|
syntax: `\\StarlitLibrary` -> `StarlitLibrary`) and trims whitespace.
|
|
Worker._split_csv applies this at parse time, but ~14 mod_parsed rows
|
|
in the live DB were written before that fix landed and will only refresh
|
|
when their wsid's time_updated advances on Steam. Normalizing here keeps
|
|
missing-dep matching correct against legacy rows."""
|
|
out: List[str] = []
|
|
for d in (deps or []):
|
|
s = (d or "").strip().lstrip("\\")
|
|
if s:
|
|
out.append(s)
|
|
return out
|
|
|
|
|
|
def _inject_addon_loadafter(mods: List[ModInfo]) -> None:
|
|
"""For each multi-mod wsid that has both addons and primaries, append the
|
|
primary's mod_id to each addon's `loadAfter`. The topo sort honours
|
|
loadAfter, so when the user explicitly ticks an addon (via the picker
|
|
+ resort), MODS_LINE places it after the primary.
|
|
|
|
Spec A's _apply_branch_rules drops the addon from MODS_LINE BY DEFAULT
|
|
(so this injection is mostly a safety net for the resort path); when
|
|
the user adds it back, ordering is preserved without any picker logic.
|
|
"""
|
|
by_wsid: Dict[str, List[ModInfo]] = {}
|
|
for m in mods:
|
|
if m.workshop_id:
|
|
by_wsid.setdefault(m.workshop_id, []).append(m)
|
|
for group in by_wsid.values():
|
|
if len(group) < 2:
|
|
continue
|
|
addons = [m for m in group if m.is_addon]
|
|
primaries = [m for m in group if not m.is_addon]
|
|
if not addons or not primaries:
|
|
continue
|
|
# Pick a single primary to anchor against. Most addon wsids have
|
|
# exactly one main mod; if there are multiple, alphabetical picks
|
|
# a stable target. The order between primaries themselves stays
|
|
# however the topo sort decides.
|
|
primary_id = sorted(p.id for p in primaries)[0]
|
|
for addon in addons:
|
|
if primary_id not in addon.loadAfter:
|
|
addon.loadAfter = list(addon.loadAfter) + [primary_id]
|
|
|
|
|
|
def _row_to_modinfo(r) -> ModInfo:
|
|
"""Build a ModInfo from a mod_parsed row. Used by both /api/sort and
|
|
/api/resort; columns must match the SELECT in both endpoints."""
|
|
return ModInfo(
|
|
id=r["mod_id"],
|
|
name=r["name"] or r["mod_id"],
|
|
workshop_id=r["workshop_id"],
|
|
category=r["category"] or "undefined",
|
|
requirements=_strip_path_prefix(r["requirements"]),
|
|
loadAfter=_strip_path_prefix(r["load_after"]),
|
|
loadBefore=_strip_path_prefix(r["load_before"]),
|
|
incompatibleMods=_strip_path_prefix(r["incompatible_mods"]),
|
|
loadFirst=r["load_first"] or "off",
|
|
loadLast=r["load_last"] or "off",
|
|
tags=list(r["tags"] or []),
|
|
maps=list(r["maps"] or []),
|
|
is_addon=bool(r["is_addon"]) if "is_addon" in r else False,
|
|
workshop_tags=list(r["workshop_tags"] or []) if "workshop_tags" in r else [],
|
|
)
|
|
|
|
|
|
def _ordered_wsids_line(payload: Dict[str, Any], all_wsids: List[str]) -> str:
|
|
"""Compose WORKSHOP_ITEMS_LINE: load-order wsids that have at least one
|
|
mod in SORTED_ORDER, then append every other input wsid (unknown,
|
|
non_mod, all-branches-deselected) in input order. Spec A §8 ownership
|
|
is preserved (set unchanged), Spec C §2 behavior is honored (order
|
|
follows the current sort).
|
|
|
|
Returns a string with trailing ';' to match the existing PZ ini convention.
|
|
"""
|
|
sorted_order = payload.get("SORTED_ORDER", []) or []
|
|
mod_to_wsid: Dict[str, str] = {
|
|
entry["modId"]: entry.get("wsid") or ""
|
|
for entry in (payload.get("MOD_DB") or [])
|
|
}
|
|
seen: set = set()
|
|
ordered: List[str] = []
|
|
for mid in sorted_order:
|
|
w = mod_to_wsid.get(mid)
|
|
if w and w not in seen:
|
|
seen.add(w)
|
|
ordered.append(w)
|
|
for w in (all_wsids or []):
|
|
if w and w not in seen:
|
|
seen.add(w)
|
|
ordered.append(w)
|
|
return ";".join(ordered) + ";" if ordered else ""
|
|
|
|
|
|
async def _lookup_wsids_for_missing(
|
|
conn,
|
|
mlos_warnings: Dict[str, Any],
|
|
pz_build: Optional[str] = None,
|
|
) -> Dict[str, str]:
|
|
"""Resolve missing-requirement mod_ids to wsids via mod_parsed cache.
|
|
|
|
Returns {mod_id -> workshop_id} for any missing dep we've previously
|
|
cached. Used to render an [add to list] action button next to the
|
|
'missing' warning. Unknown deps just get no button.
|
|
|
|
pz_build filters out wrong-build resolutions: if the user is on B42 and
|
|
the only cached wsid for `dep` is tagged `Build 41` only, we DROP the
|
|
suggestion. Adding it would just trigger a build-mismatch warning;
|
|
worse, in cases like `TacHold requires modoptions` where the author's
|
|
`require=` is over-declared, the suggestion misleads the user toward
|
|
a B41 mod that doesn't actually need to be in the load order.
|
|
"""
|
|
missing = mlos_warnings.get("missing_requirements") or {}
|
|
if not missing:
|
|
return {}
|
|
wanted: set = set()
|
|
for deps in missing.values():
|
|
for d in deps:
|
|
if d:
|
|
wanted.add(d)
|
|
if not wanted:
|
|
return {}
|
|
rows = await conn.fetch(
|
|
"""
|
|
SELECT DISTINCT ON (mp.mod_id) mp.mod_id, mp.workshop_id, wm.tags
|
|
FROM mod_parsed mp
|
|
JOIN workshop_meta wm ON wm.workshop_id = mp.workshop_id
|
|
WHERE mp.mod_id = ANY($1::text[])
|
|
ORDER BY mp.mod_id, mp.parsed_at_time_updated DESC
|
|
""",
|
|
list(wanted),
|
|
)
|
|
target_tag = "Build 42" if pz_build == "B42" else (
|
|
"Build 41" if pz_build == "B41" else None
|
|
)
|
|
other_tag = "Build 41" if target_tag == "Build 42" else (
|
|
"Build 42" if target_tag == "Build 41" else None
|
|
)
|
|
out: Dict[str, str] = {}
|
|
for r in rows:
|
|
if target_tag and other_tag:
|
|
tags = list(r["tags"] or [])
|
|
if other_tag in tags and target_tag not in tags:
|
|
# Wrong-build only — adding it would just trigger a
|
|
# build-mismatch. Skip the suggestion.
|
|
continue
|
|
out[r["mod_id"]] = r["workshop_id"]
|
|
return out
|
|
|
|
|
|
# Hand-curated build-version pairs where the B41↔B42 sibling renamed its
|
|
# mod_id (so the auto-discovery via shared mod_id can't find it). Bidirectional
|
|
# - if the user submits either side, the OTHER side is offered. Add entries as
|
|
# you encounter them; the worker's mod_id-based detection covers anything that
|
|
# kept the same mod_id across builds.
|
|
MANUAL_BUILD_PAIRS: Dict[str, str] = {
|
|
# truemusic (B41) ↔ TrueMoozic (B42). Author renamed mod_id between
|
|
# builds; B42 wsid bundles a TMMumble addon as a second mod_id.
|
|
"2613146550": "3632610172",
|
|
"3632610172": "2613146550",
|
|
}
|
|
|
|
|
|
async def _find_swap_candidate(
|
|
conn,
|
|
wsid: str,
|
|
target_tag: str,
|
|
exclude_wsids: List[str],
|
|
) -> Optional[Dict[str, str]]:
|
|
"""Find a sibling Workshop ID that publishes any of `wsid`'s mod_ids and
|
|
is tagged for `target_tag` (typically the build the user picked).
|
|
|
|
Used to attach a "swap to <other>" action onto build-mismatch warnings.
|
|
Many mod authors maintain two separate wsids — one B41, one B42 — that
|
|
share the same mod_id (e.g. tsarslib at 2392709985 and 3402491515).
|
|
Cross-referencing via mod_id finds those pairs without an explicit map.
|
|
|
|
Sources for "wsid X's mod_ids":
|
|
1. mod_parsed (current owner).
|
|
2. mod_id_conflicts left/right side (X has been an evictor or evicted
|
|
party for some mod_id, meaning it once owned that mod_id).
|
|
3. MANUAL_BUILD_PAIRS for cases where the author renamed the mod_id.
|
|
|
|
Returns {wsid, title} or None if no usable sibling found.
|
|
"""
|
|
# Manual override first: explicit cross-build pairs we curate by hand.
|
|
paired = MANUAL_BUILD_PAIRS.get(wsid)
|
|
if paired and paired not in set(exclude_wsids):
|
|
row = await conn.fetchrow(
|
|
"""
|
|
SELECT workshop_id, title FROM workshop_meta
|
|
WHERE workshop_id = $1 AND $2 = ANY(tags)
|
|
""",
|
|
paired, target_tag,
|
|
)
|
|
if row:
|
|
return {"wsid": row["workshop_id"], "title": row["title"]}
|
|
rows = await conn.fetch(
|
|
"""
|
|
WITH x_mod_ids AS (
|
|
SELECT mod_id FROM mod_parsed WHERE workshop_id = $1
|
|
UNION
|
|
SELECT mod_id FROM mod_id_conflicts
|
|
WHERE evicted_wsid = $1 OR evicting_wsid = $1
|
|
),
|
|
candidates AS (
|
|
SELECT DISTINCT workshop_id AS wsid
|
|
FROM mod_parsed
|
|
WHERE mod_id IN (SELECT mod_id FROM x_mod_ids)
|
|
AND workshop_id <> $1
|
|
UNION
|
|
SELECT DISTINCT evicting_wsid
|
|
FROM mod_id_conflicts
|
|
WHERE mod_id IN (SELECT mod_id FROM x_mod_ids)
|
|
AND evicted_wsid = $1
|
|
UNION
|
|
SELECT DISTINCT evicted_wsid
|
|
FROM mod_id_conflicts
|
|
WHERE mod_id IN (SELECT mod_id FROM x_mod_ids)
|
|
AND evicting_wsid = $1
|
|
)
|
|
SELECT c.wsid, wm.title
|
|
FROM candidates c
|
|
JOIN workshop_meta wm ON wm.workshop_id = c.wsid
|
|
WHERE wm.consumer_app_id = 108600
|
|
AND $2 = ANY(wm.tags)
|
|
AND NOT (wm.workshop_id = ANY($3::text[]))
|
|
ORDER BY wm.title
|
|
LIMIT 1
|
|
""",
|
|
wsid, target_tag, list(exclude_wsids),
|
|
)
|
|
if not rows:
|
|
return None
|
|
return {"wsid": rows[0]["wsid"], "title": rows[0]["title"]}
|
|
|
|
|
|
# Visible warning order: most actionable first. Build-mismatch leads (lets
|
|
# the user resolve build issues before chasing other warnings); next come
|
|
# the picker-driving warnings (auto-picked-branch, unmatched-addons) so
|
|
# the multi-option selection UI sits at the top of the panel; then the
|
|
# duplicate-mod_id / missing / conflict / cycle warnings.
|
|
_WARNING_TAG_PRIORITY: Dict[str, int] = {
|
|
"build-mismatch": 0,
|
|
"auto-picked-branch": 1,
|
|
"unmatched-addons": 1,
|
|
"duplicate-mod_id": 2,
|
|
"missing": 3,
|
|
"conflict": 4,
|
|
"cycle": 5,
|
|
}
|
|
|
|
|
|
def _reorder_warnings(payload: Dict[str, Any]) -> None:
|
|
"""Stable-sort `payload['WARNINGS']` by tag priority. Warnings sharing
|
|
a priority bucket keep their original relative order."""
|
|
warnings = payload.get("WARNINGS") or []
|
|
if not warnings:
|
|
return
|
|
payload["WARNINGS"] = sorted(
|
|
warnings,
|
|
key=lambda w: _WARNING_TAG_PRIORITY.get(w.get("tag", ""), 99),
|
|
)
|
|
|
|
|
|
async def _emit_build_mismatch_warnings(
|
|
conn,
|
|
payload: Dict[str, Any],
|
|
input_wsids: List[str],
|
|
pz_build: str,
|
|
) -> None:
|
|
"""Emit `build-mismatch` warnings for cached input wsids whose Steam-side
|
|
Build tags don't include the requested build.
|
|
|
|
`workshop_meta.tags` is Steam's controlled vocabulary — `Build 41` /
|
|
`Build 42` are the canonical signal. Three cases per wsid:
|
|
- both builds tagged → multi-build, no warn
|
|
- only the OPPOSITE build tagged → wrong build, emit warn
|
|
- neither build tagged → author didn't mark, can't tell, no warn
|
|
- only the TARGET build tagged → fine, no warn
|
|
|
|
Multi-branch wsids already get their own `build-mismatch` from
|
|
`adapters._apply_branch_rules` (Spec C); skip wsids with an existing
|
|
one to avoid double-warning.
|
|
"""
|
|
if not input_wsids or pz_build not in ("B41", "B42"):
|
|
return
|
|
rows = await conn.fetch(
|
|
"""
|
|
SELECT workshop_id, title, tags
|
|
FROM workshop_meta
|
|
WHERE workshop_id = ANY($1::text[])
|
|
""",
|
|
list(input_wsids),
|
|
)
|
|
target_tag = "Build 41" if pz_build == "B41" else "Build 42"
|
|
other_tag = "Build 42" if pz_build == "B41" else "Build 41"
|
|
existing = payload.get("WARNINGS") or []
|
|
already_flagged = {
|
|
w.get("wsid") for w in existing
|
|
if w.get("tag") == "build-mismatch" and w.get("wsid")
|
|
}
|
|
new_warnings: List[Dict[str, Any]] = []
|
|
for r in rows:
|
|
wsid = r["workshop_id"]
|
|
if wsid in already_flagged:
|
|
continue
|
|
tags = list(r["tags"] or [])
|
|
if target_tag in tags:
|
|
continue # supports the picked build
|
|
if other_tag not in tags:
|
|
continue # author didn't tag a build, can't tell
|
|
title = r["title"] or wsid
|
|
warn: Dict[str, Any] = {
|
|
"tag": "build-mismatch",
|
|
"level": "amber",
|
|
"msg": (
|
|
f"{title} ({wsid}) is tagged {other_tag} only — "
|
|
f"may not work with {target_tag}."
|
|
),
|
|
"wsid": wsid,
|
|
}
|
|
actions: List[Dict[str, Any]] = []
|
|
# Look for a target-build sibling that shares any of this mod's
|
|
# mod_ids (the typical "B41 wsid" + "B42 wsid" pair pattern).
|
|
swap = await _find_swap_candidate(
|
|
conn, wsid, target_tag, list(input_wsids),
|
|
)
|
|
if swap:
|
|
actions.append({
|
|
"type": "swap-wsid",
|
|
"from": wsid,
|
|
"to": swap["wsid"],
|
|
"label": f"swap to {swap['title']}",
|
|
})
|
|
# Always offer a one-click drop for wrong-build mods. Useful when no
|
|
# target-build sibling exists (the swap action is missing) or when
|
|
# the user just wants to drop it without searching for a replacement.
|
|
actions.append({
|
|
"type": "remove-wsid",
|
|
"wsid": wsid,
|
|
"label": "remove",
|
|
})
|
|
warn["actions"] = actions
|
|
new_warnings.append(warn)
|
|
if not new_warnings:
|
|
return
|
|
# Build-mismatched mods are about to be removed or swapped, so any
|
|
# `missing` warnings they sourced are noise: they'd push the user
|
|
# toward adding deps for a mod they're about to drop. Filter those
|
|
# out, but keep other warnings on the same wsid (auto-picked-branch,
|
|
# duplicate-mod_id, etc. are still meaningful).
|
|
mismatch_wsids = {w["wsid"] for w in new_warnings}
|
|
filtered = [
|
|
w for w in existing
|
|
if not (w.get("tag") == "missing" and w.get("wsid") in mismatch_wsids)
|
|
]
|
|
# Build-mismatch entries lead the list so the user resolves the
|
|
# build-version question before chasing other warnings.
|
|
payload["WARNINGS"] = new_warnings + filtered
|
|
|
|
|
|
async def _augment_with_required_items(
|
|
conn,
|
|
payload: Dict[str, Any],
|
|
input_wsids: List[str],
|
|
) -> None:
|
|
"""Mutate `payload["WARNINGS"]` in place: for each missing-dep warning
|
|
whose source mod has Steam-side Required Items in `workshop_meta`,
|
|
append `add-wsid` actions for any required wsid not already in the
|
|
user's input or already represented as an action.
|
|
|
|
Steam's anonymous GetPublishedFileDetails does NOT expose `children`
|
|
for individual mods, so the worker scrapes the public Workshop page
|
|
and stores the result in `workshop_meta.required_wsids`. This lets us
|
|
one-click resolve missing deps that aren't yet in the user's cache —
|
|
which is exactly the case the search-workshop fallback handled poorly.
|
|
"""
|
|
warnings = payload.get("WARNINGS") or []
|
|
missing = [w for w in warnings if w.get("tag") == "missing" and w.get("wsid")]
|
|
if not missing:
|
|
return
|
|
src_wsids = list({w["wsid"] for w in missing})
|
|
rows = await conn.fetch(
|
|
"""
|
|
SELECT wm.workshop_id AS src_wsid, req.req_wsid
|
|
FROM workshop_meta wm,
|
|
unnest(wm.required_wsids) AS req(req_wsid)
|
|
WHERE wm.workshop_id = ANY($1::text[])
|
|
""",
|
|
src_wsids,
|
|
)
|
|
if not rows:
|
|
return
|
|
pairs: Dict[str, List[str]] = {}
|
|
all_req: set = set()
|
|
for r in rows:
|
|
pairs.setdefault(r["src_wsid"], []).append(r["req_wsid"])
|
|
all_req.add(r["req_wsid"])
|
|
# Best-available human label for each required wsid:
|
|
# 1. mod_parsed.name (cached + parsed) — preferred
|
|
# 2. workshop_meta.title (cached, even if mod hasn't been parsed yet)
|
|
# 3. fall back to the bare wsid
|
|
label_rows = await conn.fetch(
|
|
"""
|
|
SELECT wm.workshop_id,
|
|
wm.title,
|
|
(SELECT mp.name FROM mod_parsed mp
|
|
JOIN workshop_meta wm2 ON wm2.workshop_id = mp.workshop_id
|
|
WHERE mp.workshop_id = wm.workshop_id
|
|
AND mp.parsed_at_time_updated = wm2.time_updated
|
|
ORDER BY mp.parsed_at DESC LIMIT 1) AS parsed_name,
|
|
(SELECT mp.mod_id FROM mod_parsed mp
|
|
JOIN workshop_meta wm2 ON wm2.workshop_id = mp.workshop_id
|
|
WHERE mp.workshop_id = wm.workshop_id
|
|
AND mp.parsed_at_time_updated = wm2.time_updated
|
|
ORDER BY mp.parsed_at DESC LIMIT 1) AS parsed_mod_id
|
|
FROM workshop_meta wm
|
|
WHERE wm.workshop_id = ANY($1::text[])
|
|
""",
|
|
list(all_req),
|
|
)
|
|
labels: Dict[str, Dict[str, Optional[str]]] = {
|
|
r["workshop_id"]: {
|
|
"title": r["title"],
|
|
"mod_id": r["parsed_mod_id"],
|
|
"name": r["parsed_name"],
|
|
}
|
|
for r in label_rows
|
|
}
|
|
input_set = set(input_wsids or [])
|
|
for w in missing:
|
|
existing_wsids = {
|
|
a.get("wsid") for a in (w.get("actions") or [])
|
|
if a.get("type") == "add-wsid" and a.get("wsid")
|
|
}
|
|
actions = list(w.get("actions") or [])
|
|
for req_wsid in pairs.get(w["wsid"], []):
|
|
if req_wsid in input_set or req_wsid in existing_wsids:
|
|
continue
|
|
info = labels.get(req_wsid) or {}
|
|
# Prefer the Workshop listing title (one canonical name per wsid)
|
|
# over `parsed_name` — multi-branch wsids would otherwise pick an
|
|
# arbitrary branch's name, which is confusing. mod_id is a useful
|
|
# last resort for compactness; bare wsid only when nothing exists.
|
|
label_text = (info.get("title") or info.get("name")
|
|
or info.get("mod_id") or req_wsid)
|
|
actions.append({
|
|
"type": "add-wsid",
|
|
"wsid": req_wsid,
|
|
"modId": info.get("mod_id") or "",
|
|
"label": f"add {label_text}",
|
|
})
|
|
existing_wsids.add(req_wsid)
|
|
if actions:
|
|
w["actions"] = actions
|
|
|
|
|
|
async def _lookup_mod_id_conflicts(
|
|
conn,
|
|
input_wsids: List[str],
|
|
) -> List[Dict[str, Any]]:
|
|
"""Surface mod_id ownership conflicts where the user's input includes a
|
|
wsid that lost a claim on a mod_id to another wsid.
|
|
|
|
Walks `mod_id_conflicts` (recorded by worker.py at parse-time eviction)
|
|
and filters to rows where:
|
|
- evicted_wsid is in the user's input, AND
|
|
- the evicted wsid hasn't subsequently re-claimed the mod_id (i.e., no
|
|
current mod_parsed row for (evicted_wsid, mod_id)).
|
|
|
|
Reports the *current* owner (looked up from mod_parsed) rather than the
|
|
historical evicting_wsid; multiple evictions can chain (Y→Z→W), and the
|
|
user cares about who's loaded right now. Two flavors:
|
|
- amber: current owner is also in input → mod is loaded, just not from
|
|
the wsid the user might have expected.
|
|
- red: current owner is NOT in input → mod is silently missing.
|
|
"""
|
|
if not input_wsids:
|
|
return []
|
|
rows = await conn.fetch(
|
|
"""
|
|
SELECT DISTINCT ON (mc.mod_id, mc.evicted_wsid)
|
|
mc.mod_id,
|
|
mc.evicted_wsid,
|
|
mc.evicting_wsid,
|
|
(SELECT mp.workshop_id
|
|
FROM mod_parsed mp
|
|
WHERE mp.mod_id = mc.mod_id
|
|
LIMIT 1) AS current_owner
|
|
FROM mod_id_conflicts mc
|
|
WHERE mc.evicted_wsid = ANY($1::text[])
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM mod_parsed mp
|
|
WHERE mp.workshop_id = mc.evicted_wsid
|
|
AND mp.mod_id = mc.mod_id
|
|
)
|
|
ORDER BY mc.mod_id, mc.evicted_wsid, mc.recorded_at DESC
|
|
""",
|
|
list(input_wsids),
|
|
)
|
|
if not rows:
|
|
return []
|
|
input_set = set(input_wsids)
|
|
out: List[Dict[str, Any]] = []
|
|
for r in rows:
|
|
mod_id = r["mod_id"]
|
|
evicted = r["evicted_wsid"]
|
|
# Prefer the live cache owner; fall back to the historically-evicting
|
|
# wsid if the mod_id has since been fully de-cached (e.g., manual
|
|
# force-repull recipe). Either way the user's wsid lost its claim.
|
|
owner = r["current_owner"] or r["evicting_wsid"]
|
|
if owner in input_set:
|
|
out.append({
|
|
"tag": "duplicate-mod_id",
|
|
"level": "amber",
|
|
"msg": (
|
|
f"Mod '{mod_id}' is published by both wsid {evicted} and "
|
|
f"wsid {owner}; loaded copy is from {owner}."
|
|
),
|
|
"wsid": evicted,
|
|
"mod_id": mod_id,
|
|
"evicting_wsid": owner,
|
|
})
|
|
else:
|
|
out.append({
|
|
"tag": "duplicate-mod_id",
|
|
"level": "red",
|
|
"msg": (
|
|
f"Mod '{mod_id}' is missing: wsid {evicted} declares it, "
|
|
f"but the cache holds wsid {owner}'s copy. "
|
|
f"Add {owner} to your input to load '{mod_id}'."
|
|
),
|
|
"wsid": evicted,
|
|
"mod_id": mod_id,
|
|
"evicting_wsid": owner,
|
|
})
|
|
return out
|
|
|
|
|
|
async def _build_result_for_job(
|
|
conn,
|
|
wsids: List[str],
|
|
rules_raw: Optional[str],
|
|
pz_build: Optional[str] = None,
|
|
) -> Dict[str, Any]:
|
|
"""Compute the SORTOF_DATA payload from currently-cached mod_parsed rows
|
|
for the given wsids. Used both for partial results during draining and
|
|
for the final result on phase transition to 'done'."""
|
|
if not wsids:
|
|
return _empty_payload([], "success")
|
|
rows = await conn.fetch(
|
|
"""
|
|
SELECT mp.workshop_id, mp.mod_id, mp.name, mp.category,
|
|
mp.requirements, mp.load_after, mp.load_before,
|
|
mp.incompatible_mods, mp.load_first, mp.load_last,
|
|
mp.tags, mp.maps, mp.is_addon, wm.tags AS workshop_tags
|
|
FROM mod_parsed mp
|
|
JOIN workshop_meta wm ON wm.workshop_id = mp.workshop_id
|
|
WHERE mp.workshop_id = ANY($1::text[])
|
|
AND mp.parsed_at_time_updated = wm.time_updated
|
|
ORDER BY mp.workshop_id, mp.mod_id
|
|
""",
|
|
wsids,
|
|
)
|
|
mods = [_row_to_modinfo(r) for r in rows]
|
|
rules: Dict[str, Any] = {}
|
|
if rules_raw:
|
|
try:
|
|
rules = parse_sorting_rules(rules_raw)
|
|
except Exception:
|
|
log.warning("job result: failed to parse sorting_rules")
|
|
_inject_addon_loadafter(mods)
|
|
sort_result = sort_mods(mods, rules)
|
|
cached_ids = {r["workshop_id"] for r in rows}
|
|
wsid_lookup = await _lookup_wsids_for_missing(
|
|
conn, sort_result.get("warnings", {}) or {},
|
|
pz_build=pz_build,
|
|
)
|
|
conflict_warnings = await _lookup_mod_id_conflicts(conn, wsids)
|
|
payload = adapters.build_response(
|
|
input_ids=wsids,
|
|
hit_ids=list(cached_ids),
|
|
mods=mods,
|
|
sort_result=sort_result,
|
|
status="success" if len(cached_ids) >= len(wsids) else "partial",
|
|
wsid_lookup=wsid_lookup,
|
|
pz_build=pz_build or "B42",
|
|
)
|
|
if conflict_warnings:
|
|
payload["WARNINGS"] = list(payload.get("WARNINGS") or []) + conflict_warnings
|
|
await _augment_with_required_items(conn, payload, wsids)
|
|
if pz_build:
|
|
await _emit_build_mismatch_warnings(conn, payload, wsids, pz_build)
|
|
_reorder_warnings(payload)
|
|
# Spec A §8 ownership: WORKSHOP_ITEMS_LINE preserves the SET of input
|
|
# wsids regardless of which are cached/non-mod/unknown. Within that set,
|
|
# order them by the load position of their first mod (so toggling
|
|
# branches that change wsid load position visibly reorders the line).
|
|
# Wsids without any mod in SORTED_ORDER (yet-to-cache, non_mod, unknown)
|
|
# tail at the end in their original input order.
|
|
payload["WORKSHOP_ITEMS_LINE"] = _ordered_wsids_line(payload, wsids)
|
|
# Surface terminal-failed wsids so the polling-path payload mirrors the
|
|
# sync path's unknown/non_mod arrays. Without this, jobs with broken or
|
|
# non-mod children would lack diagnostic info even after reaching `done`.
|
|
failed_rows = await conn.fetch(
|
|
"""
|
|
SELECT DISTINCT ON (workshop_id) workshop_id, status, error
|
|
FROM download_jobs
|
|
WHERE workshop_id = ANY($1::text[])
|
|
ORDER BY workshop_id, updated_at DESC
|
|
""",
|
|
wsids,
|
|
)
|
|
non_mod_set: set = set()
|
|
unknown_set: set = set()
|
|
for r in failed_rows:
|
|
if r["status"] != "failed":
|
|
continue
|
|
if r["error"] == "process_one=no_mod_info":
|
|
non_mod_set.add(r["workshop_id"])
|
|
else:
|
|
unknown_set.add(r["workshop_id"])
|
|
payload["non_mod"] = [w for w in wsids if w in non_mod_set]
|
|
payload["unknown"] = [w for w in wsids if w in unknown_set]
|
|
payload["pending"] = [
|
|
w for w in wsids
|
|
if w not in cached_ids and w not in non_mod_set and w not in unknown_set
|
|
]
|
|
return payload
|
|
|
|
|
|
def _empty_payload(input_ids: List[str], status: str) -> Dict[str, Any]:
|
|
return {
|
|
"status": status,
|
|
"MOD_DB": [],
|
|
"SORTED_ORDER": [],
|
|
"WORKSHOP_ITEMS_LINE": "",
|
|
"MODS_LINE": "",
|
|
"MAP_LINE": None,
|
|
"WARNINGS": [],
|
|
"pending": list(input_ids),
|
|
"unknown": [],
|
|
"non_mod": [],
|
|
}
|
|
|
|
|
|
@app.get("/healthz")
|
|
async def healthz() -> Dict[str, bool]:
|
|
return {"ok": True}
|
|
|
|
|
|
async def _route_to_job(
|
|
request: Request,
|
|
conn,
|
|
input_raw: str,
|
|
rules_raw: Optional[str],
|
|
bare_wsids: List[str],
|
|
collection_ids: List[str],
|
|
queue_wsids: Optional[List[str]] = None,
|
|
pz_build: Optional[str] = None,
|
|
) -> Dict[str, Any]:
|
|
"""Create a sort_jobs row and (if needed) kick off background expansion.
|
|
Returns {status, job_id} for the client to start polling.
|
|
|
|
queue_wsids: explicit subset of wsids to insert into download_jobs.
|
|
If None, every deduped bare wsid is queued (legacy default). Callers that
|
|
have already classified hit/miss MUST pass only the misses, otherwise
|
|
already-cached wsids get redundantly redownloaded.
|
|
"""
|
|
if collection_ids:
|
|
# Will resolve in the background.
|
|
job_id = await jobs.create_job(
|
|
conn,
|
|
input_raw=input_raw,
|
|
collection_ids=collection_ids,
|
|
wsids=None,
|
|
rules_raw=rules_raw,
|
|
initial_phase="expanding",
|
|
pz_build=pz_build,
|
|
)
|
|
task = asyncio.create_task(expansion.run_expansion(
|
|
request.app.state.db,
|
|
request.app.state.http,
|
|
job_id,
|
|
bare_wsids,
|
|
collection_ids,
|
|
))
|
|
request.app.state.pending_tasks.add(task)
|
|
task.add_done_callback(request.app.state.pending_tasks.discard)
|
|
log.info(
|
|
"routed to job=%s phase=expanding bare=%d collections=%d",
|
|
job_id, len(bare_wsids), len(collection_ids),
|
|
)
|
|
return {"status": "expanding", "job_id": job_id}
|
|
else:
|
|
# Bare wsids, some uncached. Dedupe; queue only the requested subset.
|
|
seen: set = set()
|
|
wsids: List[str] = []
|
|
for w in bare_wsids:
|
|
if w not in seen:
|
|
seen.add(w)
|
|
wsids.append(w)
|
|
job_id = await jobs.create_job(
|
|
conn,
|
|
input_raw=input_raw,
|
|
collection_ids=[],
|
|
wsids=wsids,
|
|
rules_raw=rules_raw,
|
|
initial_phase="queued",
|
|
pz_build=pz_build,
|
|
)
|
|
targets = list(queue_wsids) if queue_wsids is not None else wsids
|
|
for wid in targets:
|
|
async with conn.transaction():
|
|
existing = await conn.fetchval(
|
|
"SELECT 1 FROM download_jobs "
|
|
"WHERE workshop_id = $1 AND status IN ('queued','downloading') LIMIT 1",
|
|
wid,
|
|
)
|
|
if existing is None:
|
|
await conn.execute(
|
|
"INSERT INTO download_jobs (workshop_id, status) VALUES ($1, 'queued')",
|
|
wid,
|
|
)
|
|
log.info(
|
|
"routed to job=%s phase=queued wsids=%d targets=%d",
|
|
job_id, len(wsids), len(targets),
|
|
)
|
|
return {"status": "queued", "job_id": job_id}
|
|
|
|
|
|
@app.post("/api/sort")
|
|
async def sort_endpoint(req: SortRequest, request: Request) -> Dict[str, Any]:
|
|
t0 = time.monotonic()
|
|
bare_wsids, collection_ids = parse_with_collections(req.input or "")
|
|
input_ids = bare_wsids # backwards-compat alias for the existing sync flow
|
|
if not bare_wsids and not collection_ids:
|
|
raise HTTPException(status_code=400, detail="no workshop ids found in input")
|
|
# NOTE: bound is on user-input cardinality; a collection that resolves
|
|
# to >MAX_IDS children is allowed (post-expansion cap is out of scope).
|
|
if len(bare_wsids) + len(collection_ids) > MAX_IDS:
|
|
raise HTTPException(
|
|
status_code=413,
|
|
detail=f"too many workshop ids ({len(bare_wsids) + len(collection_ids)} > {MAX_IDS})",
|
|
)
|
|
|
|
log.info(
|
|
"sort received bare=%d collections=%d",
|
|
len(bare_wsids), len(collection_ids),
|
|
)
|
|
|
|
# ── B+F: collections present → route to async job immediately, no Steam call.
|
|
# expansion.run_expansion handles GetCollectionDetails + cache + drain queueing.
|
|
if collection_ids:
|
|
pool = request.app.state.db
|
|
async with pool.acquire() as conn:
|
|
return await _route_to_job(
|
|
request, conn, req.input or "", req.rules,
|
|
bare_wsids, collection_ids,
|
|
pz_build=req.pz_build or "B42",
|
|
)
|
|
|
|
http: httpx.AsyncClient = request.app.state.http
|
|
try:
|
|
steam_details = await steam.fetch_workshop_details(http, input_ids)
|
|
except httpx.HTTPError as e:
|
|
log.warning("steam api error: %s", e)
|
|
elapsed_ms = int((time.monotonic() - t0) * 1000)
|
|
log.info(
|
|
"sort done hits=0 misses=%d status=error ms=%d",
|
|
len(input_ids), elapsed_ms,
|
|
)
|
|
return _empty_payload(input_ids, "error")
|
|
|
|
pool = request.app.state.db
|
|
hit_ids: List[str] = []
|
|
miss_ids: List[str] = []
|
|
unknown_ids: List[str] = [] # Steam returned result != 1 (deleted, typo'd, private)
|
|
non_mod_ids: List[str] = [] # Steam confirmed but DD found no mod.info (collection, art, etc.)
|
|
valid_steam: Dict[str, dict] = {}
|
|
|
|
async with pool.acquire() as conn:
|
|
for wid in input_ids:
|
|
d = steam_details.get(wid)
|
|
if not d or d.get("result") != 1:
|
|
unknown_ids.append(wid)
|
|
miss_ids.append(wid)
|
|
continue
|
|
valid_steam[wid] = d
|
|
tu = int(d.get("time_updated", 0))
|
|
row = await conn.fetchrow(
|
|
"SELECT 1 FROM mod_parsed "
|
|
"WHERE workshop_id = $1 AND parsed_at_time_updated = $2 LIMIT 1",
|
|
wid, tu,
|
|
)
|
|
if row is not None:
|
|
hit_ids.append(wid)
|
|
else:
|
|
miss_ids.append(wid)
|
|
|
|
# Detect non-mod items: Steam-valid wsids whose most-recent download_jobs
|
|
# row failed with the no_mod_info reason (drain confirmed DD pulled no
|
|
# parseable .info file). Surfaces collections, art, etc., separately
|
|
# from real "still being indexed" pending items.
|
|
valid_miss_ids = [w for w in miss_ids if w in valid_steam]
|
|
if valid_miss_ids:
|
|
non_mod_rows = await conn.fetch(
|
|
"""
|
|
SELECT DISTINCT ON (workshop_id) workshop_id, status, error
|
|
FROM download_jobs
|
|
WHERE workshop_id = ANY($1::text[])
|
|
ORDER BY workshop_id, updated_at DESC
|
|
""",
|
|
valid_miss_ids,
|
|
)
|
|
for r in non_mod_rows:
|
|
if r["status"] == "failed" and r["error"] == "process_one=no_mod_info":
|
|
non_mod_ids.append(r["workshop_id"])
|
|
|
|
# ── B+F: route to async job if any wsid needs drain time.
|
|
# The job's GET endpoint will surface unknown/non_mod via its own
|
|
# response shape; the sync path remains for fully-cached input.
|
|
# `hit_ids` only contains wsids confirmed cached; anything else
|
|
# (unknown OR uncached-but-valid) is reason to switch to a job.
|
|
if len(hit_ids) < len(bare_wsids):
|
|
# Match the sync path's queue predicate: skip Steam-unknown wsids
|
|
# AND skip wsids the drain has already classified as non-mods.
|
|
# Without this filter, every cache miss including known-bad ones
|
|
# gets queued, plus any cached wsids upstream of an unknown would
|
|
# also get re-queued (the bug we're fixing).
|
|
non_mod_set_route = set(non_mod_ids)
|
|
queue_targets = [
|
|
w for w in miss_ids
|
|
if w in valid_steam and w not in non_mod_set_route
|
|
]
|
|
return await _route_to_job(
|
|
request, conn, req.input or "", req.rules,
|
|
bare_wsids, [], # collection_ids empty - already short-circuited above
|
|
queue_wsids=queue_targets,
|
|
pz_build=req.pz_build or "B42",
|
|
)
|
|
|
|
# Queue cache misses (only those Steam confirmed exist AND aren't
|
|
# already known to be non-mods - re-queueing those is pointless).
|
|
non_mod_set = set(non_mod_ids)
|
|
for wid in miss_ids:
|
|
if wid not in valid_steam:
|
|
log.info("skip queue for unknown wsid=%s", wid)
|
|
continue
|
|
if wid in non_mod_set:
|
|
log.info("skip queue for non-mod wsid=%s", wid)
|
|
continue
|
|
async with conn.transaction():
|
|
existing = await conn.fetchval(
|
|
"SELECT 1 FROM download_jobs "
|
|
"WHERE workshop_id = $1 AND status IN ('queued','downloading') "
|
|
"LIMIT 1",
|
|
wid,
|
|
)
|
|
if existing is None:
|
|
await conn.execute(
|
|
"INSERT INTO download_jobs (workshop_id, status) "
|
|
"VALUES ($1, 'queued')",
|
|
wid,
|
|
)
|
|
|
|
if hit_ids:
|
|
rows = await conn.fetch(
|
|
"""
|
|
SELECT mp.workshop_id, mp.mod_id, mp.name, mp.category,
|
|
mp.requirements, mp.load_after, mp.load_before,
|
|
mp.incompatible_mods, mp.load_first, mp.load_last,
|
|
mp.tags, mp.maps, mp.is_addon, wm.tags AS workshop_tags
|
|
FROM mod_parsed mp
|
|
JOIN workshop_meta wm ON wm.workshop_id = mp.workshop_id
|
|
WHERE mp.workshop_id = ANY($1::text[])
|
|
AND mp.parsed_at_time_updated = wm.time_updated
|
|
ORDER BY mp.workshop_id, mp.mod_id
|
|
""",
|
|
hit_ids,
|
|
)
|
|
else:
|
|
rows = []
|
|
|
|
mods: List[ModInfo] = [_row_to_modinfo(r) for r in rows]
|
|
|
|
rules: Dict[str, Any] = {}
|
|
if req.rules:
|
|
try:
|
|
rules = parse_sorting_rules(req.rules)
|
|
except Exception:
|
|
log.warning("failed to parse sorting_rules; ignoring")
|
|
rules = {}
|
|
|
|
_inject_addon_loadafter(mods)
|
|
sort_result = sort_mods(mods, rules)
|
|
|
|
if not hit_ids:
|
|
status = "cold"
|
|
elif len(hit_ids) == len(input_ids):
|
|
status = "success"
|
|
else:
|
|
status = "partial"
|
|
|
|
async with pool.acquire() as conn:
|
|
wsid_lookup = await _lookup_wsids_for_missing(
|
|
conn, sort_result.get("warnings", {}) or {},
|
|
pz_build=req.pz_build or "B42",
|
|
)
|
|
conflict_warnings = await _lookup_mod_id_conflicts(conn, input_ids)
|
|
payload = adapters.build_response(
|
|
input_ids=input_ids,
|
|
hit_ids=hit_ids,
|
|
mods=mods,
|
|
sort_result=sort_result,
|
|
status=status,
|
|
wsid_lookup=wsid_lookup,
|
|
pz_build=req.pz_build or "B42",
|
|
)
|
|
if conflict_warnings:
|
|
payload["WARNINGS"] = list(payload.get("WARNINGS") or []) + conflict_warnings
|
|
async with pool.acquire() as conn:
|
|
await _augment_with_required_items(conn, payload, input_ids)
|
|
await _emit_build_mismatch_warnings(
|
|
conn, payload, input_ids, req.pz_build or "B42",
|
|
)
|
|
_reorder_warnings(payload)
|
|
# Surface ghost / non-mod IDs separately from real pending so the UI can
|
|
# distinguish "indexing 2 mods" from "2 of your IDs are deleted/typo'd"
|
|
# and "1 of your IDs is a collection, not a mod."
|
|
payload["unknown"] = unknown_ids
|
|
payload["non_mod"] = non_mod_ids
|
|
# Trim them out of pending so each ID lives in exactly one category.
|
|
excluded = set(unknown_ids) | set(non_mod_ids)
|
|
payload["pending"] = [w for w in payload.get("pending", []) if w not in excluded]
|
|
# Spec A §8 + Spec C §2: load-order wsids that landed in the sort, then
|
|
# tail of unknown/non_mod (preserves the user's subscription set). Without
|
|
# this the sync path's WORKSHOP_ITEMS_LINE would silently drop them.
|
|
payload["WORKSHOP_ITEMS_LINE"] = _ordered_wsids_line(payload, input_ids)
|
|
|
|
elapsed_ms = int((time.monotonic() - t0) * 1000)
|
|
log.info(
|
|
"sort done hits=%d misses=%d unknown=%d non_mod=%d status=%s ms=%d",
|
|
len(hit_ids), len(miss_ids), len(unknown_ids), len(non_mod_ids), status, elapsed_ms,
|
|
)
|
|
return payload
|
|
|
|
|
|
def _merge_warnings(seeded: Optional[Dict[str, Any]], fresh: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Merge any pre-seeded WARNINGS (e.g., 'collection-partial' from expansion)
|
|
into a freshly-computed result payload. Seeded warnings come first to
|
|
preserve "this happened earlier" ordering. Dedupes by (tag, msg)."""
|
|
seeded_warnings = (seeded or {}).get("WARNINGS") or []
|
|
fresh_warnings = fresh.get("WARNINGS") or []
|
|
if not seeded_warnings:
|
|
return fresh
|
|
seen = set()
|
|
merged: List[Dict[str, Any]] = []
|
|
for w in list(seeded_warnings) + list(fresh_warnings):
|
|
key = (w.get("tag", ""), w.get("msg", ""))
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
merged.append(w)
|
|
fresh["WARNINGS"] = merged
|
|
return fresh
|
|
|
|
|
|
@app.get("/api/jobs/{job_id}")
|
|
async def get_job_endpoint(job_id: str, request: Request) -> Dict[str, Any]:
|
|
pool = request.app.state.db
|
|
async with pool.acquire() as conn:
|
|
row = await jobs.get_job_row(conn, job_id)
|
|
if row is None:
|
|
raise HTTPException(status_code=404, detail="job not found or expired")
|
|
wsids = list(row["wsids"]) if row["wsids"] else None
|
|
counts = await jobs.compute_counts(conn, wsids or [])
|
|
phase = jobs.derive_phase(row["phase"], wsids, counts)
|
|
|
|
result_json = row["result_json"]
|
|
job_pz_build = row.get("pz_build")
|
|
if phase == "done" and row["phase"] != "done":
|
|
# Job just completed. Compute fresh and persist, but preserve
|
|
# any expansion-seeded warnings (e.g., 'collection-partial').
|
|
fresh = await _build_result_for_job(
|
|
conn, wsids, row["rules_raw"], pz_build=job_pz_build,
|
|
)
|
|
result_json = _merge_warnings(row["result_json"], fresh)
|
|
await jobs.update_phase(
|
|
conn, job_id, "done", result_json=result_json,
|
|
)
|
|
elif phase != "done" and wsids:
|
|
# Mid-flight partial: compute fresh on every poll, also merging
|
|
# in any seeded warnings so the client sees them throughout.
|
|
# Don't persist; only `done` transitions write result_json.
|
|
fresh = await _build_result_for_job(
|
|
conn, wsids, row["rules_raw"], pz_build=job_pz_build,
|
|
)
|
|
result_json = _merge_warnings(row["result_json"], fresh)
|
|
|
|
# `counts` is {cached, queued, draining, terminal_failed} - additive
|
|
# extension of the Spec B+F §6 contract. JS clients ignore the extra key.
|
|
return {
|
|
"job_id": str(row["job_id"]),
|
|
"phase": phase,
|
|
"counts": counts,
|
|
"wsids": wsids,
|
|
"result": result_json,
|
|
"failure_reason": row["failure_reason"],
|
|
}
|
|
|
|
|
|
@app.delete("/api/jobs/{job_id}", status_code=204)
|
|
async def delete_job_endpoint(job_id: str, request: Request):
|
|
"""Cancel a job. Idempotent: cancelling a terminal job is a no-op 204.
|
|
Does NOT touch download_jobs (Spec B+F §8)."""
|
|
try:
|
|
uid = UUID(job_id)
|
|
except ValueError:
|
|
raise HTTPException(status_code=404, detail="job not found")
|
|
pool = request.app.state.db
|
|
async with pool.acquire() as conn:
|
|
# Single SQL: race-safe - a no-op when the row is missing OR terminal.
|
|
result = await conn.execute(
|
|
"""
|
|
UPDATE sort_jobs
|
|
SET phase = 'failed',
|
|
failure_reason = 'cancelled',
|
|
phase_started_at = now()
|
|
WHERE job_id = $1
|
|
AND phase NOT IN ('done', 'failed')
|
|
""",
|
|
uid,
|
|
)
|
|
# If 0 rows updated, either the row didn't exist OR was already terminal.
|
|
# Disambiguate so we still return 404 for nonexistent jobs.
|
|
if result == "UPDATE 0":
|
|
exists = await conn.fetchval(
|
|
"SELECT 1 FROM sort_jobs WHERE job_id = $1", uid,
|
|
)
|
|
if exists is None:
|
|
raise HTTPException(status_code=404, detail="job not found")
|
|
log.info("delete_job %s -> failed/cancelled (or no-op terminal)", job_id)
|
|
return None
|
|
|
|
|
|
@app.post("/api/resort")
|
|
async def resort_endpoint(req: ResortRequest, request: Request) -> Dict[str, Any]:
|
|
"""Re-run mlos_sort against a user-supplied subset of mod_ids.
|
|
|
|
Stateless. No Steam round-trip. Filters mod_parsed by the requested
|
|
mod_ids, runs sort_mods, returns the same shape as /api/sort.
|
|
"""
|
|
t0 = time.monotonic()
|
|
ids = list(req.selected_mod_ids or [])
|
|
|
|
if not ids:
|
|
raise HTTPException(status_code=400, detail="selected_mod_ids must be non-empty")
|
|
if len(ids) > MAX_IDS:
|
|
raise HTTPException(status_code=413, detail=f"too many mod_ids ({len(ids)} > {MAX_IDS})")
|
|
for i, mod_id in enumerate(ids):
|
|
if not isinstance(mod_id, str) or not (1 <= len(mod_id) <= 256):
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail=f"invalid mod_id at index {i}: must be a string of length 1-256",
|
|
)
|
|
|
|
pool = request.app.state.db
|
|
async with pool.acquire() as conn:
|
|
# Fetch ALL mod_ids of any wsid that has at least one selected mod_id.
|
|
# _apply_branch_rules needs the full group context to keep emitting
|
|
# auto-picked-branch warnings (the picker UI is data-driven from
|
|
# those warnings). Without the siblings, narrowing a multi-branch
|
|
# wsid to one branch would silently hide the picker.
|
|
rows = await conn.fetch(
|
|
"""
|
|
WITH selected_wsids AS (
|
|
SELECT DISTINCT mp.workshop_id
|
|
FROM mod_parsed mp
|
|
JOIN workshop_meta wm ON wm.workshop_id = mp.workshop_id
|
|
WHERE mp.mod_id = ANY($1::text[])
|
|
AND mp.parsed_at_time_updated = wm.time_updated
|
|
)
|
|
SELECT mp.workshop_id, mp.mod_id, mp.name, mp.category,
|
|
mp.requirements, mp.load_after, mp.load_before,
|
|
mp.incompatible_mods, mp.load_first, mp.load_last,
|
|
mp.tags, mp.maps, mp.is_addon, wm.tags AS workshop_tags
|
|
FROM mod_parsed mp
|
|
JOIN workshop_meta wm ON wm.workshop_id = mp.workshop_id
|
|
WHERE mp.workshop_id IN (SELECT workshop_id FROM selected_wsids)
|
|
AND mp.parsed_at_time_updated = wm.time_updated
|
|
ORDER BY mp.workshop_id, mp.mod_id
|
|
""",
|
|
ids,
|
|
)
|
|
|
|
selected_set = set(ids)
|
|
found_selected = {r["mod_id"] for r in rows if r["mod_id"] in selected_set}
|
|
missing = [mid for mid in ids if mid not in found_selected]
|
|
if missing:
|
|
log.info("resort dropped unknown mod_ids count=%d sample=%s",
|
|
len(missing), missing[:3])
|
|
if not rows:
|
|
raise HTTPException(status_code=400, detail="no known mod_ids in selection")
|
|
|
|
all_mods: List[ModInfo] = [_row_to_modinfo(r) for r in rows]
|
|
selected_mods: List[ModInfo] = [m for m in all_mods if m.id in selected_set]
|
|
_inject_addon_loadafter(selected_mods)
|
|
# Sort over the user's explicit selection so SORTED_ORDER reflects only
|
|
# what they ticked. _apply_branch_rules below still sees the full group
|
|
# via `mods=all_mods`, so it can emit picker-driving warnings.
|
|
sort_result = sort_mods(selected_mods, {})
|
|
# Per spec review #12: silently drop unknown mod_ids and return status="success".
|
|
# The frontend reconciles its sent list against MOD_DB to detect drops.
|
|
async with pool.acquire() as conn:
|
|
wsid_lookup = await _lookup_wsids_for_missing(
|
|
conn, sort_result.get("warnings", {}) or {},
|
|
pz_build=req.pz_build or "B42",
|
|
)
|
|
payload = adapters.build_response(
|
|
input_ids=[],
|
|
hit_ids=ids,
|
|
mods=all_mods,
|
|
sort_result=sort_result,
|
|
status="success",
|
|
wsid_lookup=wsid_lookup,
|
|
pz_build=req.pz_build or "B42",
|
|
is_resort=True,
|
|
selected_modids=selected_set,
|
|
)
|
|
# Resort operates on a known mod_id set - there's no concept of a
|
|
# "still-being-fetched" wsid here. With input_ids=[] above, build_response
|
|
# already produces pending=[]; this is explicit defense against future
|
|
# build_response changes.
|
|
payload["pending"] = []
|
|
# The frontend owns unknown/non_mod from the original /api/sort response
|
|
# for the lifetime of the result set (same ownership contract as
|
|
# WORKSHOP_ITEMS_LINE, see sortContextRef). Resort returns empty arrays;
|
|
# the client overrides them.
|
|
payload["unknown"] = []
|
|
payload["non_mod"] = []
|
|
# Emit the same secondary warnings /api/sort and the polling path do.
|
|
# Without these, every resort (e.g., branch toggle) wipes build-mismatch
|
|
# / duplicate-mod_id / required-item augmentations from the warnings
|
|
# panel — the user-visible "warnings disappear" bug.
|
|
# Prefer the original sort's input wsids (the user's subscription set)
|
|
# for wsid-keyed warnings. Mod_id-derived wsids omit B41 wsids whose
|
|
# mod_id has since been evicted to a B42 sibling, dropping legitimate
|
|
# build-mismatch warnings the user expects to keep seeing across resort.
|
|
derived_wsids = {m.workshop_id for m in selected_mods if m.workshop_id}
|
|
resort_wsids = list({*(req.input_wsids or []), *derived_wsids})
|
|
if resort_wsids:
|
|
async with pool.acquire() as conn:
|
|
conflict_warnings = await _lookup_mod_id_conflicts(conn, resort_wsids)
|
|
await _augment_with_required_items(conn, payload, resort_wsids)
|
|
if req.pz_build:
|
|
await _emit_build_mismatch_warnings(
|
|
conn, payload, resort_wsids, req.pz_build,
|
|
)
|
|
if conflict_warnings:
|
|
payload["WARNINGS"] = list(payload.get("WARNINGS") or []) + conflict_warnings
|
|
_reorder_warnings(payload)
|
|
|
|
elapsed_ms = int((time.monotonic() - t0) * 1000)
|
|
log.info(
|
|
"resort done count=%d dropped=%d ms=%d",
|
|
len(rows), len(missing), elapsed_ms,
|
|
)
|
|
return payload
|
|
|
|
|
|
# ── Broken-mod report board ────────────────────────────────────────────────
|
|
|
|
|
|
class BrokenModReportRequest(BaseModel):
|
|
workshop_id: str = Field(..., min_length=1, max_length=32)
|
|
version: str = Field(..., min_length=1, max_length=32)
|
|
|
|
|
|
class BrokenModVoteRequest(BaseModel):
|
|
direction: str = Field(..., pattern=r"^(up|down)$")
|
|
|
|
|
|
@app.get("/api/pz-versions")
|
|
async def pz_versions_endpoint(request: Request) -> Dict[str, str]:
|
|
"""Branch-key → human-readable build label. Source: data/pz_versions.json
|
|
loaded at lifespan startup. Frontend dropdown reads from here."""
|
|
return dict(request.app.state.pz_versions or {})
|
|
|
|
|
|
@app.get("/api/broken-mods")
|
|
async def list_broken_mods(
|
|
request: Request,
|
|
q: Optional[str] = None,
|
|
limit: int = 100,
|
|
) -> List[Dict[str, Any]]:
|
|
"""List reports newest-first. `q` filters by wsid substring."""
|
|
limit = max(1, min(int(limit or 100), 500))
|
|
pool = request.app.state.db
|
|
async with pool.acquire() as conn:
|
|
if q:
|
|
rows = await conn.fetch(
|
|
"""
|
|
SELECT id, workshop_id, mod_name, version,
|
|
upvotes, downvotes,
|
|
created_at, updated_at
|
|
FROM broken_mod_reports
|
|
WHERE workshop_id LIKE '%' || $1 || '%'
|
|
ORDER BY updated_at DESC
|
|
LIMIT $2
|
|
""",
|
|
q.strip(), limit,
|
|
)
|
|
else:
|
|
rows = await conn.fetch(
|
|
"""
|
|
SELECT id, workshop_id, mod_name, version,
|
|
upvotes, downvotes,
|
|
created_at, updated_at
|
|
FROM broken_mod_reports
|
|
ORDER BY updated_at DESC
|
|
LIMIT $1
|
|
""",
|
|
limit,
|
|
)
|
|
return [
|
|
{
|
|
"id": int(r["id"]),
|
|
"workshop_id": r["workshop_id"],
|
|
"mod_name": r["mod_name"] or "",
|
|
"version": r["version"],
|
|
"upvotes": int(r["upvotes"]),
|
|
"downvotes": int(r["downvotes"]),
|
|
"created_at": r["created_at"].isoformat(),
|
|
"updated_at": r["updated_at"].isoformat(),
|
|
}
|
|
for r in rows
|
|
]
|
|
|
|
|
|
@app.post("/api/broken-mods")
|
|
async def create_broken_mod_report(
|
|
req: BrokenModReportRequest,
|
|
request: Request,
|
|
) -> Dict[str, Any]:
|
|
"""Create or refresh a broken-mod report. Re-submitting the same
|
|
(workshop_id, version) bumps `updated_at` and preserves vote counts.
|
|
|
|
Resolves `mod_name` from the existing workshop_meta cache; if missing
|
|
we fetch from Steam (anonymous GetPublishedFileDetails) so the user
|
|
gets a meaningful label without having to type it.
|
|
"""
|
|
if not req.workshop_id.isdigit():
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail="workshop_id must be a numeric Steam Workshop ID",
|
|
)
|
|
valid_versions = set(request.app.state.pz_versions or {}) or {
|
|
"stable", "unstable", "outdatedunstable",
|
|
}
|
|
if req.version not in valid_versions:
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail=f"version must be one of: {sorted(valid_versions)}",
|
|
)
|
|
|
|
pool = request.app.state.db
|
|
mod_name: Optional[str] = None
|
|
async with pool.acquire() as conn:
|
|
row = await conn.fetchrow(
|
|
"SELECT title FROM workshop_meta WHERE workshop_id = $1",
|
|
req.workshop_id,
|
|
)
|
|
if row and row["title"]:
|
|
mod_name = row["title"]
|
|
|
|
if not mod_name:
|
|
# Cache miss → ask Steam directly. Treat any failure as "unknown
|
|
# name" rather than hard-erroring; the user still gets a row.
|
|
try:
|
|
details = await steam.fetch_workshop_details(
|
|
request.app.state.http, [req.workshop_id],
|
|
)
|
|
d = details.get(req.workshop_id) or {}
|
|
if d.get("result") == 1:
|
|
mod_name = d.get("title") or None
|
|
except Exception as e:
|
|
log.info("broken-mod: Steam lookup for %s failed: %s",
|
|
req.workshop_id, e)
|
|
|
|
async with pool.acquire() as conn:
|
|
row = await conn.fetchrow(
|
|
"""
|
|
INSERT INTO broken_mod_reports
|
|
(workshop_id, mod_name, version)
|
|
VALUES ($1, $2, $3)
|
|
ON CONFLICT (workshop_id, version) DO UPDATE
|
|
SET updated_at = now(),
|
|
mod_name = COALESCE(EXCLUDED.mod_name, broken_mod_reports.mod_name)
|
|
RETURNING id, workshop_id, mod_name, version,
|
|
upvotes, downvotes, created_at, updated_at
|
|
""",
|
|
req.workshop_id, mod_name, req.version,
|
|
)
|
|
return {
|
|
"id": int(row["id"]),
|
|
"workshop_id": row["workshop_id"],
|
|
"mod_name": row["mod_name"] or "",
|
|
"version": row["version"],
|
|
"upvotes": int(row["upvotes"]),
|
|
"downvotes": int(row["downvotes"]),
|
|
"created_at": row["created_at"].isoformat(),
|
|
"updated_at": row["updated_at"].isoformat(),
|
|
}
|
|
|
|
|
|
@app.post("/api/broken-mods/{report_id}/vote")
|
|
async def vote_broken_mod(
|
|
report_id: int,
|
|
req: BrokenModVoteRequest,
|
|
request: Request,
|
|
) -> Dict[str, int]:
|
|
"""Increment upvotes or downvotes on an existing report."""
|
|
column = "upvotes" if req.direction == "up" else "downvotes"
|
|
pool = request.app.state.db
|
|
async with pool.acquire() as conn:
|
|
row = await conn.fetchrow(
|
|
f"""
|
|
UPDATE broken_mod_reports
|
|
SET {column} = {column} + 1
|
|
WHERE id = $1
|
|
RETURNING upvotes, downvotes
|
|
""",
|
|
report_id,
|
|
)
|
|
if row is None:
|
|
raise HTTPException(status_code=404, detail="report not found")
|
|
return {"upvotes": int(row["upvotes"]), "downvotes": int(row["downvotes"])}
|
|
|
|
|
|
# ── static frontend ────────────────────────────────────────────────────────
|
|
# Mount LAST so all API routes win path resolution.
|
|
_FRONTEND_DIR = Path(__file__).resolve().parent.parent / "frontend"
|
|
if _FRONTEND_DIR.is_dir():
|
|
app.mount(
|
|
"/",
|
|
StaticFiles(directory=str(_FRONTEND_DIR), html=True),
|
|
name="frontend",
|
|
)
|