309 lines
10 KiB
Python
309 lines
10 KiB
Python
"""Persistent SQLite-backed conversion job store with WAL mode and spooling."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from app.schemas.convert import JobStatus
|
|
|
|
from contextlib import contextmanager
|
|
|
|
_DEFAULT_SPOOL_THRESHOLD = 512 * 1024 # 512 KB
|
|
|
|
|
|
class JobStore:
|
|
"""Persistent SQLite job store with WAL mode, disk spooling, and crash recovery."""
|
|
|
|
def __init__(
|
|
self,
|
|
ttl_seconds: int = 3 * 60 * 60,
|
|
db_path: str | Path | None = None,
|
|
spool_dir: str | Path | None = None,
|
|
):
|
|
self._lock = threading.RLock()
|
|
self._ttl = ttl_seconds
|
|
|
|
base_data = Path(__file__).resolve().parents[3] / "data"
|
|
if db_path is None:
|
|
db_path = os.environ.get("CONVERT_JOBS_DB", str(base_data / "jobs.sqlite3"))
|
|
if spool_dir is None:
|
|
spool_dir = os.environ.get("CONVERT_JOBS_SPOOL", str(base_data / "jobs_spool"))
|
|
|
|
self._db_path = Path(db_path)
|
|
self._spool_dir = Path(spool_dir)
|
|
|
|
self._db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
self._spool_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
self._init_db()
|
|
self._recover_crashed_jobs()
|
|
|
|
@contextmanager
|
|
def _conn(self):
|
|
conn = sqlite3.connect(
|
|
str(self._db_path),
|
|
timeout=30.0,
|
|
check_same_thread=False,
|
|
)
|
|
conn.row_factory = sqlite3.Row
|
|
try:
|
|
with conn:
|
|
yield conn
|
|
finally:
|
|
try:
|
|
conn.close()
|
|
except Exception:
|
|
pass
|
|
|
|
def _init_db(self) -> None:
|
|
with self._lock, self._conn() as conn:
|
|
conn.execute("PRAGMA journal_mode=WAL;")
|
|
conn.execute("PRAGMA synchronous=NORMAL;")
|
|
conn.execute(
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS jobs (
|
|
job_id TEXT PRIMARY KEY,
|
|
status TEXT NOT NULL,
|
|
source TEXT NOT NULL,
|
|
target TEXT NOT NULL,
|
|
filename TEXT NOT NULL,
|
|
fidelity TEXT,
|
|
warnings TEXT,
|
|
size_bytes INTEGER,
|
|
error TEXT,
|
|
result_path TEXT,
|
|
result_blob BLOB,
|
|
result_filename TEXT,
|
|
download_path TEXT,
|
|
created_at REAL NOT NULL,
|
|
updated_at REAL NOT NULL,
|
|
meta TEXT,
|
|
cancel_requested INTEGER DEFAULT 0
|
|
);
|
|
"""
|
|
)
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_jobs_created_at ON jobs (created_at);")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs (status);")
|
|
|
|
def _recover_crashed_jobs(self) -> None:
|
|
"""Mark any jobs left queued or running from a crashed server as failed."""
|
|
with self._lock, self._conn() as conn:
|
|
conn.execute(
|
|
"""
|
|
UPDATE jobs
|
|
SET status = ?, error = ?, updated_at = ?
|
|
WHERE status IN (?, ?)
|
|
""",
|
|
(
|
|
JobStatus.failed.value,
|
|
"Server process restarted during execution",
|
|
time.time(),
|
|
JobStatus.queued.value,
|
|
JobStatus.running.value,
|
|
),
|
|
)
|
|
|
|
def create(self, source: str, target: str, filename: str) -> dict[str, Any]:
|
|
self._purge_expired()
|
|
job_id = str(uuid.uuid4())
|
|
now = time.time()
|
|
job = {
|
|
"job_id": job_id,
|
|
"status": JobStatus.queued.value,
|
|
"source": source,
|
|
"target": target,
|
|
"filename": filename,
|
|
"fidelity": None,
|
|
"warnings": [],
|
|
"size_bytes": None,
|
|
"error": None,
|
|
"result_bytes": None,
|
|
"result_filename": None,
|
|
"download_path": None,
|
|
"created_at": now,
|
|
"updated_at": now,
|
|
"meta": {},
|
|
"cancel_requested": False,
|
|
}
|
|
with self._lock, self._conn() as conn:
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO jobs (
|
|
job_id, status, source, target, filename, fidelity, warnings,
|
|
size_bytes, error, result_path, result_blob, result_filename,
|
|
download_path, created_at, updated_at, meta, cancel_requested
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
(
|
|
job_id,
|
|
job["status"],
|
|
source,
|
|
target,
|
|
filename,
|
|
None,
|
|
json.dumps([]),
|
|
None,
|
|
None,
|
|
None,
|
|
None,
|
|
None,
|
|
None,
|
|
now,
|
|
now,
|
|
json.dumps({}),
|
|
0,
|
|
),
|
|
)
|
|
return job
|
|
|
|
def update(self, job_id: str, **fields: Any) -> dict[str, Any] | None:
|
|
with self._lock:
|
|
existing = self.get(job_id)
|
|
if not existing:
|
|
return None
|
|
|
|
now = time.time()
|
|
set_clauses = ["updated_at = ?"]
|
|
params: list[Any] = [now]
|
|
|
|
# Spool large result bytes to disk
|
|
if "result_bytes" in fields:
|
|
rb = fields["result_bytes"]
|
|
if rb is not None and len(rb) > _DEFAULT_SPOOL_THRESHOLD:
|
|
spool_file = self._spool_dir / f"{job_id}.bin"
|
|
spool_file.write_bytes(rb)
|
|
set_clauses.extend(["result_path = ?", "result_blob = ?"])
|
|
params.extend([str(spool_file), None])
|
|
elif rb is not None:
|
|
set_clauses.extend(["result_blob = ?", "result_path = ?"])
|
|
params.extend([rb, None])
|
|
else:
|
|
set_clauses.extend(["result_blob = ?", "result_path = ?"])
|
|
params.extend([None, None])
|
|
|
|
for k, v in fields.items():
|
|
if k in ("result_bytes", "updated_at"):
|
|
continue
|
|
if k == "warnings":
|
|
set_clauses.append("warnings = ?")
|
|
params.append(json.dumps(v if isinstance(v, list) else []))
|
|
elif k == "meta":
|
|
set_clauses.append("meta = ?")
|
|
params.append(json.dumps(v if isinstance(v, dict) else {}))
|
|
elif k == "cancel_requested":
|
|
set_clauses.append("cancel_requested = ?")
|
|
params.append(1 if v else 0)
|
|
elif k in (
|
|
"status",
|
|
"fidelity",
|
|
"size_bytes",
|
|
"error",
|
|
"result_filename",
|
|
"download_path",
|
|
):
|
|
set_clauses.append(f"{k} = ?")
|
|
params.append(v)
|
|
|
|
params.append(job_id)
|
|
query = f"UPDATE jobs SET {', '.join(set_clauses)} WHERE job_id = ?"
|
|
with self._conn() as conn:
|
|
conn.execute(query, params)
|
|
|
|
return self.get(job_id)
|
|
|
|
def request_cancel(self, job_id: str) -> None:
|
|
"""Soft-cancel: checked between pages when cancel_check is wired."""
|
|
with self._lock, self._conn() as conn:
|
|
conn.execute(
|
|
"UPDATE jobs SET cancel_requested = 1, updated_at = ? WHERE job_id = ?",
|
|
(time.time(), job_id),
|
|
)
|
|
|
|
def is_cancelled(self, job_id: str) -> bool:
|
|
with self._lock, self._conn() as conn:
|
|
cur = conn.execute("SELECT cancel_requested FROM jobs WHERE job_id = ?", (job_id,))
|
|
row = cur.fetchone()
|
|
return bool(row and row["cancel_requested"])
|
|
|
|
def get(self, job_id: str) -> dict[str, Any] | None:
|
|
self._purge_expired()
|
|
with self._lock, self._conn() as conn:
|
|
cur = conn.execute("SELECT * FROM jobs WHERE job_id = ?", (job_id,))
|
|
row = cur.fetchone()
|
|
if not row:
|
|
return None
|
|
return self._row_to_dict(row)
|
|
|
|
def get_result_bytes(self, job_id: str) -> bytes | None:
|
|
with self._lock, self._conn() as conn:
|
|
cur = conn.execute(
|
|
"SELECT result_blob, result_path FROM jobs WHERE job_id = ?",
|
|
(job_id,),
|
|
)
|
|
row = cur.fetchone()
|
|
if not row:
|
|
return None
|
|
if row["result_path"] and Path(row["result_path"]).is_file():
|
|
try:
|
|
return Path(row["result_path"]).read_bytes()
|
|
except OSError:
|
|
return None
|
|
return row["result_blob"]
|
|
|
|
def _row_to_dict(self, row: sqlite3.Row) -> dict[str, Any]:
|
|
warnings = []
|
|
if row["warnings"]:
|
|
try:
|
|
warnings = json.loads(row["warnings"])
|
|
except Exception:
|
|
warnings = []
|
|
|
|
meta = {}
|
|
if row["meta"]:
|
|
try:
|
|
meta = json.loads(row["meta"])
|
|
except Exception:
|
|
meta = {}
|
|
|
|
return {
|
|
"job_id": row["job_id"],
|
|
"status": row["status"],
|
|
"source": row["source"],
|
|
"target": row["target"],
|
|
"filename": row["filename"],
|
|
"fidelity": row["fidelity"],
|
|
"warnings": warnings,
|
|
"size_bytes": row["size_bytes"],
|
|
"error": row["error"],
|
|
"result_filename": row["result_filename"],
|
|
"download_path": row["download_path"],
|
|
"created_at": row["created_at"],
|
|
"updated_at": row["updated_at"],
|
|
"meta": meta,
|
|
"cancel_requested": bool(row["cancel_requested"]),
|
|
}
|
|
|
|
def _purge_expired(self) -> None:
|
|
now = time.time()
|
|
cutoff = now - self._ttl
|
|
with self._lock, self._conn() as conn:
|
|
cur = conn.execute(
|
|
"SELECT job_id, result_path FROM jobs WHERE created_at < ?",
|
|
(cutoff,),
|
|
)
|
|
expired = cur.fetchall()
|
|
for row in expired:
|
|
if row["result_path"]:
|
|
Path(row["result_path"]).unlink(missing_ok=True)
|
|
conn.execute("DELETE FROM jobs WHERE created_at < ?", (cutoff,))
|
|
|
|
|
|
job_store = JobStore()
|