upload
This commit is contained in:
Binary file not shown.
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
|
||||
def test_apply_command(args):
|
||||
print("DEBUG: test_apply_command started", flush=True)
|
||||
print(f"Args: {args}", flush=True)
|
||||
print("DEBUG: test_apply_command finished", flush=True)
|
||||
|
||||
def main():
|
||||
print("DEBUG: Starting main()", flush=True)
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
subparsers = parser.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
apply_parser = subparsers.add_parser("apply")
|
||||
apply_parser.add_argument("--env")
|
||||
apply_parser.add_argument("--kind")
|
||||
apply_parser.add_argument("--db")
|
||||
apply_parser.add_argument("--root", action="append")
|
||||
apply_parser.add_argument("--manage-nfo", action="store_true")
|
||||
apply_parser.add_argument("--dry-run", action="store_true")
|
||||
apply_parser.add_argument("--live-on-miss", action="store_true")
|
||||
|
||||
print("DEBUG: Parser built", flush=True)
|
||||
args = parser.parse_args()
|
||||
print(f"DEBUG: Args parsed - cmd: {args.cmd}", flush=True)
|
||||
|
||||
if args.cmd == "apply":
|
||||
print("DEBUG: Calling test_apply_command", flush=True)
|
||||
test_apply_command(args)
|
||||
print("DEBUG: test_apply_command returned", flush=True)
|
||||
else:
|
||||
parser.error("unknown command")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,32 @@
|
||||
nohup: ignoring input
|
||||
[2025-08-24 10:00:18] Found 14669 media file(s) under /mnt/unionfs/Media/TV
|
||||
[2025-08-24 10:00:19] Progress: 500/14669 (updated: 500, misses: 0)
|
||||
[2025-08-24 10:00:19] Progress: 1000/14669 (updated: 1000, misses: 0)
|
||||
[2025-08-24 10:00:20] Progress: 1500/14669 (updated: 1500, misses: 0)
|
||||
[2025-08-24 10:00:20] Progress: 2000/14669 (updated: 2000, misses: 0)
|
||||
[2025-08-24 10:00:20] Progress: 2500/14669 (updated: 2500, misses: 0)
|
||||
[2025-08-24 10:00:21] Progress: 3000/14669 (updated: 3000, misses: 0)
|
||||
[2025-08-24 10:00:21] Progress: 3500/14669 (updated: 3500, misses: 0)
|
||||
[2025-08-24 10:00:22] Progress: 4000/14669 (updated: 4000, misses: 0)
|
||||
[2025-08-24 10:00:22] Progress: 4500/14669 (updated: 4500, misses: 0)
|
||||
[2025-08-24 10:00:23] Progress: 5000/14669 (updated: 5000, misses: 0)
|
||||
[2025-08-24 10:00:23] Progress: 5500/14669 (updated: 5500, misses: 0)
|
||||
[2025-08-24 10:00:23] Progress: 6000/14669 (updated: 6000, misses: 0)
|
||||
[2025-08-24 10:00:24] Progress: 6500/14669 (updated: 6500, misses: 0)
|
||||
[2025-08-24 10:00:24] Progress: 7000/14669 (updated: 7000, misses: 0)
|
||||
[2025-08-24 10:00:25] Progress: 7500/14669 (updated: 7500, misses: 0)
|
||||
[2025-08-24 10:00:25] Progress: 8000/14669 (updated: 8000, misses: 0)
|
||||
[2025-08-24 10:00:26] Progress: 8500/14669 (updated: 8500, misses: 0)
|
||||
[2025-08-24 10:00:26] Progress: 9000/14669 (updated: 9000, misses: 0)
|
||||
[2025-08-24 10:00:27] Progress: 9500/14669 (updated: 9500, misses: 0)
|
||||
[2025-08-24 10:00:27] Progress: 10000/14669 (updated: 10000, misses: 0)
|
||||
[2025-08-24 10:00:28] Progress: 10500/14669 (updated: 10500, misses: 0)
|
||||
[2025-08-24 10:00:28] Progress: 11000/14669 (updated: 11000, misses: 0)
|
||||
[2025-08-24 10:00:29] Progress: 11500/14669 (updated: 11500, misses: 0)
|
||||
[2025-08-24 10:00:29] Progress: 12000/14669 (updated: 12000, misses: 0)
|
||||
[2025-08-24 10:00:30] Progress: 12500/14669 (updated: 12500, misses: 0)
|
||||
[2025-08-24 10:00:30] Progress: 13000/14669 (updated: 13000, misses: 0)
|
||||
[2025-08-24 10:00:31] Progress: 13500/14669 (updated: 13500, misses: 0)
|
||||
[2025-08-24 10:00:31] Progress: 14000/14669 (updated: 14000, misses: 0)
|
||||
[2025-08-24 10:00:32] Progress: 14500/14669 (updated: 14500, misses: 0)
|
||||
[2025-08-24 10:00:32] Done. kind=tv checked=14669 updated=14669 misses=0 dry_run=False
|
||||
File diff suppressed because it is too large
Load Diff
+1124
File diff suppressed because it is too large
Load Diff
Executable
+860
@@ -0,0 +1,860 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
Media Date Cache:
|
||||
- Caches and applies mtimes for TV episodes (Sonarr) and Movies (Radarr)
|
||||
- First preference: FIRST import date from *history*
|
||||
- Fallbacks: TV -> episode airDateUtc ; Movies -> digitalRelease or physicalRelease
|
||||
- Works from DB cache; with --live-on-miss it queries Sonarr/Radarr when missing, inserts, and applies immediately
|
||||
- Multi-root aware via .env (TV_ROOTS, MOVIE_ROOTS) and/or --root
|
||||
- Gotify progress notifications supported
|
||||
|
||||
Only dependency: requests
|
||||
"""
|
||||
|
||||
import os, re, sys, time, json, sqlite3, argparse, stat, traceback
|
||||
from datetime import datetime, timezone
|
||||
from typing import Dict, Any, List, Optional, Tuple
|
||||
|
||||
try:
|
||||
import requests
|
||||
except ImportError:
|
||||
print("ERROR: 'requests' is required. Activate your venv and run: pip install requests", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# -----------------------------
|
||||
# Utilities / ENV
|
||||
# -----------------------------
|
||||
|
||||
ENV_DEFAULT_PATH = os.path.join(os.path.dirname(__file__), ".env")
|
||||
ISO_FMT = "%Y-%m-%dT%H:%M:%S%z"
|
||||
|
||||
def now_iso() -> str:
|
||||
return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
def log(msg: str):
|
||||
print(f"[{now_iso()}] {msg}", flush=True)
|
||||
|
||||
def load_env(dotenv_path: str = ENV_DEFAULT_PATH) -> Dict[str, str]:
|
||||
"""Simple .env loader with inline comment stripping."""
|
||||
env = dict(os.environ)
|
||||
if os.path.exists(dotenv_path):
|
||||
with open(dotenv_path, "r", encoding="utf-8", errors="ignore") as f:
|
||||
for raw in f:
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
if "=" not in line:
|
||||
continue
|
||||
k, v = line.split("=", 1)
|
||||
k = k.strip()
|
||||
v = v.strip()
|
||||
# Remove inline comments (# ...) unless inside quotes
|
||||
quoted = (v.startswith("'") and v.endswith("'")) or (v.startswith('"') and v.endswith('"'))
|
||||
if not quoted and "#" in v:
|
||||
v = v.split("#", 1)[0].strip()
|
||||
if quoted:
|
||||
v = v[1:-1]
|
||||
env[k] = v
|
||||
return env
|
||||
|
||||
def env_get(env: Dict[str, str], key: str, default: Optional[str] = None) -> Optional[str]:
|
||||
return env.get(key, default)
|
||||
|
||||
def env_get_int(env: Dict[str, str], key: str, default: int) -> int:
|
||||
try:
|
||||
val = env.get(key, None)
|
||||
if val is None:
|
||||
return default
|
||||
return int(str(val).split("#", 1)[0].strip())
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
# -----------------------------
|
||||
# Gotify
|
||||
# -----------------------------
|
||||
|
||||
def gotify_send(env: Dict[str, str], title: str, message: str, priority: Optional[int] = None):
|
||||
url = env_get(env, "GOTIFY_URL", "")
|
||||
token = env_get(env, "GOTIFY_TOKEN", "")
|
||||
if not url or not token:
|
||||
return
|
||||
prio = priority if priority is not None else env_get_int(env, "GOTIFY_PRIORITY", 5)
|
||||
try:
|
||||
r = requests.post(
|
||||
url.rstrip("/") + "/message",
|
||||
data={"title": title, "message": message, "priority": prio},
|
||||
headers={"X-Gotify-Key": token},
|
||||
timeout=15,
|
||||
)
|
||||
r.raise_for_status()
|
||||
except Exception as e:
|
||||
# Never crash on Gotify issues
|
||||
log(f"WARN: gotify failed: {e}")
|
||||
|
||||
# -----------------------------
|
||||
# DB
|
||||
# -----------------------------
|
||||
|
||||
SCHEMA_SQL = """
|
||||
PRAGMA journal_mode=WAL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS files (
|
||||
path TEXT PRIMARY KEY,
|
||||
ts INTEGER NOT NULL,
|
||||
source TEXT NOT NULL, -- history|airdate|digital|physical|manual
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
-- Optional per-episode cache (Sonarr)
|
||||
CREATE TABLE IF NOT EXISTS episodes (
|
||||
series_id INTEGER NOT NULL,
|
||||
season INTEGER NOT NULL,
|
||||
episode INTEGER NOT NULL,
|
||||
import_ts INTEGER,
|
||||
airdate_ts INTEGER,
|
||||
PRIMARY KEY(series_id, season, episode)
|
||||
);
|
||||
|
||||
-- Optional per-movie cache (Radarr)
|
||||
CREATE TABLE IF NOT EXISTS movies (
|
||||
movie_id INTEGER PRIMARY KEY,
|
||||
import_ts INTEGER,
|
||||
digital_ts INTEGER,
|
||||
physical_ts INTEGER
|
||||
);
|
||||
"""
|
||||
|
||||
def db_open(db_path: str) -> sqlite3.Connection:
|
||||
os.makedirs(os.path.dirname(db_path), exist_ok=True)
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
conn.executescript(SCHEMA_SQL)
|
||||
return conn
|
||||
|
||||
def db_upsert_file(conn: sqlite3.Connection, path: str, ts: int, source: str):
|
||||
conn.execute(
|
||||
"INSERT INTO files(path, ts, source, updated_at) VALUES(?,?,?,?) "
|
||||
"ON CONFLICT(path) DO UPDATE SET ts=excluded.ts, source=excluded.source, updated_at=excluded.updated_at",
|
||||
(path, ts, source, int(time.time())),
|
||||
)
|
||||
|
||||
def db_get_file_ts(conn: sqlite3.Connection, path: str) -> Optional[Tuple[int, str]]:
|
||||
cur = conn.execute("SELECT ts, source FROM files WHERE path=?", (path,))
|
||||
row = cur.fetchone()
|
||||
return (row[0], row[1]) if row else None
|
||||
|
||||
def db_upsert_episode(conn: sqlite3.Connection, series_id: int, season: int, episode: int, import_ts: Optional[int], airdate_ts: Optional[int]):
|
||||
conn.execute(
|
||||
"INSERT INTO episodes(series_id, season, episode, import_ts, airdate_ts) VALUES(?,?,?,?,?) "
|
||||
"ON CONFLICT(series_id, season, episode) DO UPDATE SET import_ts=COALESCE(excluded.import_ts, episodes.import_ts), airdate_ts=COALESCE(excluded.airdate_ts, episodes.airdate_ts)",
|
||||
(series_id, season, episode, import_ts, airdate_ts),
|
||||
)
|
||||
|
||||
def db_get_episode(conn: sqlite3.Connection, series_id: int, season: int, episode: int) -> Optional[Tuple[Optional[int], Optional[int]]]:
|
||||
cur = conn.execute(
|
||||
"SELECT import_ts, airdate_ts FROM episodes WHERE series_id=? AND season=? AND episode=?",
|
||||
(series_id, season, episode),
|
||||
)
|
||||
return cur.fetchone()
|
||||
|
||||
def db_upsert_movie(conn: sqlite3.Connection, movie_id: int, import_ts: Optional[int], digital_ts: Optional[int], physical_ts: Optional[int]):
|
||||
conn.execute(
|
||||
"INSERT INTO movies(movie_id, import_ts, digital_ts, physical_ts) VALUES(?,?,?,?) "
|
||||
"ON CONFLICT(movie_id) DO UPDATE SET import_ts=COALESCE(excluded.import_ts, movies.import_ts), digital_ts=COALESCE(excluded.digital_ts, movies.digital_ts), physical_ts=COALESCE(excluded.physical_ts, movies.physical_ts)",
|
||||
(movie_id, import_ts, digital_ts, physical_ts),
|
||||
)
|
||||
|
||||
def db_get_movie(conn: sqlite3.Connection, movie_id: int) -> Optional[Tuple[Optional[int], Optional[int], Optional[int]]]:
|
||||
cur = conn.execute("SELECT import_ts, digital_ts, physical_ts FROM movies WHERE movie_id=?", (movie_id,))
|
||||
return cur.fetchone()
|
||||
|
||||
# -----------------------------
|
||||
# Sonarr / Radarr clients
|
||||
# -----------------------------
|
||||
|
||||
class BaseArr:
|
||||
def __init__(self, base: str, api_key: str):
|
||||
self.base = base.rstrip("/")
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update({"X-Api-Key": api_key})
|
||||
|
||||
def _get(self, path: str, params: Dict[str, Any] = None):
|
||||
url = self.base + path
|
||||
r = self.session.get(url, params=params or {}, timeout=60)
|
||||
r.raise_for_status()
|
||||
try:
|
||||
return r.json()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
class Sonarr(BaseArr):
|
||||
# Endpoints: v3
|
||||
def all_series(self) -> List[Dict[str, Any]]:
|
||||
data = self._get("/api/v3/series")
|
||||
return data if isinstance(data, list) else []
|
||||
|
||||
def get_series(self, series_id: int) -> Optional[Dict[str, Any]]:
|
||||
data = self._get(f"/api/v3/series/{series_id}")
|
||||
return data if isinstance(data, dict) else None
|
||||
|
||||
def episodes_for_series(self, series_id: int) -> List[Dict[str, Any]]:
|
||||
# Returns list
|
||||
data = self._get("/api/v3/episode", params={"seriesId": series_id})
|
||||
return data if isinstance(data, list) else []
|
||||
|
||||
def history_page(self, page: int, page_size: int = 250, series_id: Optional[int] = None) -> Tuple[List[Dict[str, Any]], int]:
|
||||
params = {
|
||||
"page": page,
|
||||
"pageSize": page_size,
|
||||
"sortDirection": "ascending",
|
||||
"includeEpisode": "true",
|
||||
"includeSeries": "true",
|
||||
}
|
||||
if series_id:
|
||||
params["seriesId"] = series_id
|
||||
data = self._get("/api/v3/history", params=params)
|
||||
# Sonarr returns dict with records/totalRecords
|
||||
if isinstance(data, dict):
|
||||
recs = data.get("records") or data.get("Records") or []
|
||||
total = int(data.get("totalRecords") or data.get("TotalRecords") or 0)
|
||||
return recs, total
|
||||
# Fallback: maybe a raw list (rare)
|
||||
if isinstance(data, list):
|
||||
return data, len(data)
|
||||
return [], 0
|
||||
|
||||
class Radarr(BaseArr):
|
||||
def all_movies(self) -> List[Dict[str, Any]]:
|
||||
data = self._get("/api/v3/movie")
|
||||
return data if isinstance(data, list) else []
|
||||
|
||||
def history_page(self, page: int, page_size: int = 250, movie_id: Optional[int] = None) -> Tuple[List[Dict[str, Any]], int]:
|
||||
params = {
|
||||
"page": page,
|
||||
"pageSize": page_size,
|
||||
"sortDirection": "ascending",
|
||||
"includeMovie": "true",
|
||||
}
|
||||
if movie_id:
|
||||
params["movieId"] = movie_id
|
||||
data = self._get("/api/v3/history", params=params)
|
||||
if isinstance(data, dict):
|
||||
recs = data.get("records") or []
|
||||
total = int(data.get("totalRecords") or 0)
|
||||
return recs, total
|
||||
if isinstance(data, list):
|
||||
return data, len(data)
|
||||
return [], 0
|
||||
|
||||
# -----------------------------
|
||||
# Parsing helpers
|
||||
# -----------------------------
|
||||
|
||||
IMDB_RE = re.compile(r"\bimdb-(tt\d+)\b", re.IGNORECASE)
|
||||
SXXEYY_RE = re.compile(r"[Ss](\d{1,2})[ ._-]*[Ee](\d{2})(?:[ ._-]*[Ee-](\d{2}))?")
|
||||
SEASON_DIR_RE = re.compile(r"[Ss]eason[ _-]*(\d+)")
|
||||
YEAR_PAREN_RE = re.compile(r"\((\d{4})\)")
|
||||
|
||||
def parse_imdb(folder: str) -> Optional[str]:
|
||||
m = IMDB_RE.search(folder)
|
||||
return m.group(1).lower() if m else None
|
||||
|
||||
def parse_series_season_episode(path: str) -> Optional[Tuple[int, List[int]]]:
|
||||
# Try filename first
|
||||
fn = os.path.basename(path)
|
||||
m = SXXEYY_RE.search(fn)
|
||||
if m:
|
||||
season = int(m.group(1))
|
||||
e1 = int(m.group(2))
|
||||
e2 = m.group(3)
|
||||
eps = [e1] if not e2 else list(range(e1, int(e2) + 1))
|
||||
return season, eps
|
||||
# Try parent directory (rare)
|
||||
parts = path.split(os.sep)
|
||||
for part in reversed(parts):
|
||||
m2 = SXXEYY_RE.search(part)
|
||||
if m2:
|
||||
season = int(m2.group(1))
|
||||
e1 = int(m2.group(2))
|
||||
e2 = m2.group(3)
|
||||
eps = [e1] if not e2 else list(range(e1, int(e2) + 1))
|
||||
return season, eps
|
||||
return None
|
||||
|
||||
def extract_series_folder(path: str) -> str:
|
||||
# e.g. /.../TV/tv/Friends (1994) [imdb-tt0108778]/Season 05/...
|
||||
# series folder is the directory directly under tv/tv6
|
||||
parts = path.split(os.sep)
|
||||
# Find 'tv' or 'tv6' and take the next component
|
||||
for i, p in enumerate(parts):
|
||||
if p in ("tv", "tv6"):
|
||||
if i + 1 < len(parts):
|
||||
return parts[i + 1]
|
||||
# Fallback: take grandparent of the file
|
||||
return os.path.basename(os.path.dirname(os.path.dirname(path)))
|
||||
|
||||
def extract_title_and_year(series_folder: str) -> Tuple[str, Optional[int]]:
|
||||
# "Friends (1994) [imdb-tt0108778]" -> ("Friends", 1994)
|
||||
year = None
|
||||
m = YEAR_PAREN_RE.search(series_folder)
|
||||
if m:
|
||||
try:
|
||||
year = int(m.group(1))
|
||||
except Exception:
|
||||
year = None
|
||||
# title is everything before "(YYYY"
|
||||
if "(" in series_folder:
|
||||
title = series_folder.split("(")[0].strip()
|
||||
else:
|
||||
title = series_folder
|
||||
return title, year
|
||||
|
||||
def to_epoch(ts_str: str) -> Optional[int]:
|
||||
if not ts_str:
|
||||
return None
|
||||
try:
|
||||
# Normalize to UTC if no timezone provided
|
||||
if ts_str.endswith("Z"):
|
||||
dt = datetime.fromisoformat(ts_str.replace("Z", "+00:00"))
|
||||
else:
|
||||
# Sonarr/Radarr dates are iso8601 with Z; but just in case:
|
||||
dt = datetime.fromisoformat(ts_str if "+" in ts_str or ts_str.endswith("Z") else ts_str + "+00:00")
|
||||
return int(dt.timestamp())
|
||||
except Exception:
|
||||
# Try a few loose formats
|
||||
try:
|
||||
dt = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%S.%fZ")
|
||||
return int(dt.replace(tzinfo=timezone.utc).timestamp())
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def earliest_valid(*vals: Optional[int]) -> Optional[int]:
|
||||
xs = [v for v in vals if isinstance(v, int) and v > 0]
|
||||
return min(xs) if xs else None
|
||||
|
||||
# -----------------------------
|
||||
# Root resolution
|
||||
# -----------------------------
|
||||
|
||||
def split_paths(s: str) -> List[str]:
|
||||
# split on ':' or ',' and keep existing directories only
|
||||
raw = [p.strip() for p in re.split(r"[:,]", s) if p.strip()]
|
||||
return [p for p in raw if os.path.isdir(p)]
|
||||
|
||||
def resolve_roots(kind: str, root_arg: Optional[str], env: Dict[str, str]) -> List[str]:
|
||||
roots: List[str] = []
|
||||
if kind == "tv":
|
||||
if root_arg and os.path.isdir(root_arg):
|
||||
roots.append(root_arg)
|
||||
tv_env = env_get(env, "TV_ROOTS", "")
|
||||
if tv_env:
|
||||
roots.extend(split_paths(tv_env))
|
||||
# If root points to /Media/TV, include known subfolders if present
|
||||
more = []
|
||||
for r in list(roots):
|
||||
if os.path.isdir(os.path.join(r, "tv")):
|
||||
more.append(os.path.join(r, "tv"))
|
||||
if os.path.isdir(os.path.join(r, "tv6")):
|
||||
more.append(os.path.join(r, "tv6"))
|
||||
roots.extend(more)
|
||||
elif kind == "movie":
|
||||
if root_arg and os.path.isdir(root_arg):
|
||||
roots.append(root_arg)
|
||||
mv_env = env_get(env, "MOVIE_ROOTS", "")
|
||||
if mv_env:
|
||||
roots.extend(split_paths(mv_env))
|
||||
more = []
|
||||
for r in list(roots):
|
||||
if os.path.isdir(os.path.join(r, "movies")):
|
||||
more.append(os.path.join(r, "movies"))
|
||||
if os.path.isdir(os.path.join(r, "movies6")):
|
||||
more.append(os.path.join(r, "movies6"))
|
||||
roots.extend(more)
|
||||
# Dedup while preserving order:
|
||||
seen = set()
|
||||
dedup = []
|
||||
for p in roots:
|
||||
if p not in seen:
|
||||
seen.add(p)
|
||||
dedup.append(p)
|
||||
return dedup or ([root_arg] if root_arg else [])
|
||||
|
||||
def iter_media_files(roots: List[str], kind: str):
|
||||
exts = {".mkv", ".mp4", ".avi", ".m4v", ".mov"}
|
||||
for base in roots:
|
||||
if not base or not os.path.isdir(base):
|
||||
continue
|
||||
for dirpath, _, filenames in os.walk(base):
|
||||
for fn in filenames:
|
||||
if os.path.splitext(fn)[1].lower() in exts:
|
||||
yield os.path.join(dirpath, fn)
|
||||
|
||||
# -----------------------------
|
||||
# Build caches from Sonarr/Radarr
|
||||
# -----------------------------
|
||||
|
||||
def build_sonarr_maps(env: Dict[str, str]) -> Tuple[Sonarr, Dict[int, Dict[str, Any]], Dict[str, int], Dict[Tuple[int, int, int], int], Dict[int, Dict[Tuple[int, int], Dict[str, Any]]]]:
|
||||
"""
|
||||
Returns:
|
||||
client,
|
||||
series_by_id,
|
||||
series_by_imdb (imdbId -> seriesId),
|
||||
earliest_import_by_episode_id,
|
||||
episodes_map_by_series (seriesId -> {(season,episode): episode_obj})
|
||||
"""
|
||||
url = env_get(env, "SONARR_URL", "")
|
||||
key = env_get(env, "SONARR_API_KEY", "")
|
||||
if not url or not key:
|
||||
raise RuntimeError("SONARR_URL/SONARR_API_KEY not set in environment or .env")
|
||||
client = Sonarr(url, key)
|
||||
|
||||
series_list = client.all_series()
|
||||
series_by_id: Dict[int, Dict[str, Any]] = {}
|
||||
series_by_imdb: Dict[str, int] = {}
|
||||
for s in series_list:
|
||||
sid = int(s.get("id"))
|
||||
series_by_id[sid] = s
|
||||
imdb = (s.get("imdbId") or "").lower().strip()
|
||||
if imdb:
|
||||
series_by_imdb[imdb] = sid
|
||||
|
||||
# Episodes per series
|
||||
episodes_map_by_series: Dict[int, Dict[Tuple[int, int], Dict[str, Any]]] = {}
|
||||
for sid in series_by_id.keys():
|
||||
eps = client.episodes_for_series(sid)
|
||||
m: Dict[Tuple[int, int], Dict[str, Any]] = {}
|
||||
for e in eps:
|
||||
key = (int(e.get("seasonNumber", -1)), int(e.get("episodeNumber", -1)))
|
||||
m[key] = e
|
||||
episodes_map_by_series[sid] = m
|
||||
|
||||
# Build earliest import per episode via paged history
|
||||
earliest_import_by_episode_id: Dict[Tuple[int], int] = {}
|
||||
page = 1
|
||||
page_size = 250
|
||||
total = None
|
||||
while True:
|
||||
recs, tot = client.history_page(page, page_size)
|
||||
if total is None:
|
||||
total = tot
|
||||
if not recs:
|
||||
break
|
||||
for r in recs:
|
||||
et = str(r.get("eventType") or "").lower()
|
||||
if "import" not in et:
|
||||
continue
|
||||
ep = r.get("episode") or {}
|
||||
eid = ep.get("id")
|
||||
date = to_epoch(r.get("date"))
|
||||
if eid and date:
|
||||
cur = earliest_import_by_episode_id.get(eid)
|
||||
earliest_import_by_episode_id[eid] = min(cur, date) if cur else date
|
||||
if len(recs) < page_size:
|
||||
break
|
||||
page += 1
|
||||
|
||||
return client, series_by_id, series_by_imdb, earliest_import_by_episode_id, episodes_map_by_series
|
||||
|
||||
def build_radarr_maps(env: Dict[str, str]) -> Tuple[Radarr, Dict[int, Dict[str, Any]], Dict[str, int], Dict[int, int], Dict[int, Tuple[Optional[int], Optional[int]]]]:
|
||||
"""
|
||||
Returns:
|
||||
client,
|
||||
movies_by_id,
|
||||
movies_by_imdb (imdbId -> movieId),
|
||||
earliest_import_by_movie_id,
|
||||
releases_by_movie_id (digital_ts, physical_ts)
|
||||
"""
|
||||
url = env_get(env, "RADARR_URL", "")
|
||||
key = env_get(env, "RADARR_API_KEY", "")
|
||||
if not url or not key:
|
||||
raise RuntimeError("RADARR_URL/RADARR_API_KEY not set in environment or .env")
|
||||
client = Radarr(url, key)
|
||||
|
||||
movies = client.all_movies()
|
||||
movies_by_id: Dict[int, Dict[str, Any]] = {}
|
||||
movies_by_imdb: Dict[str, int] = {}
|
||||
releases_by_movie_id: Dict[int, Tuple[Optional[int], Optional[int]]] = {}
|
||||
|
||||
for m in movies:
|
||||
mid = int(m.get("id"))
|
||||
movies_by_id[mid] = m
|
||||
imdb = (m.get("imdbId") or "").lower().strip()
|
||||
if imdb:
|
||||
movies_by_imdb[imdb] = mid
|
||||
# known release fields
|
||||
digital = to_epoch(m.get("digitalRelease") or m.get("digitalReleaseDate") or m.get("inCinemas"))
|
||||
physical = to_epoch(m.get("physicalRelease") or m.get("physicalReleaseDate"))
|
||||
releases_by_movie_id[mid] = (digital, physical)
|
||||
|
||||
# History earliest import by movie
|
||||
earliest_by_movie: Dict[int, int] = {}
|
||||
page = 1
|
||||
page_size = 250
|
||||
total = None
|
||||
while True:
|
||||
recs, tot = client.history_page(page, page_size)
|
||||
if total is None:
|
||||
total = tot
|
||||
if not recs:
|
||||
break
|
||||
for r in recs:
|
||||
et = str(r.get("eventType") or "").lower()
|
||||
if "import" not in et:
|
||||
continue
|
||||
movie = r.get("movie") or {}
|
||||
mid = movie.get("id")
|
||||
date = to_epoch(r.get("date"))
|
||||
if mid and date:
|
||||
cur = earliest_by_movie.get(mid)
|
||||
earliest_by_movie[mid] = min(cur, date) if cur else date
|
||||
if len(recs) < page_size:
|
||||
break
|
||||
page += 1
|
||||
|
||||
return client, movies_by_id, movies_by_imdb, earliest_by_movie, releases_by_movie_id
|
||||
|
||||
# -----------------------------
|
||||
# Core: get preferred ts per file
|
||||
# -----------------------------
|
||||
|
||||
def tv_preferred_ts_for_file(
|
||||
path: str,
|
||||
conn: sqlite3.Connection,
|
||||
sonarr_pack: Tuple[Sonarr, Dict[int, Dict[str, Any]], Dict[str, int], Dict[int, int], Dict[int, Dict[Tuple[int, int], Dict[str, Any]]]],
|
||||
live_insert_db: bool = True,
|
||||
) -> Optional[Tuple[int, str]]:
|
||||
client, series_by_id, series_by_imdb, earliest_import_by_episode_id, episodes_map_by_series = sonarr_pack
|
||||
|
||||
series_folder = extract_series_folder(path)
|
||||
imdb = parse_imdb(series_folder)
|
||||
title, year = extract_title_and_year(series_folder)
|
||||
|
||||
# Get seriesId
|
||||
series_id = None
|
||||
if imdb and imdb in series_by_imdb:
|
||||
series_id = series_by_imdb[imdb]
|
||||
else:
|
||||
# fallback: match by (title, year)
|
||||
cand = []
|
||||
tnorm = title.strip().lower()
|
||||
for sid, s in series_by_id.items():
|
||||
stitle = str(s.get("title") or "").strip().lower()
|
||||
syear = s.get("year")
|
||||
if stitle == tnorm and (year is None or syear == year):
|
||||
cand.append(sid)
|
||||
if cand:
|
||||
series_id = cand[0]
|
||||
if not series_id:
|
||||
return None
|
||||
|
||||
# Parse SxxEyy
|
||||
se = parse_series_season_episode(path)
|
||||
if not se:
|
||||
return None
|
||||
season, eps = se
|
||||
|
||||
# Ensure we have episode map
|
||||
emap = episodes_map_by_series.get(series_id, {})
|
||||
# Find best ts among target episodes
|
||||
import_ts_list: List[int] = []
|
||||
air_ts_list: List[int] = []
|
||||
for e in eps:
|
||||
epi = emap.get((season, e))
|
||||
if not epi:
|
||||
continue
|
||||
eid = epi.get("id")
|
||||
air_ts = to_epoch(epi.get("airDateUtc") or epi.get("airDate"))
|
||||
if air_ts:
|
||||
air_ts_list.append(air_ts)
|
||||
if eid in earliest_import_by_episode_id:
|
||||
import_ts_list.append(earliest_import_by_episode_id[eid])
|
||||
|
||||
# write into episodes cache table if enabled
|
||||
if live_insert_db:
|
||||
db_upsert_episode(conn, series_id, season, e,
|
||||
earliest_import_by_episode_id.get(eid),
|
||||
air_ts)
|
||||
|
||||
# Preference: earliest import; else earliest airdate
|
||||
imp = min(import_ts_list) if import_ts_list else None
|
||||
air = min(air_ts_list) if air_ts_list else None
|
||||
if imp:
|
||||
return imp, "history"
|
||||
if air:
|
||||
return air, "airdate"
|
||||
return None
|
||||
|
||||
def movie_preferred_ts_for_file(
|
||||
path: str,
|
||||
conn: sqlite3.Connection,
|
||||
radarr_pack: Tuple[Radarr, Dict[int, Dict[str, Any]], Dict[str, int], Dict[int, int], Dict[int, Tuple[Optional[int], Optional[int]]]],
|
||||
live_insert_db: bool = True,
|
||||
) -> Optional[Tuple[int, str]]:
|
||||
client, movies_by_id, movies_by_imdb, earliest_by_movie, releases_by_movie_id = radarr_pack
|
||||
|
||||
# Movie folder right under movies/movies6
|
||||
parts = path.split(os.sep)
|
||||
movie_folder = ""
|
||||
for i, p in enumerate(parts):
|
||||
if p in ("movies", "movies6"):
|
||||
if i + 1 < len(parts):
|
||||
movie_folder = parts[i + 1]
|
||||
break
|
||||
if not movie_folder:
|
||||
movie_folder = os.path.basename(os.path.dirname(path))
|
||||
|
||||
imdb = parse_imdb(movie_folder)
|
||||
title, year = extract_title_and_year(movie_folder)
|
||||
|
||||
# Map to movie_id
|
||||
movie_id = None
|
||||
if imdb and imdb in movies_by_imdb:
|
||||
movie_id = movies_by_imdb[imdb]
|
||||
else:
|
||||
# title/year fallback
|
||||
tnorm = title.strip().lower()
|
||||
cand = []
|
||||
for mid, m in movies_by_id.items():
|
||||
mtitle = str(m.get("title") or "").strip().lower()
|
||||
myear = m.get("year")
|
||||
if mtitle == tnorm and (year is None or myear == year):
|
||||
cand.append(mid)
|
||||
if cand:
|
||||
movie_id = cand[0]
|
||||
if not movie_id:
|
||||
return None
|
||||
|
||||
imp = earliest_by_movie.get(movie_id)
|
||||
digital, physical = releases_by_movie_id.get(movie_id, (None, None))
|
||||
|
||||
# Preference: earliest import; fallbacks: digital -> physical
|
||||
if live_insert_db:
|
||||
db_upsert_movie(conn, movie_id, imp, digital, physical)
|
||||
|
||||
if imp:
|
||||
return imp, "history"
|
||||
if digital:
|
||||
return digital, "digital"
|
||||
if physical:
|
||||
return physical, "physical"
|
||||
return None
|
||||
|
||||
# -----------------------------
|
||||
# Apply mtimes
|
||||
# -----------------------------
|
||||
|
||||
def apply_mtime(path: str, ts: int):
|
||||
try:
|
||||
os.utime(path, (ts, ts), follow_symlinks=False)
|
||||
except Exception:
|
||||
# Try with follow_symlinks default if FS complains
|
||||
os.utime(path, (ts, ts))
|
||||
|
||||
# -----------------------------
|
||||
# Commands
|
||||
# -----------------------------
|
||||
|
||||
def cmd_build_cache(args, env):
|
||||
kind = args.kind
|
||||
db = db_open(args.db)
|
||||
|
||||
if kind == "tv":
|
||||
log("Refreshing Sonarr caches (series, episodes, history earliest-import)…")
|
||||
sonarr_pack = build_sonarr_maps(env)
|
||||
# Optionally write episodes into DB (import & airdate)
|
||||
conn = db
|
||||
_, series_by_id, _, earliest_imp_by_eid, episodes_map_by_series = sonarr_pack
|
||||
wrote = 0
|
||||
for sid, epmap in episodes_map_by_series.items():
|
||||
for (season, epnum), epi in epmap.items():
|
||||
eid = epi.get("id")
|
||||
air = to_epoch(epi.get("airDateUtc") or epi.get("airDate"))
|
||||
imp = earliest_imp_by_eid.get(eid)
|
||||
db_upsert_episode(conn, sid, season, epnum, imp, air)
|
||||
wrote += 1
|
||||
if wrote % 1000 == 0:
|
||||
conn.commit()
|
||||
conn.commit()
|
||||
log(f"Episode cache rows upserted: {wrote}")
|
||||
|
||||
elif kind == "movie":
|
||||
log("Refreshing Radarr caches (movies, history earliest-import)…")
|
||||
radarr_pack = build_radarr_maps(env)
|
||||
conn = db
|
||||
_, movies_by_id, _, earliest_by_movie, releases = radarr_pack
|
||||
wrote = 0
|
||||
for mid in movies_by_id.keys():
|
||||
imp = earliest_by_movie.get(mid)
|
||||
dig, phy = releases.get(mid, (None, None))
|
||||
db_upsert_movie(conn, mid, imp, dig, phy)
|
||||
wrote += 1
|
||||
if wrote % 1000 == 0:
|
||||
conn.commit()
|
||||
conn.commit()
|
||||
log(f"Movie cache rows upserted: {wrote}")
|
||||
else:
|
||||
raise SystemExit("--kind must be tv or movie")
|
||||
|
||||
# Reports (optional)
|
||||
if args.report_json or args.report_csv:
|
||||
export_report(db, kind, args.report_json, args.report_csv)
|
||||
|
||||
gotify_send(env, "Cache build: done", f"kind={kind}")
|
||||
db.close()
|
||||
|
||||
def export_report(conn: sqlite3.Connection, kind: str, json_path: Optional[str], csv_path: Optional[str]):
|
||||
if kind == "tv":
|
||||
rows = conn.execute("SELECT series_id, season, episode, import_ts, airdate_ts FROM episodes").fetchall()
|
||||
records = [
|
||||
{"series_id": r[0], "season": r[1], "episode": r[2], "import_ts": r[3], "airdate_ts": r[4]}
|
||||
for r in rows
|
||||
]
|
||||
else:
|
||||
rows = conn.execute("SELECT movie_id, import_ts, digital_ts, physical_ts FROM movies").fetchall()
|
||||
records = [
|
||||
{"movie_id": r[0], "import_ts": r[1], "digital_ts": r[2], "physical_ts": r[3]}
|
||||
for r in rows
|
||||
]
|
||||
|
||||
if json_path:
|
||||
os.makedirs(os.path.dirname(json_path), exist_ok=True)
|
||||
with open(json_path, "w", encoding="utf-8") as f:
|
||||
json.dump(records, f, indent=2)
|
||||
|
||||
if csv_path:
|
||||
os.makedirs(os.path.dirname(csv_path), exist_ok=True)
|
||||
import csv
|
||||
with open(csv_path, "w", newline="", encoding="utf-8") as f:
|
||||
w = csv.DictWriter(f, fieldnames=list(records[0].keys()) if records else [])
|
||||
if records:
|
||||
w.writeheader()
|
||||
w.writerows(records)
|
||||
|
||||
def cmd_apply(args, env):
|
||||
kind = args.kind
|
||||
db = db_open(args.db)
|
||||
gotify_send(env, "Apply mtimes: start", f"kind={kind}")
|
||||
|
||||
roots = resolve_roots(kind, args.root, env)
|
||||
if not roots:
|
||||
raise SystemExit("No valid roots found. Set --root or TV_ROOTS/MOVIE_ROOTS in .env")
|
||||
|
||||
# If live-on-miss, build in-memory maps once to avoid hammering endpoints
|
||||
sonarr_pack = radarr_pack = None
|
||||
if args.live_on_miss:
|
||||
if kind == "tv":
|
||||
log("Cache miss detected; refreshing Sonarr data live once…")
|
||||
sonarr_pack = build_sonarr_maps(env)
|
||||
else:
|
||||
log("Cache miss detected; refreshing Radarr data live once…")
|
||||
radarr_pack = build_radarr_maps(env)
|
||||
|
||||
# Scan
|
||||
files = list(iter_media_files(roots, kind))
|
||||
log(f"Found {len(files)} media file(s) under {', '.join(roots)}")
|
||||
|
||||
# Ticker
|
||||
notify_every = env_get_int(env, "GOTIFY_TICK_EVERY", 500) # files
|
||||
last_notify = 0
|
||||
done = 0
|
||||
updated = 0
|
||||
skipped = 0
|
||||
failures = 0
|
||||
|
||||
for idx, path in enumerate(files, 1):
|
||||
relog = False
|
||||
try:
|
||||
# Already cached by path?
|
||||
cached = db_get_file_ts(db, path)
|
||||
if cached and not args.reapply_all:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
# Compute preferred ts (from cache tables if present; or live-on-miss)
|
||||
ts_src = None
|
||||
|
||||
if kind == "tv":
|
||||
ts_src = None
|
||||
# If we already built episode cache and path is not in files yet: try live maps for this file
|
||||
if sonarr_pack:
|
||||
got = tv_preferred_ts_for_file(path, db, sonarr_pack, live_insert_db=True)
|
||||
if got:
|
||||
ts, src = got
|
||||
db_upsert_file(db, path, ts, src)
|
||||
ts_src = (ts, src)
|
||||
if not ts_src:
|
||||
# Ultimate fallback: none
|
||||
pass
|
||||
|
||||
else: # movie
|
||||
ts_src = None
|
||||
if radarr_pack:
|
||||
got = movie_preferred_ts_for_file(path, db, radarr_pack, live_insert_db=True)
|
||||
if got:
|
||||
ts, src = got
|
||||
db_upsert_file(db, path, ts, src)
|
||||
ts_src = (ts, src)
|
||||
|
||||
if not ts_src:
|
||||
# If we couldn't determine a date at all
|
||||
log(f"WARN: no import/release date for {path}")
|
||||
failures += 1
|
||||
else:
|
||||
ts, src = ts_src
|
||||
apply_mtime(path, ts)
|
||||
updated += 1
|
||||
|
||||
except Exception as e:
|
||||
failures += 1
|
||||
log(f"WARN: failed on {path}: {e}")
|
||||
# Uncomment to debug deeply
|
||||
# traceback.print_exc()
|
||||
|
||||
done += 1
|
||||
|
||||
if notify_every > 0 and done - last_notify >= notify_every:
|
||||
gotify_send(env, f"Apply {kind}: progress", f"{done}/{len(files)} files (updated {updated}, skipped {skipped}, failures {failures})")
|
||||
last_notify = done
|
||||
|
||||
db.commit()
|
||||
gotify_send(env, f"Apply mtimes: done ({kind})", f"total={done} updated={updated} skipped={skipped} failures={failures}")
|
||||
log(f"Apply complete: total={done} updated={updated} skipped={skipped} failures={failures}")
|
||||
db.close()
|
||||
|
||||
# -----------------------------
|
||||
# CLI
|
||||
# -----------------------------
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Media date cache and mtime applier (Sonarr/Radarr).")
|
||||
sub = parser.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
# build-cache
|
||||
p_build = sub.add_parser("build-cache", help="Build caches from Sonarr/Radarr into SQLite")
|
||||
p_build.add_argument("--kind", choices=["tv", "movie"], required=True)
|
||||
p_build.add_argument("--db", required=True)
|
||||
p_build.add_argument("--report-json")
|
||||
p_build.add_argument("--report-csv")
|
||||
|
||||
# apply
|
||||
p_apply = sub.add_parser("apply", help="Apply file mtimes using DB cache; with --live-on-miss it will query Sonarr/Radarr for misses")
|
||||
p_apply.add_argument("--kind", choices=["tv", "movie"], required=True)
|
||||
p_apply.add_argument("--db", required=True)
|
||||
p_apply.add_argument("--root", help="Root path; if omitted, uses TV_ROOTS/MOVIE_ROOTS from .env (supports multi-roots)")
|
||||
p_apply.add_argument("--live-on-miss", action="store_true", help="Query Sonarr/Radarr at runtime for files missing in DB and cache them")
|
||||
p_apply.add_argument("--reapply-all", action="store_true", help="Ignore existing per-file cache and reapply")
|
||||
|
||||
args = parser.parse_args()
|
||||
env = load_env()
|
||||
|
||||
if args.cmd == "build-cache":
|
||||
gotify_send(env, "Cache build: start", f"kind={args.kind}")
|
||||
cmd_build_cache(args, env)
|
||||
elif args.cmd == "apply":
|
||||
cmd_apply(args, env)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Executable
+697
@@ -0,0 +1,697 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
Media Date Cache (lazy Sonarr/Radarr):
|
||||
- Prefers FIRST import date from history
|
||||
- TV fallback: episode airDateUtc
|
||||
- Movie fallback: digitalRelease, then physicalRelease
|
||||
- Works from DB cache; with --live-on-miss it fetches Sonarr/Radarr lazily per series/movie encounter
|
||||
- Multi-root from .env (TV_ROOTS, MOVIE_ROOTS) and/or --root
|
||||
- Gotify progress notifications
|
||||
"""
|
||||
|
||||
import os, re, sys, time, json, sqlite3, argparse, traceback
|
||||
from datetime import datetime, timezone
|
||||
from typing import Dict, Any, List, Optional, Tuple
|
||||
|
||||
try:
|
||||
import requests
|
||||
except ImportError:
|
||||
print("ERROR: 'requests' is required. In your venv: pip install requests", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# -----------------------------
|
||||
# Utility / ENV
|
||||
# -----------------------------
|
||||
|
||||
ENV_DEFAULT_PATH = os.path.join(os.path.dirname(__file__), ".env")
|
||||
|
||||
def now_iso() -> str:
|
||||
return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
def log(msg: str):
|
||||
print(f"[{now_iso()}] {msg}", flush=True)
|
||||
|
||||
def load_env(dotenv_path: str = ENV_DEFAULT_PATH) -> Dict[str, str]:
|
||||
"""Minimal .env loader with inline comment stripping."""
|
||||
env = dict(os.environ)
|
||||
if os.path.exists(dotenv_path):
|
||||
with open(dotenv_path, "r", encoding="utf-8", errors="ignore") as f:
|
||||
for raw in f:
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
k, v = line.split("=", 1)
|
||||
k = k.strip()
|
||||
v = v.strip()
|
||||
quoted = (v.startswith("'") and v.endswith("'")) or (v.startswith('"') and v.endswith('"'))
|
||||
if not quoted and "#" in v:
|
||||
v = v.split("#", 1)[0].strip()
|
||||
if quoted:
|
||||
v = v[1:-1]
|
||||
env[k] = v
|
||||
return env
|
||||
|
||||
def env_get(env: Dict[str, str], key: str, default: Optional[str] = None) -> Optional[str]:
|
||||
return env.get(key, default)
|
||||
|
||||
def env_get_int(env: Dict[str, str], key: str, default: int) -> int:
|
||||
try:
|
||||
val = env.get(key, None)
|
||||
if val is None:
|
||||
return default
|
||||
return int(str(val).split("#", 1)[0].strip())
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
# -----------------------------
|
||||
# Gotify
|
||||
# -----------------------------
|
||||
|
||||
def gotify_send(env: Dict[str, str], title: str, message: str, priority: Optional[int] = None):
|
||||
url = env_get(env, "GOTIFY_URL", "")
|
||||
token = env_get(env, "GOTIFY_TOKEN", "")
|
||||
if not url or not token:
|
||||
return
|
||||
prio = priority if priority is not None else env_get_int(env, "GOTIFY_PRIORITY", 5)
|
||||
try:
|
||||
requests.post(
|
||||
url.rstrip("/") + "/message",
|
||||
data={"title": title, "message": message, "priority": prio},
|
||||
headers={"X-Gotify-Key": token},
|
||||
timeout=15,
|
||||
).raise_for_status()
|
||||
except Exception as e:
|
||||
log(f"WARN: gotify failed: {e}")
|
||||
|
||||
# -----------------------------
|
||||
# DB
|
||||
# -----------------------------
|
||||
|
||||
SCHEMA_SQL = """
|
||||
PRAGMA journal_mode=WAL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS files (
|
||||
path TEXT PRIMARY KEY,
|
||||
ts INTEGER NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS episodes (
|
||||
series_id INTEGER NOT NULL,
|
||||
season INTEGER NOT NULL,
|
||||
episode INTEGER NOT NULL,
|
||||
import_ts INTEGER,
|
||||
airdate_ts INTEGER,
|
||||
PRIMARY KEY(series_id, season, episode)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS movies (
|
||||
movie_id INTEGER PRIMARY KEY,
|
||||
import_ts INTEGER,
|
||||
digital_ts INTEGER,
|
||||
physical_ts INTEGER
|
||||
);
|
||||
"""
|
||||
|
||||
def db_open(db_path: str) -> sqlite3.Connection:
|
||||
os.makedirs(os.path.dirname(db_path), exist_ok=True)
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
conn.executescript(SCHEMA_SQL)
|
||||
return conn
|
||||
|
||||
def db_upsert_file(conn: sqlite3.Connection, path: str, ts: int, source: str):
|
||||
conn.execute(
|
||||
"INSERT INTO files(path, ts, source, updated_at) VALUES(?,?,?,?) "
|
||||
"ON CONFLICT(path) DO UPDATE SET ts=excluded.ts, source=excluded.source, updated_at=excluded.updated_at",
|
||||
(path, ts, source, int(time.time())),
|
||||
)
|
||||
|
||||
def db_get_file_ts(conn: sqlite3.Connection, path: str) -> Optional[Tuple[int, str]]:
|
||||
cur = conn.execute("SELECT ts, source FROM files WHERE path=?", (path,))
|
||||
row = cur.fetchone()
|
||||
return (row[0], row[1]) if row else None
|
||||
|
||||
def db_upsert_episode(conn, series_id, season, episode, import_ts, airdate_ts):
|
||||
conn.execute(
|
||||
"INSERT INTO episodes(series_id, season, episode, import_ts, airdate_ts) VALUES(?,?,?,?,?) "
|
||||
"ON CONFLICT(series_id, season, episode) DO UPDATE SET "
|
||||
"import_ts=COALESCE(excluded.import_ts, episodes.import_ts), "
|
||||
"airdate_ts=COALESCE(excluded.airdate_ts, episodes.airdate_ts)",
|
||||
(series_id, season, episode, import_ts, airdate_ts),
|
||||
)
|
||||
|
||||
def db_get_episode(conn, series_id, season, episode):
|
||||
cur = conn.execute(
|
||||
"SELECT import_ts, airdate_ts FROM episodes WHERE series_id=? AND season=? AND episode=?",
|
||||
(series_id, season, episode),
|
||||
)
|
||||
return cur.fetchone()
|
||||
|
||||
def db_upsert_movie(conn, movie_id, import_ts, digital_ts, physical_ts):
|
||||
conn.execute(
|
||||
"INSERT INTO movies(movie_id, import_ts, digital_ts, physical_ts) VALUES(?,?,?,?) "
|
||||
"ON CONFLICT(movie_id) DO UPDATE SET "
|
||||
"import_ts=COALESCE(excluded.import_ts, movies.import_ts), "
|
||||
"digital_ts=COALESCE(excluded.digital_ts, movies.digital_ts), "
|
||||
"physical_ts=COALESCE(excluded.physical_ts, movies.physical_ts)",
|
||||
(movie_id, import_ts, digital_ts, physical_ts),
|
||||
)
|
||||
|
||||
def db_get_movie(conn, movie_id):
|
||||
cur = conn.execute("SELECT import_ts, digital_ts, physical_ts FROM movies WHERE movie_id=?", (movie_id,))
|
||||
return cur.fetchone()
|
||||
|
||||
# -----------------------------
|
||||
# ARR clients
|
||||
# -----------------------------
|
||||
|
||||
class BaseArr:
|
||||
def __init__(self, base: str, api_key: str, timeout: int = 60):
|
||||
self.base = base.rstrip("/")
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update({"X-Api-Key": api_key})
|
||||
self.timeout = timeout
|
||||
|
||||
def _get(self, path: str, params: Dict[str, Any] = None):
|
||||
url = self.base + path
|
||||
r = self.session.get(url, params=params or {}, timeout=self.timeout)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
class Sonarr(BaseArr):
|
||||
def all_series(self) -> List[Dict[str, Any]]:
|
||||
data = self._get("/api/v3/series")
|
||||
return data if isinstance(data, list) else []
|
||||
|
||||
def episodes_for_series(self, series_id: int) -> List[Dict[str, Any]]:
|
||||
data = self._get("/api/v3/episode", params={"seriesId": series_id})
|
||||
return data if isinstance(data, list) else []
|
||||
|
||||
def history_page(self, page: int, page_size: int = 250, series_id: Optional[int] = None):
|
||||
params = {
|
||||
"page": page,
|
||||
"pageSize": page_size,
|
||||
"sortDirection": "ascending",
|
||||
"includeEpisode": "true",
|
||||
"includeSeries": "true",
|
||||
}
|
||||
if series_id:
|
||||
params["seriesId"] = series_id
|
||||
data = self._get("/api/v3/history", params=params)
|
||||
if isinstance(data, dict):
|
||||
recs = data.get("records") or data.get("Records") or []
|
||||
total = int(data.get("totalRecords") or data.get("TotalRecords") or 0)
|
||||
return recs, total
|
||||
return [], 0
|
||||
|
||||
def history_all_for_series(self, series_id: int) -> List[Dict[str, Any]]:
|
||||
out: List[Dict[str, Any]] = []
|
||||
page, size = 1, 250
|
||||
while True:
|
||||
recs, _ = self.history_page(page, size, series_id=series_id)
|
||||
if not recs:
|
||||
break
|
||||
out.extend(recs)
|
||||
if len(recs) < size:
|
||||
break
|
||||
page += 1
|
||||
return out
|
||||
|
||||
class Radarr(BaseArr):
|
||||
def movie_lookup_by_imdb(self, imdb_id: str) -> Optional[Dict[str, Any]]:
|
||||
# Radarr supports term={imdbid:tt1234567}
|
||||
data = self._get("/api/v3/movie/lookup", params={"term": f"imdbid:{imdb_id}"})
|
||||
if isinstance(data, list) and data:
|
||||
return data[0]
|
||||
return None
|
||||
|
||||
def movie_by_id(self, movie_id: int) -> Optional[Dict[str, Any]]:
|
||||
try:
|
||||
data = self._get(f"/api/v3/movie/{movie_id}")
|
||||
return data if isinstance(data, dict) else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def history_all_for_movie(self, movie_id: int) -> List[Dict[str, Any]]:
|
||||
out: List[Dict[str, Any]] = []
|
||||
page, size = 1, 250
|
||||
while True:
|
||||
params = {
|
||||
"page": page,
|
||||
"pageSize": size,
|
||||
"sortDirection": "ascending",
|
||||
"includeMovie": "true",
|
||||
"movieId": movie_id,
|
||||
}
|
||||
data = self._get("/api/v3/history", params=params)
|
||||
recs = data.get("records") if isinstance(data, dict) else []
|
||||
if not recs:
|
||||
break
|
||||
out.extend(recs)
|
||||
if len(recs) < size:
|
||||
break
|
||||
page += 1
|
||||
return out
|
||||
|
||||
# -----------------------------
|
||||
# Parsing helpers
|
||||
# -----------------------------
|
||||
|
||||
IMDB_RE = re.compile(r"\bimdb-(tt\d+)\b", re.IGNORECASE)
|
||||
SXXEYY_RE = re.compile(r"[Ss](\d{1,2})[ ._-]*[Ee](\d{2})(?:[ ._-]*[Ee-]?(\d{2}))?")
|
||||
SEASON_DIR_RE = re.compile(r"[Ss]eason[ _-]*(\d+)")
|
||||
YEAR_PAREN_RE = re.compile(r"\((\d{4})\)")
|
||||
|
||||
def parse_imdb(s: str) -> Optional[str]:
|
||||
m = IMDB_RE.search(s)
|
||||
return m.group(1).lower() if m else None
|
||||
|
||||
def parse_series_season_episode(path: str) -> Optional[Tuple[int, List[int]]]:
|
||||
fn = os.path.basename(path)
|
||||
m = SXXEYY_RE.search(fn)
|
||||
if m:
|
||||
season = int(m.group(1))
|
||||
e1 = int(m.group(2))
|
||||
e2 = m.group(3)
|
||||
eps = [e1] if not e2 else list(range(e1, int(e2) + 1))
|
||||
return season, eps
|
||||
# try a parent piece
|
||||
for part in reversed(path.split(os.sep)):
|
||||
m2 = SXXEYY_RE.search(part)
|
||||
if m2:
|
||||
season = int(m2.group(1))
|
||||
e1 = int(m2.group(2))
|
||||
e2 = m2.group(3)
|
||||
eps = [e1] if not e2 else list(range(e1, int(e2) + 1))
|
||||
return season, eps
|
||||
return None
|
||||
|
||||
def extract_series_folder(path: str) -> str:
|
||||
parts = path.split(os.sep)
|
||||
for i, p in enumerate(parts):
|
||||
if p in ("tv", "tv6"):
|
||||
if i + 1 < len(parts):
|
||||
return parts[i + 1]
|
||||
return os.path.basename(os.path.dirname(os.path.dirname(path)))
|
||||
|
||||
def extract_title_and_year(name: str) -> Tuple[str, Optional[int]]:
|
||||
year = None
|
||||
m = YEAR_PAREN_RE.search(name)
|
||||
if m:
|
||||
try: year = int(m.group(1))
|
||||
except: year = None
|
||||
title = name.split("(")[0].strip() if "(" in name else name
|
||||
return title, year
|
||||
|
||||
def to_epoch(ts_str: Optional[str]) -> Optional[int]:
|
||||
if not ts_str:
|
||||
return None
|
||||
try:
|
||||
s = ts_str.strip()
|
||||
if s.endswith("Z"):
|
||||
s = s.replace("Z", "+00:00")
|
||||
if "+" not in s:
|
||||
s += "+00:00"
|
||||
return int(datetime.fromisoformat(s).timestamp())
|
||||
except Exception:
|
||||
try:
|
||||
dt = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%S.%fZ")
|
||||
return int(dt.replace(tzinfo=timezone.utc).timestamp())
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
# -----------------------------
|
||||
# Roots / scanning
|
||||
# -----------------------------
|
||||
|
||||
def split_paths(s: str) -> List[str]:
|
||||
raw = [p.strip() for p in re.split(r"[:,]", s) if p.strip()]
|
||||
return [p for p in raw if os.path.isdir(p)]
|
||||
|
||||
def resolve_roots(kind: str, root_arg: Optional[str], env: Dict[str, str]) -> List[str]:
|
||||
roots: List[str] = []
|
||||
if root_arg and os.path.isdir(root_arg):
|
||||
roots.append(root_arg)
|
||||
if kind == "tv":
|
||||
tv_env = env_get(env, "TV_ROOTS", "")
|
||||
if tv_env:
|
||||
roots.extend(split_paths(tv_env))
|
||||
more = []
|
||||
for r in list(roots):
|
||||
if os.path.isdir(os.path.join(r, "tv")): more.append(os.path.join(r, "tv"))
|
||||
if os.path.isdir(os.path.join(r, "tv6")): more.append(os.path.join(r, "tv6"))
|
||||
roots.extend(more)
|
||||
else:
|
||||
mv_env = env_get(env, "MOVIE_ROOTS", "")
|
||||
if mv_env:
|
||||
roots.extend(split_paths(mv_env))
|
||||
more = []
|
||||
for r in list(roots):
|
||||
if os.path.isdir(os.path.join(r, "movies")): more.append(os.path.join(r, "movies"))
|
||||
if os.path.isdir(os.path.join(r, "movies6")): more.append(os.path.join(r, "movies6"))
|
||||
roots.extend(more)
|
||||
seen, out = set(), []
|
||||
for p in roots:
|
||||
if p not in seen:
|
||||
seen.add(p); out.append(p)
|
||||
return out or ([root_arg] if root_arg else [])
|
||||
|
||||
def iter_media_files(roots: List[str]):
|
||||
exts = {".mkv", ".mp4", ".avi", ".m4v", ".mov"}
|
||||
for base in roots:
|
||||
if not base or not os.path.isdir(base): continue
|
||||
for dirpath, _, filenames in os.walk(base):
|
||||
for fn in filenames:
|
||||
if os.path.splitext(fn)[1].lower() in exts:
|
||||
yield os.path.join(dirpath, fn)
|
||||
|
||||
# -----------------------------
|
||||
# Lazy Sonarr & Radarr helpers
|
||||
# -----------------------------
|
||||
|
||||
def build_sonarr_index(env: Dict[str, str]) -> Tuple[Sonarr, Dict[int, Dict[str, Any]], Dict[str, int], Dict[Tuple[str, Optional[int]], int]]:
|
||||
url = env_get(env, "SONARR_URL", "")
|
||||
key = env_get(env, "SONARR_API_KEY", "")
|
||||
if not url or not key:
|
||||
raise RuntimeError("SONARR_URL/SONARR_API_KEY not set")
|
||||
timeout = env_get_int(env, "ARR_TIMEOUT", 60)
|
||||
client = Sonarr(url, key, timeout=timeout)
|
||||
|
||||
log("Sonarr: loading series index…")
|
||||
series = client.all_series()
|
||||
series_by_id: Dict[int, Dict[str, Any]] = {}
|
||||
by_imdb: Dict[str, int] = {}
|
||||
title_year_map: Dict[Tuple[str, Optional[int]], int] = {}
|
||||
for s in series:
|
||||
sid = int(s.get("id"))
|
||||
series_by_id[sid] = s
|
||||
imdb = (s.get("imdbId") or "").lower().strip()
|
||||
if imdb: by_imdb[imdb] = sid
|
||||
title = (s.get("title") or "").strip().lower()
|
||||
year = s.get("year")
|
||||
title_year_map[(title, year)] = sid
|
||||
log(f"Sonarr: indexed {len(series_by_id)} series")
|
||||
return client, series_by_id, by_imdb, title_year_map
|
||||
|
||||
def get_series_id_for_path(path: str, series_by_imdb: Dict[str, int], title_year_map: Dict[Tuple[str, Optional[int]], int]) -> Optional[int]:
|
||||
series_folder = extract_series_folder(path)
|
||||
imdb = parse_imdb(series_folder)
|
||||
if imdb and imdb in series_by_imdb:
|
||||
return series_by_imdb[imdb]
|
||||
title, year = extract_title_and_year(series_folder)
|
||||
return title_year_map.get((title.strip().lower(), year))
|
||||
|
||||
def tv_preferred_ts_lazy(
|
||||
path: str,
|
||||
conn: sqlite3.Connection,
|
||||
client: Sonarr,
|
||||
series_by_id: Dict[int, Dict[str, Any]],
|
||||
series_by_imdb: Dict[str, int],
|
||||
title_year_map: Dict[Tuple[str, Optional[int]], int],
|
||||
episodes_cache: Dict[int, Dict[Tuple[int, int], Dict[str, Any]]],
|
||||
earliest_import_cache: Dict[int, Dict[int, int]],
|
||||
) -> Optional[Tuple[int, str]]:
|
||||
sid = get_series_id_for_path(path, series_by_imdb, title_year_map)
|
||||
if not sid: return None
|
||||
se = parse_series_season_episode(path)
|
||||
if not se: return None
|
||||
season, eps = se
|
||||
|
||||
# Fetch episodes for this series lazily
|
||||
if sid not in episodes_cache:
|
||||
log(f"Sonarr: loading episodes for series {sid}…")
|
||||
ep_list = client.episodes_for_series(sid)
|
||||
m: Dict[Tuple[int, int], Dict[str, Any]] = {}
|
||||
for e in ep_list:
|
||||
key = (int(e.get("seasonNumber", -1)), int(e.get("episodeNumber", -1)))
|
||||
m[key] = e
|
||||
episodes_cache[sid] = m
|
||||
|
||||
# Fetch history earliest-import for this series lazily
|
||||
if sid not in earliest_import_cache:
|
||||
log(f"Sonarr: loading history for series {sid}…")
|
||||
recs = client.history_all_for_series(sid)
|
||||
by_epid: Dict[int, int] = {}
|
||||
for r in recs:
|
||||
et = str(r.get("eventType") or "").lower()
|
||||
if "import" not in et:
|
||||
continue
|
||||
ep = r.get("episode") or {}
|
||||
eid = ep.get("id")
|
||||
date = to_epoch(r.get("date"))
|
||||
if eid and date:
|
||||
cur = by_epid.get(eid)
|
||||
by_epid[eid] = min(cur, date) if cur else date
|
||||
earliest_import_cache[sid] = by_epid
|
||||
|
||||
epmap = episodes_cache.get(sid, {})
|
||||
by_epid = earliest_import_cache.get(sid, {})
|
||||
|
||||
import_ts_list: List[int] = []
|
||||
air_ts_list: List[int] = []
|
||||
|
||||
for e in eps:
|
||||
epi = epmap.get((season, e))
|
||||
if not epi: continue
|
||||
eid = epi.get("id")
|
||||
air_ts = to_epoch(epi.get("airDateUtc") or epi.get("airDate"))
|
||||
if air_ts: air_ts_list.append(air_ts)
|
||||
if eid and eid in by_epid:
|
||||
import_ts_list.append(by_epid[eid])
|
||||
|
||||
# persist to episode table for future runs
|
||||
db_upsert_episode(conn, sid, season, e, by_epid.get(eid), air_ts)
|
||||
|
||||
if import_ts_list:
|
||||
return min(import_ts_list), "history"
|
||||
if air_ts_list:
|
||||
return min(air_ts_list), "airdate"
|
||||
return None
|
||||
|
||||
def movie_preferred_ts_lazy(
|
||||
path: str,
|
||||
conn: sqlite3.Connection,
|
||||
client: Radarr,
|
||||
) -> Optional[Tuple[int, str]]:
|
||||
# Identify movie folder under movies/movies6
|
||||
parts = path.split(os.sep)
|
||||
movie_folder = ""
|
||||
for i, p in enumerate(parts):
|
||||
if p in ("movies", "movies6") and i + 1 < len(parts):
|
||||
movie_folder = parts[i + 1]
|
||||
break
|
||||
if not movie_folder:
|
||||
movie_folder = os.path.basename(os.path.dirname(path))
|
||||
imdb = parse_imdb(movie_folder)
|
||||
title, year = extract_title_and_year(movie_folder)
|
||||
|
||||
movie: Optional[Dict[str, Any]] = None
|
||||
if imdb:
|
||||
movie = client.movie_lookup_by_imdb(imdb)
|
||||
# If no imdb or lookup failed, try guessing by path title/year is not implemented lazily (to avoid /api/v3/movie bulk pull).
|
||||
if not movie:
|
||||
return None
|
||||
|
||||
movie_id = movie.get("id")
|
||||
if not movie_id:
|
||||
return None
|
||||
|
||||
# Build earliest import from history for THIS movie only
|
||||
recs = client.history_all_for_movie(movie_id)
|
||||
import_ts: Optional[int] = None
|
||||
for r in recs:
|
||||
et = str(r.get("eventType") or "").lower()
|
||||
if "import" in et:
|
||||
dt = to_epoch(r.get("date"))
|
||||
import_ts = min(import_ts, dt) if (import_ts and dt) else (dt or import_ts)
|
||||
|
||||
# Preferred: import; fallback digital -> physical
|
||||
digital = to_epoch(movie.get("digitalRelease") or movie.get("digitalReleaseDate") or movie.get("inCinemas"))
|
||||
physical = to_epoch(movie.get("physicalRelease") or movie.get("physicalReleaseDate"))
|
||||
|
||||
# persist to movie table for future runs
|
||||
db_upsert_movie(conn, movie_id, import_ts, digital, physical)
|
||||
|
||||
if import_ts: return import_ts, "history"
|
||||
if digital: return digital, "digital"
|
||||
if physical: return physical, "physical"
|
||||
return None
|
||||
|
||||
# -----------------------------
|
||||
# Apply mtimes
|
||||
# -----------------------------
|
||||
|
||||
def apply_mtime(path: str, ts: int):
|
||||
try:
|
||||
os.utime(path, (ts, ts), follow_symlinks=False)
|
||||
except Exception:
|
||||
os.utime(path, (ts, ts))
|
||||
|
||||
# -----------------------------
|
||||
# Commands
|
||||
# -----------------------------
|
||||
|
||||
def cmd_apply(args, env):
|
||||
db = db_open(args.db)
|
||||
kind = args.kind
|
||||
gotify_send(env, "Apply mtimes: start", f"kind={kind}")
|
||||
|
||||
roots = resolve_roots(kind, args.root, env)
|
||||
if not roots:
|
||||
raise SystemExit("No valid roots. Set --root or TV_ROOTS/MOVIE_ROOTS in .env")
|
||||
|
||||
files = list(iter_media_files(roots))
|
||||
log(f"Found {len(files)} media file(s) under {', '.join(roots)}")
|
||||
|
||||
# Lazy ARR setup
|
||||
sonarr_client = None
|
||||
radarr_client = None
|
||||
series_by_id: Dict[int, Dict[str, Any]] = {}
|
||||
series_by_imdb: Dict[str, int] = {}
|
||||
title_year_map: Dict[Tuple[str, Optional[int]], int] = {}
|
||||
episodes_cache: Dict[int, Dict[Tuple[int, int], Dict[str, Any]]] = {}
|
||||
earliest_import_cache: Dict[int, Dict[int, int]] = {}
|
||||
|
||||
if args.live_on_miss:
|
||||
if kind == "tv":
|
||||
log("Initializing Sonarr (series index only)…")
|
||||
client, s_by_id, s_by_imdb, tmap = build_sonarr_index(env)
|
||||
sonarr_client = client
|
||||
series_by_id = s_by_id
|
||||
series_by_imdb = s_by_imdb
|
||||
title_year_map = tmap
|
||||
else:
|
||||
url = env_get(env, "RADARR_URL", "")
|
||||
key = env_get(env, "RADARR_API_KEY", "")
|
||||
if not url or not key:
|
||||
raise SystemExit("RADARR_URL/RADARR_API_KEY not set")
|
||||
timeout = env_get_int(env, "ARR_TIMEOUT", 60)
|
||||
radarr_client = Radarr(url, key, timeout=timeout)
|
||||
log("Initialized Radarr client (lazy per-movie lookups).")
|
||||
|
||||
# Progress
|
||||
notify_every = env_get_int(env, "GOTIFY_TICK_EVERY", 500)
|
||||
last_notify = 0
|
||||
done = skipped = updated = failures = 0
|
||||
|
||||
for idx, path in enumerate(files, 1):
|
||||
try:
|
||||
# cached file path?
|
||||
cached = db_get_file_ts(db, path)
|
||||
if cached and not args.reapply_all:
|
||||
skipped += 1
|
||||
else:
|
||||
ts_src: Optional[Tuple[int, str]] = None
|
||||
if kind == "tv":
|
||||
if sonarr_client:
|
||||
ts_src = tv_preferred_ts_lazy(
|
||||
path, db, sonarr_client,
|
||||
series_by_id, series_by_imdb, title_year_map,
|
||||
episodes_cache, earliest_import_cache
|
||||
)
|
||||
else:
|
||||
if radarr_client:
|
||||
ts_src = movie_preferred_ts_lazy(path, db, radarr_client)
|
||||
|
||||
if ts_src:
|
||||
ts, src = ts_src
|
||||
db_upsert_file(db, path, ts, src)
|
||||
apply_mtime(path, ts)
|
||||
updated += 1
|
||||
else:
|
||||
log(f"WARN: no import/release date for {path}")
|
||||
failures += 1
|
||||
done += 1
|
||||
|
||||
if notify_every > 0 and done - last_notify >= notify_every:
|
||||
gotify_send(env, f"Apply {kind}: progress",
|
||||
f"{done}/{len(files)} files (updated {updated}, skipped {skipped}, failures {failures})")
|
||||
last_notify = done
|
||||
except Exception as e:
|
||||
failures += 1
|
||||
log(f"WARN: failed on {path}: {e}")
|
||||
# traceback.print_exc()
|
||||
|
||||
db.commit()
|
||||
gotify_send(env, f"Apply mtimes: done ({kind})", f"total={done} updated={updated} skipped={skipped} failures={failures}")
|
||||
log(f"Apply complete: total={done} updated={updated} skipped={skipped} failures={failures}")
|
||||
db.close()
|
||||
|
||||
def cmd_build_cache(args, env):
|
||||
# Optional now; lazy apply fills DB as it goes.
|
||||
# Keeping a light build for people who still want a pre-warm step.
|
||||
kind = args.kind
|
||||
db = db_open(args.db)
|
||||
if kind == "tv":
|
||||
client, series_by_id, series_by_imdb, title_year_map = build_sonarr_index(env)
|
||||
wrote = 0
|
||||
for sid in series_by_id.keys():
|
||||
eps = client.episodes_for_series(sid)
|
||||
recs = client.history_all_for_series(sid)
|
||||
by_epid: Dict[int, int] = {}
|
||||
for r in recs:
|
||||
et = str(r.get("eventType") or "").lower()
|
||||
if "import" not in et: continue
|
||||
ep = r.get("episode") or {}
|
||||
eid = ep.get("id"); dt = to_epoch(r.get("date"))
|
||||
if eid and dt:
|
||||
cur = by_epid.get(eid)
|
||||
by_epid[eid] = min(cur, dt) if cur else dt
|
||||
for e in eps:
|
||||
season = int(e.get("seasonNumber", -1))
|
||||
epno = int(e.get("episodeNumber", -1))
|
||||
eid = e.get("id")
|
||||
air = to_epoch(e.get("airDateUtc") or e.get("airDate"))
|
||||
imp = by_epid.get(eid)
|
||||
db_upsert_episode(db, sid, season, epno, imp, air)
|
||||
wrote += 1
|
||||
if wrote % 2000 == 0:
|
||||
db.commit()
|
||||
db.commit()
|
||||
log(f"Episode cache rows upserted: {wrote}")
|
||||
else:
|
||||
url = env_get(env, "RADARR_URL", "")
|
||||
key = env_get(env, "RADARR_API_KEY", "")
|
||||
if not url or not key:
|
||||
raise SystemExit("RADARR_URL/RADARR_API_KEY not set")
|
||||
timeout = env_get_int(env, "ARR_TIMEOUT", 60)
|
||||
rad = Radarr(url, key, timeout=timeout)
|
||||
log("Movie cache build (lazy is preferred; this just verifies connectivity).")
|
||||
gotify_send(env, "Cache build: done", f"kind={kind}")
|
||||
db.close()
|
||||
|
||||
# -----------------------------
|
||||
# CLI
|
||||
# -----------------------------
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Media date cache and mtime applier (lazy Sonarr/Radarr).")
|
||||
sub = parser.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
p_build = sub.add_parser("build-cache", help="(Optional) Prewarm caches")
|
||||
p_build.add_argument("--kind", choices=["tv", "movie"], required=True)
|
||||
p_build.add_argument("--db", required=True)
|
||||
|
||||
p_apply = sub.add_parser("apply", help="Apply file mtimes; with --live-on-miss it lazily queries Sonarr/Radarr for misses")
|
||||
p_apply.add_argument("--kind", choices=["tv", "movie"], required=True)
|
||||
p_apply.add_argument("--db", required=True)
|
||||
p_apply.add_argument("--root", help="Root path; if omitted uses TV_ROOTS/MOVIE_ROOTS from .env")
|
||||
p_apply.add_argument("--live-on-miss", action="store_true")
|
||||
p_apply.add_argument("--reapply-all", action="store_true")
|
||||
|
||||
args = parser.parse_args()
|
||||
env = load_env()
|
||||
|
||||
if args.cmd == "build-cache":
|
||||
gotify_send(env, "Cache build: start", f"kind={args.kind}")
|
||||
cmd_build_cache(args, env)
|
||||
else:
|
||||
cmd_apply(args, env)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Executable
+942
@@ -0,0 +1,942 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
media_date_cache.py
|
||||
|
||||
Builds and applies "best possible" dates to media files, caching choices in SQLite.
|
||||
Sources, in priority order:
|
||||
|
||||
TV (kind=tv):
|
||||
1) Sonarr first-import date (history and/or episodeFile.dateAdded)
|
||||
2) Sonarr episode airDateUtc
|
||||
3) TMDB episode air_date (via folder IMDb ID)
|
||||
4) (optional) file mtime fallback if --fallback file_mtime
|
||||
|
||||
Movies (kind=movie):
|
||||
1) Radarr first-import (history and/or movieFile.dateAdded)
|
||||
2) Radarr release dates (digital -> physical -> theatrical)
|
||||
3) TMDB release dates (digital -> physical -> theatrical) via folder IMDb ID
|
||||
4) (optional) file mtime fallback if --fallback file_mtime
|
||||
|
||||
Environment (.env supported):
|
||||
SONARR_URL, SONARR_API_KEY
|
||||
RADARR_URL, RADARR_API_KEY
|
||||
TMDB_API_KEY
|
||||
TMDB_PRIMARY_COUNTRY=US
|
||||
GOTIFY_URL, GOTIFY_TOKEN, GOTIFY_PRIORITY=5
|
||||
GOTIFY_TICK_EVERY=500
|
||||
TV_ROOTS=/mnt/unionfs/Media/TV:/mnt/unionfs/Media/TV/tv:/mnt/unionfs/Media/TV/tv6
|
||||
MOVIE_ROOTS=/mnt/unionfs/Media/Movies:/mnt/unionfs/Media/Movies/movies:/mnt/unionfs/Media/Movies/movies6
|
||||
|
||||
Usage examples:
|
||||
python media_date_cache.py apply --kind tv --db /opt/scripts/unmanic/.cache/media_dates.db --root /mnt/unionfs/Media/TV --live-on-miss
|
||||
python media_date_cache.py apply --kind movie --db /opt/scripts/unmanic/.cache/media_dates.db --root /mnt/unionfs/Media/Movies --live-on-miss
|
||||
|
||||
Requires:
|
||||
pip install requests python-dotenv
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
except Exception:
|
||||
load_dotenv = None
|
||||
|
||||
import requests
|
||||
|
||||
VIDEO_EXTS = {".mkv", ".mp4", ".avi", ".m4v", ".mov", ".wmv", ".ts", ".m2ts"}
|
||||
|
||||
# ---------- Utilities ----------
|
||||
|
||||
def log(msg: str):
|
||||
now = dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
print(f"[{now}] {msg}", flush=True)
|
||||
|
||||
def load_env(env_file: Optional[str] = None) -> Dict[str, str]:
|
||||
if env_file and os.path.isfile(env_file):
|
||||
if load_dotenv:
|
||||
load_dotenv(env_file)
|
||||
else:
|
||||
# Auto-load .env from cwd if present
|
||||
local = Path(".env")
|
||||
if local.exists() and load_dotenv:
|
||||
load_dotenv(local)
|
||||
|
||||
return dict(os.environ)
|
||||
|
||||
def env_get(env: Dict[str, str], key: str, default: Optional[str] = None) -> Optional[str]:
|
||||
val = env.get(key, default)
|
||||
return val
|
||||
|
||||
def env_get_int(env: Dict[str, str], key: str, default: int) -> int:
|
||||
raw = str(env.get(key, str(default)) or str(default))
|
||||
# tolerate trailing comments like: "5 # optional"
|
||||
raw = raw.strip().split()[0]
|
||||
try:
|
||||
return int(raw)
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
def parse_utc(ts: str) -> Optional[int]:
|
||||
if not ts:
|
||||
return None
|
||||
# Try a few common formats
|
||||
for fmt in ("%Y-%m-%dT%H:%M:%S%z", "%Y-%m-%dT%H:%M:%S.%f%z",
|
||||
"%Y-%m-%dT%H:%M:%SZ", "%Y-%m-%d"):
|
||||
try:
|
||||
if fmt.endswith("Z"):
|
||||
dt_obj = dt.datetime.strptime(ts, fmt).replace(tzinfo=dt.timezone.utc)
|
||||
elif fmt == "%Y-%m-%d":
|
||||
# interpret bare dates as midnight UTC
|
||||
dt_obj = dt.datetime.strptime(ts, "%Y-%m-%d").replace(tzinfo=dt.timezone.utc)
|
||||
else:
|
||||
dt_obj = dt.datetime.strptime(ts, fmt)
|
||||
return int(dt_obj.timestamp())
|
||||
except Exception:
|
||||
continue
|
||||
return None
|
||||
|
||||
def dt_to_epoch_utc(d: dt.datetime) -> int:
|
||||
if d.tzinfo is None:
|
||||
d = d.replace(tzinfo=dt.timezone.utc)
|
||||
return int(d.timestamp())
|
||||
|
||||
def safe_join_paths(paths: List[str]) -> List[str]:
|
||||
out = []
|
||||
for p in paths:
|
||||
if p and os.path.isdir(p):
|
||||
out.append(p)
|
||||
return out
|
||||
|
||||
IMDB_RX = re.compile(r"(?:^|\W)(?:imdb-)?(tt\d{6,9})(?:\W|$)", re.IGNORECASE)
|
||||
SXXEYY_RX = re.compile(r"[Ss](\d{1,3})[ ._-]*[Ee](\d{1,3})")
|
||||
|
||||
def extract_imdb_from_path(path: str) -> Optional[str]:
|
||||
# Search each segment (show or movie folder often has it)
|
||||
for part in Path(path).parts[::-1]:
|
||||
m = IMDB_RX.search(part)
|
||||
if m:
|
||||
return m.group(1).lower()
|
||||
return None
|
||||
|
||||
def parse_sxxeyy(path: str) -> Optional[Tuple[int, int]]:
|
||||
m = SXXEYY_RX.search(path)
|
||||
if not m:
|
||||
return None
|
||||
try:
|
||||
return int(m.group(1)), int(m.group(2))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
# ---------- Gotify ----------
|
||||
|
||||
def gotify_send(env: Dict[str, str], title: str, message: str, priority: Optional[int] = None):
|
||||
url = env_get(env, "GOTIFY_URL")
|
||||
token = env_get(env, "GOTIFY_TOKEN")
|
||||
if not url or not token:
|
||||
return
|
||||
prio = priority if priority is not None else env_get_int(env, "GOTIFY_PRIORITY", 5)
|
||||
try:
|
||||
r = requests.post(
|
||||
f"{url.rstrip('/')}/message",
|
||||
headers={"X-Gotify-Key": token},
|
||||
json={"title": title, "message": message, "priority": prio},
|
||||
timeout=10,
|
||||
)
|
||||
r.raise_for_status()
|
||||
except Exception as e:
|
||||
log(f"GOTIFY WARN: {e}")
|
||||
|
||||
# ---------- SQLite ----------
|
||||
|
||||
DDL_MEDIA_DATES = """
|
||||
CREATE TABLE IF NOT EXISTS media_dates (
|
||||
path TEXT PRIMARY KEY,
|
||||
kind TEXT NOT NULL,
|
||||
chosen_ts INTEGER NOT NULL,
|
||||
chosen_src TEXT NOT NULL,
|
||||
extra_json TEXT,
|
||||
first_seen INTEGER NOT NULL,
|
||||
last_update INTEGER NOT NULL
|
||||
);
|
||||
"""
|
||||
IDX_MEDIA_DATES_KIND = "CREATE INDEX IF NOT EXISTS idx_media_dates_kind ON media_dates(kind);"
|
||||
|
||||
def ensure_db(db_path: str):
|
||||
os.makedirs(os.path.dirname(db_path), exist_ok=True)
|
||||
with sqlite3.connect(db_path) as cx:
|
||||
cx.execute(DDL_MEDIA_DATES)
|
||||
cx.execute(IDX_MEDIA_DATES_KIND)
|
||||
cx.commit()
|
||||
|
||||
def db_get_row(cx: sqlite3.Connection, path: str) -> Optional[Tuple]:
|
||||
cur = cx.execute("SELECT path, kind, chosen_ts, chosen_src, extra_json, first_seen, last_update FROM media_dates WHERE path=?", (path,))
|
||||
return cur.fetchone()
|
||||
|
||||
def db_upsert(cx: sqlite3.Connection, path: str, kind: str, chosen_ts: int, chosen_src: str, extra: dict):
|
||||
now = int(time.time())
|
||||
prev = db_get_row(cx, path)
|
||||
if prev:
|
||||
first_seen = prev[5]
|
||||
cx.execute(
|
||||
"UPDATE media_dates SET chosen_ts=?, chosen_src=?, extra_json=?, last_update=? WHERE path=?",
|
||||
(chosen_ts, chosen_src, json.dumps(extra, ensure_ascii=False), now, path),
|
||||
)
|
||||
else:
|
||||
first_seen = now
|
||||
cx.execute(
|
||||
"INSERT INTO media_dates (path, kind, chosen_ts, chosen_src, extra_json, first_seen, last_update) VALUES (?,?,?,?,?,?,?)",
|
||||
(path, kind, chosen_ts, chosen_src, json.dumps(extra, ensure_ascii=False), first_seen, now),
|
||||
)
|
||||
|
||||
# ---------- API clients (Sonarr, Radarr, TMDB) ----------
|
||||
|
||||
def _req_json(method: str, url: str, headers=None, params=None, timeout=30):
|
||||
r = requests.request(method, url, headers=headers or {}, params=params or {}, timeout=timeout)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
class SonarrClient:
|
||||
def __init__(self, base: Optional[str], key: Optional[str]):
|
||||
self.base = base.rstrip("/") if base else None
|
||||
self.key = key
|
||||
self._series = None # list
|
||||
self._episodes = {} # seriesId -> list
|
||||
self._history = {} # seriesId -> list
|
||||
self._epfiles = {} # seriesId -> list
|
||||
|
||||
def ok(self) -> bool:
|
||||
return bool(self.base and self.key)
|
||||
|
||||
def _hdr(self):
|
||||
return {"X-Api-Key": self.key}
|
||||
|
||||
def series(self) -> List[dict]:
|
||||
if self._series is not None:
|
||||
return self._series
|
||||
if not self.ok():
|
||||
self._series = []
|
||||
return self._series
|
||||
log("Sonarr: loading series index…")
|
||||
js = _req_json("GET", f"{self.base}/api/v3/series", headers=self._hdr())
|
||||
if isinstance(js, list):
|
||||
self._series = js
|
||||
else:
|
||||
# defensive
|
||||
self._series = list(js) if js else []
|
||||
log(f"Sonarr: indexed {len(self._series)} series")
|
||||
return self._series
|
||||
|
||||
def episodes_for(self, series_id: int) -> List[dict]:
|
||||
if series_id in self._episodes:
|
||||
return self._episodes[series_id]
|
||||
if not self.ok():
|
||||
self._episodes[series_id] = []
|
||||
return self._episodes[series_id]
|
||||
js = _req_json("GET", f"{self.base}/api/v3/episode", headers=self._hdr(), params={"seriesId": series_id})
|
||||
if isinstance(js, list):
|
||||
out = js
|
||||
elif isinstance(js, dict) and "records" in js:
|
||||
out = js["records"]
|
||||
else:
|
||||
out = []
|
||||
self._episodes[series_id] = out
|
||||
return out
|
||||
|
||||
def episode_files_for(self, series_id: int) -> List[dict]:
|
||||
if series_id in self._epfiles:
|
||||
return self._epfiles[series_id]
|
||||
if not self.ok():
|
||||
self._epfiles[series_id] = []
|
||||
return self._epfiles[series_id]
|
||||
js = _req_json("GET", f"{self.base}/api/v3/episodefile", headers=self._hdr(), params={"seriesId": series_id})
|
||||
self._epfiles[series_id] = js if isinstance(js, list) else []
|
||||
return self._epfiles[series_id]
|
||||
|
||||
def history_for(self, series_id: int) -> List[dict]:
|
||||
if series_id in self._history:
|
||||
return self._history[series_id]
|
||||
if not self.ok():
|
||||
self._history[series_id] = []
|
||||
return self._history[series_id]
|
||||
# series history paginated
|
||||
page = 1
|
||||
page_size = 500
|
||||
records: List[dict] = []
|
||||
while True:
|
||||
js = _req_json(
|
||||
"GET",
|
||||
f"{self.base}/api/v3/history/series",
|
||||
headers=self._hdr(),
|
||||
params={"seriesId": series_id, "includeEpisode": "true", "page": page, "pageSize": page_size},
|
||||
)
|
||||
chunk = []
|
||||
if isinstance(js, dict):
|
||||
chunk = js.get("records") or js.get("results") or []
|
||||
total = js.get("totalRecords") or js.get("total") or 0
|
||||
elif isinstance(js, list):
|
||||
chunk = js
|
||||
total = len(js)
|
||||
records.extend(chunk)
|
||||
if isinstance(js, dict) and js.get("page") is not None:
|
||||
# If paginated, stop when we fetched all
|
||||
if len(records) >= total:
|
||||
break
|
||||
page += 1
|
||||
else:
|
||||
# Not paginated
|
||||
break
|
||||
self._history[series_id] = records
|
||||
return records
|
||||
|
||||
class RadarrClient:
|
||||
def __init__(self, base: Optional[str], key: Optional[str]):
|
||||
self.base = base.rstrip("/") if base else None
|
||||
self.key = key
|
||||
self._movies = None
|
||||
self._history = None
|
||||
self._files_by_movie = {}
|
||||
|
||||
def ok(self) -> bool:
|
||||
return bool(self.base and self.key)
|
||||
|
||||
def _hdr(self):
|
||||
return {"X-Api-Key": self.key}
|
||||
|
||||
def movies(self) -> List[dict]:
|
||||
if self._movies is not None:
|
||||
return self._movies
|
||||
if not self.ok():
|
||||
self._movies = []
|
||||
return self._movies
|
||||
js = _req_json("GET", f"{self.base}/api/v3/movie", headers=self._hdr())
|
||||
self._movies = js if isinstance(js, list) else []
|
||||
return self._movies
|
||||
|
||||
def history(self) -> List[dict]:
|
||||
if self._history is not None:
|
||||
return self._history
|
||||
if not self.ok():
|
||||
self._history = []
|
||||
return self._history
|
||||
page = 1
|
||||
page_size = 500
|
||||
records: List[dict] = []
|
||||
while True:
|
||||
js = _req_json("GET", f"{self.base}/api/v3/history", headers=self._hdr(), params={"page": page, "pageSize": page_size})
|
||||
chunk = []
|
||||
total = 0
|
||||
if isinstance(js, dict):
|
||||
chunk = js.get("records") or js.get("results") or []
|
||||
total = js.get("totalRecords") or js.get("total") or 0
|
||||
elif isinstance(js, list):
|
||||
chunk = js
|
||||
total = len(js)
|
||||
records.extend(chunk)
|
||||
if isinstance(js, dict) and js.get("page") is not None:
|
||||
if len(records) >= total:
|
||||
break
|
||||
page += 1
|
||||
else:
|
||||
break
|
||||
self._history = records
|
||||
return records
|
||||
|
||||
def moviefile_for(self, movie_id: int) -> Optional[dict]:
|
||||
if movie_id in self._files_by_movie:
|
||||
return self._files_by_movie[movie_id]
|
||||
if not self.ok():
|
||||
self._files_by_movie[movie_id] = None
|
||||
return None
|
||||
js = _req_json("GET", f"{self.base}/api/v3/moviefile", headers=self._hdr(), params={"movieId": movie_id})
|
||||
# could be list, pick first if any
|
||||
mf = js[0] if isinstance(js, list) and js else None
|
||||
self._files_by_movie[movie_id] = mf
|
||||
return mf
|
||||
|
||||
class TMDBClient:
|
||||
def __init__(self, api_key: Optional[str], primary_country: str = "US"):
|
||||
self.key = api_key
|
||||
self.primary_country = (primary_country or "US").upper()
|
||||
self.base = "https://api.themoviedb.org/3"
|
||||
self._tv_id_cache: Dict[str, int] = {} # imdb_id -> tmdb_tv_id
|
||||
self._movie_id_cache: Dict[str, int] = {} # imdb_id -> tmdb_movie_id
|
||||
self._season_cache: Dict[Tuple[int, int], dict] = {} # (tv_id, season)-> episodes mapping
|
||||
|
||||
def ok(self) -> bool:
|
||||
return bool(self.key)
|
||||
|
||||
def _get(self, path: str, params: dict = None):
|
||||
if not self.ok():
|
||||
return None
|
||||
params = dict(params or {})
|
||||
params["api_key"] = self.key
|
||||
return _req_json("GET", f"{self.base}{path}", params=params)
|
||||
|
||||
def tv_id_by_imdb(self, imdb_id: str) -> Optional[int]:
|
||||
imdb_id = imdb_id.lower()
|
||||
if imdb_id in self._tv_id_cache:
|
||||
return self._tv_id_cache[imdb_id]
|
||||
js = self._get(f"/find/{imdb_id}", {"external_source": "imdb_id"})
|
||||
tid = None
|
||||
if isinstance(js, dict):
|
||||
tvres = js.get("tv_results") or []
|
||||
if tvres:
|
||||
tid = tvres[0].get("id")
|
||||
self._tv_id_cache[imdb_id] = tid
|
||||
return tid
|
||||
|
||||
def movie_id_by_imdb(self, imdb_id: str) -> Optional[int]:
|
||||
imdb_id = imdb_id.lower()
|
||||
if imdb_id in self._movie_id_cache:
|
||||
return self._movie_id_cache[imdb_id]
|
||||
js = self._get(f"/find/{imdb_id}", {"external_source": "imdb_id"})
|
||||
mid = None
|
||||
if isinstance(js, dict):
|
||||
mres = js.get("movie_results") or []
|
||||
if mres:
|
||||
mid = mres[0].get("id")
|
||||
self._movie_id_cache[imdb_id] = mid
|
||||
return mid
|
||||
|
||||
def season_airdates(self, tv_id: int, season_number: int) -> Dict[int, str]:
|
||||
key = (tv_id, season_number)
|
||||
if key in self._season_cache:
|
||||
return self._season_cache[key]
|
||||
js = self._get(f"/tv/{tv_id}/season/{season_number}")
|
||||
mapping: Dict[int, str] = {}
|
||||
if isinstance(js, dict):
|
||||
for ep in js.get("episodes") or []:
|
||||
epno = ep.get("episode_number")
|
||||
ad = ep.get("air_date")
|
||||
if epno and ad:
|
||||
mapping[int(epno)] = ad
|
||||
self._season_cache[key] = mapping
|
||||
return mapping
|
||||
|
||||
def movie_best_release(self, movie_id: int) -> Optional[Tuple[str, str]]:
|
||||
"""
|
||||
Returns (src_key, release_date_iso) where src_key in {'tmdb_digital','tmdb_physical','tmdb_theatrical'}
|
||||
Preference order: digital(4) -> physical(5) -> theatrical(3), prioritizing primary_country, else any.
|
||||
"""
|
||||
js = self._get(f"/movie/{movie_id}/release_dates")
|
||||
if not isinstance(js, dict):
|
||||
return None
|
||||
results = js.get("results") or []
|
||||
def pick(for_country: Optional[str]) -> Optional[Tuple[str, str]]:
|
||||
best = {}
|
||||
for entry in results:
|
||||
if for_country and (entry.get("iso_3166_1") != for_country):
|
||||
continue
|
||||
for rel in entry.get("release_dates") or []:
|
||||
rtype = int(rel.get("type") or 0)
|
||||
rdate = rel.get("release_date")
|
||||
if not rdate:
|
||||
continue
|
||||
if rtype == 4: # digital
|
||||
best.setdefault("tmdb_digital", []).append(rdate)
|
||||
elif rtype == 5: # physical
|
||||
best.setdefault("tmdb_physical", []).append(rdate)
|
||||
elif rtype == 3: # theatrical
|
||||
best.setdefault("tmdb_theatrical", []).append(rdate)
|
||||
# choose earliest of preferred buckets
|
||||
for key in ("tmdb_digital", "tmdb_physical", "tmdb_theatrical"):
|
||||
if key in best:
|
||||
return key, sorted(best[key])[0]
|
||||
return None
|
||||
# Try primary country first, then any
|
||||
chosen = pick(self.primary_country) or pick(None)
|
||||
return chosen
|
||||
|
||||
# ---------- Matching helpers ----------
|
||||
|
||||
def longest_prefix_match(target: str, prefix_map: Dict[str, dict]) -> Optional[dict]:
|
||||
"""
|
||||
prefix_map: series_path -> series_obj (Sonarr), or movie_path -> movie_obj (Radarr)
|
||||
Returns the mapped object whose path is the longest prefix of target.
|
||||
"""
|
||||
candidates = []
|
||||
t = target.lower()
|
||||
for p, obj in prefix_map.items():
|
||||
pl = p.lower()
|
||||
if t.startswith(pl.rstrip("/") + "/") or t == pl:
|
||||
candidates.append((len(pl), obj))
|
||||
if not candidates:
|
||||
return None
|
||||
candidates.sort(key=lambda x: x[0], reverse=True)
|
||||
return candidates[0][1]
|
||||
|
||||
def expand_roots_from_env(env: Dict[str, str], kind: str, cli_roots: List[str]) -> List[str]:
|
||||
roots = []
|
||||
if cli_roots:
|
||||
roots.extend(cli_roots)
|
||||
else:
|
||||
env_key = "TV_ROOTS" if kind == "tv" else "MOVIE_ROOTS"
|
||||
val = env_get(env, env_key)
|
||||
if val:
|
||||
roots.extend([p.strip() for p in val.split(":") if p.strip()])
|
||||
# Add known child dirs automatically if they exist
|
||||
expanded = []
|
||||
for r in roots:
|
||||
expanded.append(r)
|
||||
# Only add children for top-level TV/Movie roots
|
||||
if kind == "tv":
|
||||
for child in ("tv", "tv6"):
|
||||
c = os.path.join(r, child)
|
||||
if os.path.isdir(c):
|
||||
expanded.append(c)
|
||||
else:
|
||||
for child in ("movies", "movies6"):
|
||||
c = os.path.join(r, child)
|
||||
if os.path.isdir(c):
|
||||
expanded.append(c)
|
||||
# dedupe and only keep existing dirs
|
||||
uniq = []
|
||||
seen = set()
|
||||
for p in expanded:
|
||||
if os.path.isdir(p) and p not in seen:
|
||||
uniq.append(p)
|
||||
seen.add(p)
|
||||
return uniq
|
||||
|
||||
def walk_media_files(roots: List[str]) -> List[str]:
|
||||
files = []
|
||||
for root in roots:
|
||||
for dirpath, _, filenames in os.walk(root):
|
||||
for fn in filenames:
|
||||
if Path(fn).suffix.lower() in VIDEO_EXTS:
|
||||
files.append(os.path.join(dirpath, fn))
|
||||
return files
|
||||
|
||||
# ---------- Choosing timestamps ----------
|
||||
|
||||
def sonarr_choose_for_file(sc: SonarrClient, file_path: str) -> Optional[Tuple[int, str, dict]]:
|
||||
"""
|
||||
If the file belongs to a Sonarr series (by path prefix), try to determine a timestamp:
|
||||
- earliest import (history or episodeFile.dateAdded)
|
||||
- else episode airDateUtc
|
||||
Returns (ts, src, extra) or None
|
||||
"""
|
||||
if not sc.ok():
|
||||
return None
|
||||
series_index = {s.get("path") or "": s for s in sc.series() if (s.get("path") or "").strip()}
|
||||
series = longest_prefix_match(file_path, series_index)
|
||||
if not series:
|
||||
return None
|
||||
|
||||
sid = series.get("id")
|
||||
eps = sc.episodes_for(sid)
|
||||
epfiles = sc.episode_files_for(sid)
|
||||
hist = sc.history_for(sid)
|
||||
|
||||
# Build lookup maps
|
||||
# Map (season,episode) -> episode id and airDateUtc
|
||||
ep_map: Dict[Tuple[int, int], dict] = {}
|
||||
for e in eps:
|
||||
try:
|
||||
k = (int(e.get("seasonNumber") or 0), int(e.get("episodeNumber") or 0))
|
||||
ep_map[k] = e
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# episodeId -> earliest import date (from history)
|
||||
import_map: Dict[int, int] = {}
|
||||
for r in hist:
|
||||
try:
|
||||
ev = (r.get("eventType") or r.get("event") or "")
|
||||
evl = str(ev).lower()
|
||||
if "import" not in evl:
|
||||
continue
|
||||
edate = r.get("date")
|
||||
ts = parse_utc(edate)
|
||||
if ts is None:
|
||||
continue
|
||||
ep_obj = r.get("episode") or {}
|
||||
eid = ep_obj.get("id")
|
||||
if not eid:
|
||||
continue
|
||||
prev = import_map.get(eid)
|
||||
if prev is None or ts < prev:
|
||||
import_map[eid] = ts
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# episodeFile dateAdded (another hint)
|
||||
# episodeFile doesn't directly map to episodeId always, but often the filename/relativePath implies SxxEyy.
|
||||
# We'll only use dateAdded as a general earliest among files that share the same S/E when we can parse S/E.
|
||||
# We'll handle per-file below.
|
||||
|
||||
# Parse S/E from the path
|
||||
se = parse_sxxeyy(file_path)
|
||||
if not se:
|
||||
return None
|
||||
s_num, e_num = se
|
||||
ep = ep_map.get((s_num, e_num))
|
||||
if not ep:
|
||||
return None
|
||||
eid = ep.get("id")
|
||||
|
||||
# Candidate 1: history first import for this episode
|
||||
candidates: List[Tuple[int, str]] = []
|
||||
if eid and eid in import_map:
|
||||
candidates.append((import_map[eid], "sonarr_import"))
|
||||
|
||||
# Candidate 2: episodeFile.dateAdded for matching S/E if any
|
||||
# Try to match on season/episode parsed from episodeFile.relativePath
|
||||
earliest_epfile_ts = None
|
||||
for ef in epfiles:
|
||||
rel = ef.get("relativePath") or ""
|
||||
se2 = parse_sxxeyy(rel)
|
||||
if se2 == (s_num, e_num):
|
||||
ts2 = parse_utc(ef.get("dateAdded"))
|
||||
if ts2 is not None and (earliest_epfile_ts is None or ts2 < earliest_epfile_ts):
|
||||
earliest_epfile_ts = ts2
|
||||
if earliest_epfile_ts is not None:
|
||||
candidates.append((earliest_epfile_ts, "episodefile_dateAdded"))
|
||||
|
||||
# Candidate 3: airDateUtc
|
||||
air_ts = parse_utc(ep.get("airDateUtc") or ep.get("airDate"))
|
||||
if air_ts is not None:
|
||||
candidates.append((air_ts, "tv_airdate"))
|
||||
|
||||
if not candidates:
|
||||
return None
|
||||
candidates.sort(key=lambda x: x[0])
|
||||
chosen_ts, chosen_src = candidates[0]
|
||||
extra = {
|
||||
"seriesId": sid,
|
||||
"seriesTitle": series.get("title"),
|
||||
"season": s_num,
|
||||
"episode": e_num,
|
||||
"candidates": [{"ts": ts, "src": src} for ts, src in candidates],
|
||||
}
|
||||
return chosen_ts, chosen_src, extra
|
||||
|
||||
def radarr_choose_for_file(rc: RadarrClient, file_path: str) -> Optional[Tuple[int, str, dict]]:
|
||||
"""
|
||||
Determine movie timestamp using Radarr:
|
||||
- earliest import from history/movieFile.dateAdded
|
||||
- else Radarr release dates (digital->physical->theatrical)
|
||||
"""
|
||||
if not rc.ok():
|
||||
return None
|
||||
movies_index = {m.get("path") or "": m for m in rc.movies() if (m.get("path") or "").strip()}
|
||||
movie = longest_prefix_match(file_path, movies_index)
|
||||
if not movie:
|
||||
return None
|
||||
mid = movie.get("id")
|
||||
|
||||
candidates: List[Tuple[int, str]] = []
|
||||
|
||||
# History import events
|
||||
for r in rc.history():
|
||||
try:
|
||||
ev = (r.get("eventType") or r.get("event") or "")
|
||||
evl = str(ev).lower()
|
||||
if "import" not in evl:
|
||||
continue
|
||||
mobj = r.get("movie") or {}
|
||||
if mobj.get("id") != mid:
|
||||
continue
|
||||
ts = parse_utc(r.get("date"))
|
||||
if ts is not None:
|
||||
candidates.append((ts, "radarr_import"))
|
||||
except Exception:
|
||||
continue
|
||||
# Movie file dateAdded
|
||||
mf = rc.moviefile_for(mid)
|
||||
mf_ts = parse_utc(mf.get("dateAdded")) if mf else None
|
||||
if mf_ts is not None:
|
||||
candidates.append((mf_ts, "moviefile_dateAdded"))
|
||||
|
||||
# Radarr release dates on the movie object itself
|
||||
# Radarr's movie object has fields like 'digitalRelease', 'physicalRelease', 'inCinemas'
|
||||
for key, src in (("digitalRelease", "movie_digital"),
|
||||
("physicalRelease", "movie_physical"),
|
||||
("inCinemas", "movie_cinema")):
|
||||
ts = parse_utc(movie.get(key))
|
||||
if ts is not None:
|
||||
candidates.append((ts, src))
|
||||
|
||||
if not candidates:
|
||||
return None
|
||||
candidates.sort(key=lambda x: x[0])
|
||||
chosen_ts, chosen_src = candidates[0]
|
||||
extra = {
|
||||
"movieId": mid,
|
||||
"movieTitle": movie.get("title"),
|
||||
"candidates": [{"ts": ts, "src": src} for ts, src in candidates],
|
||||
}
|
||||
return chosen_ts, chosen_src, extra
|
||||
|
||||
def tmdb_tv_choose(file_path: str, tmdb: TMDBClient) -> Optional[Tuple[int, str, dict]]:
|
||||
"""
|
||||
For a TV episode when Sonarr lacks it:
|
||||
- Extract IMDb ID from path
|
||||
- Map to TMDB tv_id
|
||||
- Parse season/episode from filename
|
||||
- Fetch season's episodes and get air_date for the episode
|
||||
"""
|
||||
if not tmdb.ok():
|
||||
return None
|
||||
imdb_id = extract_imdb_from_path(file_path)
|
||||
if not imdb_id:
|
||||
return None
|
||||
se = parse_sxxeyy(file_path)
|
||||
if not se:
|
||||
return None
|
||||
s_num, e_num = se
|
||||
tv_id = tmdb.tv_id_by_imdb(imdb_id)
|
||||
if not tv_id:
|
||||
return None
|
||||
airdates = tmdb.season_airdates(tv_id, s_num)
|
||||
iso = airdates.get(e_num)
|
||||
if not iso:
|
||||
return None
|
||||
ts = parse_utc(iso)
|
||||
if ts is None:
|
||||
return None
|
||||
extra = {
|
||||
"imdb": imdb_id,
|
||||
"tmdb_tv_id": tv_id,
|
||||
"season": s_num,
|
||||
"episode": e_num,
|
||||
"tmdb_air_date": iso,
|
||||
}
|
||||
return ts, "tmdb_airdate", extra
|
||||
|
||||
def tmdb_movie_choose(file_path: str, tmdb: TMDBClient) -> Optional[Tuple[int, str, dict]]:
|
||||
if not tmdb.ok():
|
||||
return None
|
||||
imdb_id = extract_imdb_from_path(file_path)
|
||||
if not imdb_id:
|
||||
return None
|
||||
movie_id = tmdb.movie_id_by_imdb(imdb_id)
|
||||
if not movie_id:
|
||||
return None
|
||||
picked = tmdb.movie_best_release(movie_id)
|
||||
if not picked:
|
||||
return None
|
||||
src, iso = picked
|
||||
ts = parse_utc(iso)
|
||||
if ts is None:
|
||||
return None
|
||||
extra = {"imdb": imdb_id, "tmdb_movie_id": movie_id, "tmdb_release": {"src": src, "date": iso}}
|
||||
return ts, src, extra
|
||||
|
||||
# ---------- Apply ----------
|
||||
|
||||
def apply_mtime(path: str, ts: int, dry_run: bool = False) -> bool:
|
||||
try:
|
||||
if dry_run:
|
||||
return True
|
||||
os.utime(path, (ts, ts))
|
||||
return True
|
||||
except Exception as e:
|
||||
log(f"ERROR utime failed for {path}: {e}")
|
||||
return False
|
||||
|
||||
def apply_command(args):
|
||||
env = load_env(args.env)
|
||||
ensure_db(args.db)
|
||||
|
||||
# Clients
|
||||
sc = SonarrClient(env_get(env, "SONARR_URL"), env_get(env, "SONARR_API_KEY"))
|
||||
rc = RadarrClient(env_get(env, "RADARR_URL"), env_get(env, "RADARR_API_KEY"))
|
||||
tmdb = TMDBClient(env_get(env, "TMDB_API_KEY"), env_get(env, "TMDB_PRIMARY_COUNTRY") or "US")
|
||||
|
||||
tick_every = env_get_int(env, "GOTIFY_TICK_EVERY", 500)
|
||||
|
||||
# Roots
|
||||
roots = expand_roots_from_env(env, args.kind, args.root or [])
|
||||
if not roots:
|
||||
log(f"ERROR: No roots found for kind={args.kind}. Use --root or set TV_ROOTS/MOVIE_ROOTS.")
|
||||
sys.exit(2)
|
||||
|
||||
files = walk_media_files(roots)
|
||||
log(f"Found {len(files)} media file(s) under {', '.join(roots)}")
|
||||
|
||||
# If asked, prewarm Sonarr/Radarr series indexes (episodes/history lazy anyway)
|
||||
if args.kind == "tv" and args.live_on_miss:
|
||||
log("Initializing Sonarr (series index only)…")
|
||||
sc.series() # index
|
||||
if args.kind == "movie" and args.live_on_miss:
|
||||
log("Initializing Radarr (movies index only)…")
|
||||
rc.movies()
|
||||
|
||||
gotify_send(env, "Apply mtimes: start", f"kind={args.kind}, files={len(files)}")
|
||||
|
||||
updated = 0
|
||||
checked = 0
|
||||
misses = 0
|
||||
with sqlite3.connect(args.db) as cx:
|
||||
for path in files:
|
||||
checked += 1
|
||||
# Existing cached?
|
||||
row = db_get_row(cx, path)
|
||||
if row and not args.refresh:
|
||||
# Already have it — apply to file (unless skipping)
|
||||
ts, src = row[2], row[3]
|
||||
ok = apply_mtime(path, ts, dry_run=args.dry_run)
|
||||
if ok:
|
||||
updated += 1
|
||||
if checked % tick_every == 0:
|
||||
gotify_send(env, "Apply mtimes: progress", f"{checked}/{len(files)} (cached)")
|
||||
continue
|
||||
|
||||
# Need to compute live
|
||||
chosen: Optional[Tuple[int, str, dict]] = None
|
||||
|
||||
if args.kind == "tv":
|
||||
if args.live_on_miss and sc.ok():
|
||||
chosen = sonarr_choose_for_file(sc, path)
|
||||
if not chosen:
|
||||
# TMDB fallback
|
||||
chosen = tmdb_tv_choose(path, tmdb)
|
||||
else:
|
||||
# movies
|
||||
if args.live_on_miss and rc.ok():
|
||||
chosen = radarr_choose_for_file(rc, path)
|
||||
if not chosen:
|
||||
chosen = tmdb_movie_choose(path, tmdb)
|
||||
|
||||
if not chosen and args.fallback == "file_mtime":
|
||||
try:
|
||||
st = os.stat(path)
|
||||
ts = int(st.st_mtime)
|
||||
chosen = (ts, "file_mtime", {})
|
||||
except Exception:
|
||||
chosen = None
|
||||
|
||||
if not chosen:
|
||||
log(f"WARN: no import/release date for {path}")
|
||||
misses += 1
|
||||
if checked % tick_every == 0:
|
||||
gotify_send(env, "Apply mtimes: progress", f"{checked}/{len(files)} (misses so far: {misses})")
|
||||
continue
|
||||
|
||||
ts, src, extra = chosen
|
||||
# Upsert DB & apply
|
||||
db_upsert(cx, path, args.kind, ts, src, extra)
|
||||
ok = apply_mtime(path, ts, dry_run=args.dry_run)
|
||||
if ok:
|
||||
updated += 1
|
||||
|
||||
if checked % tick_every == 0:
|
||||
gotify_send(env, "Apply mtimes: progress", f"{checked}/{len(files)} (updated: {updated})")
|
||||
|
||||
cx.commit()
|
||||
|
||||
gotify_send(env, "Apply mtimes: done", f"kind={args.kind}, updated={updated}, checked={checked}, misses={misses}")
|
||||
log(f"Done. kind={args.kind} checked={checked} updated={updated} misses={misses} dry_run={args.dry_run}")
|
||||
|
||||
# ---------- Build cache (optional prefetch) ----------
|
||||
|
||||
def build_cache_command(args):
|
||||
env = load_env(args.env)
|
||||
ensure_db(args.db)
|
||||
|
||||
sc = SonarrClient(env_get(env, "SONARR_URL"), env_get(env, "SONARR_API_KEY"))
|
||||
rc = RadarrClient(env_get(env, "RADARR_URL"), env_get(env, "RADARR_API_KEY"))
|
||||
tmdb = TMDBClient(env_get(env, "TMDB_API_KEY"), env_get(env, "TMDB_PRIMARY_COUNTRY") or "US")
|
||||
|
||||
gotify_send(env, "Cache build: start", f"kind={args.kind}")
|
||||
|
||||
roots = expand_roots_from_env(env, args.kind, args.root or [])
|
||||
files = walk_media_files(roots) if roots else []
|
||||
if roots:
|
||||
log(f"Found {len(files)} media file(s) under {', '.join(roots)}")
|
||||
|
||||
inserted = 0
|
||||
checked = 0
|
||||
with sqlite3.connect(args.db) as cx:
|
||||
if args.kind == "tv" and sc.ok():
|
||||
series = sc.series()
|
||||
total = len(series)
|
||||
for i, s in enumerate(series, start=1):
|
||||
log(f"Sonarr: loading episodes for series {s.get('id')} ({i}/{total})…")
|
||||
eps = sc.episodes_for(s.get("id"))
|
||||
log(f"Sonarr: loading history for series {s.get('id')}…")
|
||||
hist = sc.history_for(s.get("id"))
|
||||
epfiles = sc.episode_files_for(s.get("id"))
|
||||
# We are not mapping specific files here; build-cache is optional.
|
||||
# If roots provided, we will also pass over files and insert rows using live logic:
|
||||
# If roots provided, populate using the same chooser logic as apply but without touching mtimes.
|
||||
if files:
|
||||
for path in files:
|
||||
checked += 1
|
||||
if db_get_row(cx, path) and not args.refresh:
|
||||
continue
|
||||
chosen = None
|
||||
if args.kind == "tv":
|
||||
if sc.ok():
|
||||
chosen = sonarr_choose_for_file(sc, path)
|
||||
if not chosen:
|
||||
chosen = tmdb_tv_choose(path, tmdb)
|
||||
else:
|
||||
if rc.ok():
|
||||
chosen = radarr_choose_for_file(rc, path)
|
||||
if not chosen:
|
||||
chosen = tmdb_movie_choose(path, tmdb)
|
||||
if chosen:
|
||||
ts, src, extra = chosen
|
||||
db_upsert(cx, path, args.kind, ts, src, extra)
|
||||
inserted += 1
|
||||
elif args.fallback == "file_mtime":
|
||||
try:
|
||||
st = os.stat(path)
|
||||
ts = int(st.st_mtime)
|
||||
db_upsert(cx, path, args.kind, ts, "file_mtime", {})
|
||||
inserted += 1
|
||||
except Exception:
|
||||
pass
|
||||
if checked % 500 == 0:
|
||||
gotify_send(env, "Cache build: progress", f"{checked}/{len(files)} cached")
|
||||
cx.commit()
|
||||
|
||||
gotify_send(env, "Cache build: done", f"kind={args.kind}, inserted={inserted}, scanned={checked or 'n/a'}")
|
||||
log(f"Cache build done. kind={args.kind} inserted={inserted} scanned={checked}")
|
||||
|
||||
# ---------- CLI ----------
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="Media date cache & mtime applier (Sonarr/Radarr + TMDB fallback)")
|
||||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
common = argparse.ArgumentParser(add_help=False)
|
||||
common.add_argument("--db", required=True, help="SQLite database path")
|
||||
common.add_argument("--env", default=None, help="Path to .env (optional, defaults to ./.env if present)")
|
||||
common.add_argument("--kind", choices=["tv", "movie"], required=True, help="Kind of media this run targets")
|
||||
common.add_argument("--root", action="append", help="Root path(s) to scan; can repeat. If omitted, uses TV_ROOTS/MOVIE_ROOTS.")
|
||||
common.add_argument("--refresh", action="store_true", help="Ignore existing DB rows; recompute live")
|
||||
common.add_argument("--fallback", choices=["none", "file_mtime"], default="none", help="Enable last-resort fallback")
|
||||
common.add_argument("--dry-run", action="store_true", help="Compute and write DB, but do not change file mtimes")
|
||||
|
||||
ap_apply = sub.add_parser("apply", parents=[common], help="Apply mtimes to files, caching choices")
|
||||
ap_apply.add_argument("--live-on-miss", action="store_true", help="If DB row missing, query Sonarr/Radarr/TMDB live")
|
||||
|
||||
ap_build = sub.add_parser("build-cache", parents=[common], help="Build/populate cache without touching mtimes")
|
||||
# build-cache shares flags
|
||||
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.cmd == "apply":
|
||||
apply_command(args)
|
||||
elif args.cmd == "build-cache":
|
||||
build_cache_command(args)
|
||||
else:
|
||||
ap.print_help()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Binary file not shown.
Reference in New Issue
Block a user