update: to sonarr process
Local Docker Build (Dev) / build-dev (push) Successful in 4s

This commit is contained in:
2025-10-19 10:14:53 -04:00
parent 7089c883a9
commit d1d1e1a3d1
3 changed files with 131 additions and 294 deletions
+1 -1
View File
@@ -1 +1 @@
2.5.3
2.5.4
+27 -226
View File
@@ -1,66 +1,42 @@
#!/usr/bin/env python3
"""
Database management for NFOGuard
Handles SQLite and PostgreSQL database operations for tracking media dates and processing history
PostgreSQL database management for NFOGuard
Handles database operations for tracking media dates and processing history
"""
import sqlite3
import json
import threading
import os
from pathlib import Path
from datetime import datetime
from typing import Optional, Dict, List, Any, Union
from typing import Optional, Dict, List, Any
from contextlib import contextmanager
try:
import psycopg2
import psycopg2.extras
POSTGRESQL_AVAILABLE = True
except ImportError:
psycopg2 = None
POSTGRESQL_AVAILABLE = False
import psycopg2
import psycopg2.extras
class NFOGuardDatabase:
"""Manages NFOGuard database operations for both SQLite and PostgreSQL"""
"""PostgreSQL database manager for NFOGuard media tracking and processing history"""
def __init__(self, config=None, db_path: Optional[Path] = None):
def __init__(self, config):
"""
Initialize database connection
Initialize PostgreSQL database connection
Args:
config: Configuration object with database settings
db_path: Path for SQLite database (backward compatibility)
"""
if config:
self.db_type = config.db_type
if self.db_type == "postgresql":
if not POSTGRESQL_AVAILABLE:
raise ImportError("psycopg2 is required for PostgreSQL support")
if not config:
raise ValueError("PostgreSQL configuration is required")
self.db_host = config.db_host
self.db_port = config.db_port
self.db_name = config.db_name
self.db_user = config.db_user
self.db_password = config.db_password
else:
self.db_path = Path(config.db_path)
self.db_path.parent.mkdir(parents=True, exist_ok=True)
else:
# Backward compatibility
self.db_type = "sqlite"
self.db_path = Path(db_path) if db_path else Path("/app/data/media_dates.db")
self.db_path.parent.mkdir(parents=True, exist_ok=True)
self._local = threading.local()
self._init_database()
def _get_placeholder(self) -> str:
"""Get the correct parameter placeholder for this database type"""
return "%s" if self.db_type == "postgresql" else "?"
def _get_connection(self) -> Union[sqlite3.Connection, 'psycopg2.extensions.connection']:
"""Get thread-local database connection"""
def _get_connection(self) -> 'psycopg2.extensions.connection':
"""Get thread-local PostgreSQL database connection"""
if not hasattr(self._local, 'connection'):
if self.db_type == "postgresql":
self._local.connection = psycopg2.connect(
host=self.db_host,
port=self.db_port,
@@ -70,57 +46,37 @@ class NFOGuardDatabase:
cursor_factory=psycopg2.extras.RealDictCursor
)
self._local.connection.autocommit = True
else:
self._local.connection = sqlite3.connect(
self.db_path,
check_same_thread=False,
timeout=30.0
)
self._local.connection.row_factory = sqlite3.Row
return self._local.connection
def _get_first_value(self, row):
"""Get first value from row, handling both dict and tuple types"""
if self.db_type == "postgresql":
"""Get first value from row from PostgreSQL RealDictCursor"""
# RealDictCursor returns dict-like objects
return list(row.values())[0] if row else None
else:
# SQLite returns tuples
return row[0] if row else None
@contextmanager
def get_connection(self):
"""Context manager for database connections"""
"""Context manager for PostgreSQL database connections"""
conn = self._get_connection()
try:
yield conn
if self.db_type == "sqlite":
conn.commit() # PostgreSQL uses autocommit
# PostgreSQL uses autocommit - no manual commit needed
except Exception:
if self.db_type == "sqlite":
conn.rollback() # PostgreSQL uses autocommit
# PostgreSQL uses autocommit - no manual rollback needed
raise
def _init_database(self):
"""Initialize database tables with migration support"""
"""Initialize PostgreSQL database tables"""
with self.get_connection() as conn:
cursor = conn.cursor()
if self.db_type == "postgresql":
self._init_postgresql_tables(cursor)
else:
self._init_sqlite_tables(cursor)
# Test the connection works and verify autocommit
cursor.execute("SELECT 1")
if self.db_type == "postgresql":
print(f"✅ Database initialized and connection verified: {self.db_type.upper()}")
print(f"🔍 PostgreSQL autocommit status: {conn.autocommit}")
else:
print(f"✅ Database initialized and connection verified: {self.db_type.upper()}")
print(f"✅ PostgreSQL database initialized and connection verified")
print(f"🔍 Autocommit status: {conn.autocommit}")
def _init_postgresql_tables(self, cursor):
"""Initialize PostgreSQL tables"""
"""Initialize database tables"""
# Series table
cursor.execute("""
CREATE TABLE IF NOT EXISTS series (
@@ -177,105 +133,12 @@ class NFOGuardDatabase:
cursor.execute("CREATE INDEX IF NOT EXISTS idx_episodes_video ON episodes(has_video_file)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_movies_video ON movies(has_video_file)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_history_imdb ON processing_history(imdb_id)")
def _init_sqlite_tables(self, cursor):
"""Initialize SQLite tables"""
# Series table
cursor.execute("""
CREATE TABLE IF NOT EXISTS series (
imdb_id TEXT PRIMARY KEY,
path TEXT NOT NULL,
last_updated TEXT NOT NULL,
metadata TEXT
)
""")
# Episodes table
cursor.execute("""
CREATE TABLE IF NOT EXISTS episodes (
imdb_id TEXT NOT NULL,
season INTEGER NOT NULL,
episode INTEGER NOT NULL,
aired TEXT,
dateadded TEXT,
source TEXT,
last_updated TEXT NOT NULL,
has_video_file BOOLEAN DEFAULT FALSE,
PRIMARY KEY (imdb_id, season, episode),
FOREIGN KEY (imdb_id) REFERENCES series(imdb_id)
)
""")
# Movies table
cursor.execute("""
CREATE TABLE IF NOT EXISTS movies (
imdb_id TEXT PRIMARY KEY,
path TEXT NOT NULL,
released TEXT,
dateadded TEXT,
source TEXT,
last_updated TEXT NOT NULL,
has_video_file BOOLEAN DEFAULT FALSE
)
""")
# Processing history table
cursor.execute("""
CREATE TABLE IF NOT EXISTS processing_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
imdb_id TEXT NOT NULL,
media_type TEXT NOT NULL,
event_type TEXT NOT NULL,
processed_at TEXT NOT NULL,
details TEXT
)
""")
# Handle SQLite migrations for existing databases
self._handle_sqlite_migrations(cursor)
# Create indexes for SQLite
cursor.execute("CREATE INDEX IF NOT EXISTS idx_episodes_imdb ON episodes(imdb_id)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_episodes_video ON episodes(has_video_file)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_movies_video ON movies(has_video_file)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_history_imdb ON processing_history(imdb_id)")
def _handle_sqlite_migrations(self, cursor):
"""Handle SQLite database migrations for existing databases"""
# Check current schema and add missing columns
cursor.execute("PRAGMA table_info(movies)")
movie_columns = [row[1] for row in cursor.fetchall()]
cursor.execute("PRAGMA table_info(episodes)")
episode_columns = [row[1] for row in cursor.fetchall()]
# Add missing columns to movies table
if 'path' not in movie_columns:
cursor.execute("ALTER TABLE movies ADD COLUMN path TEXT")
cursor.execute("UPDATE movies SET path = '/unknown/path/' || imdb_id WHERE path IS NULL")
if 'has_video_file' not in movie_columns:
cursor.execute("ALTER TABLE movies ADD COLUMN has_video_file BOOLEAN DEFAULT FALSE")
if 'last_updated' not in movie_columns:
cursor.execute("ALTER TABLE movies ADD COLUMN last_updated TEXT")
cursor.execute("UPDATE movies SET last_updated = datetime('now') WHERE last_updated IS NULL")
# Add missing columns to episodes table
if 'has_video_file' not in episode_columns:
cursor.execute("ALTER TABLE episodes ADD COLUMN has_video_file BOOLEAN DEFAULT FALSE")
if 'last_updated' not in episode_columns:
cursor.execute("ALTER TABLE episodes ADD COLUMN last_updated TEXT")
cursor.execute("UPDATE episodes SET last_updated = datetime('now') WHERE last_updated IS NULL")
def upsert_series(self, imdb_id: str, path: str, metadata: Optional[Dict] = None):
"""Insert or update series record"""
with self.get_connection() as conn:
cursor = conn.cursor()
timestamp = datetime.utcnow()
if self.db_type == "postgresql":
cursor.execute("""
INSERT INTO series (imdb_id, path, last_updated, metadata)
VALUES (%s, %s, %s, %s)
@@ -284,11 +147,6 @@ class NFOGuardDatabase:
last_updated = EXCLUDED.last_updated,
metadata = EXCLUDED.metadata
""", (imdb_id, path, timestamp, json.dumps(metadata) if metadata else None))
else:
cursor.execute("""
INSERT OR REPLACE INTO series (imdb_id, path, last_updated, metadata)
VALUES (?, ?, ?, ?)
""", (imdb_id, path, timestamp.isoformat(), json.dumps(metadata) if metadata else None))
def upsert_episode_date(self, imdb_id: str, season: int, episode: int,
aired: Optional[str], dateadded: Optional[str],
@@ -298,7 +156,6 @@ class NFOGuardDatabase:
cursor = conn.cursor()
timestamp = datetime.utcnow()
if self.db_type == "postgresql":
cursor.execute("""
INSERT INTO episodes
(imdb_id, season, episode, aired, dateadded, source, has_video_file, last_updated)
@@ -311,12 +168,6 @@ class NFOGuardDatabase:
last_updated = EXCLUDED.last_updated
""", (imdb_id, season, episode, aired, dateadded, source, has_video_file, timestamp))
print(f"🔍 DEBUG: PostgreSQL upsert executed for {imdb_id} S{season:02d}E{episode:02d}, rows affected: {cursor.rowcount}")
else:
cursor.execute("""
INSERT OR REPLACE INTO episodes
(imdb_id, season, episode, aired, dateadded, source, has_video_file, last_updated)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""", (imdb_id, season, episode, aired, dateadded, source, has_video_file, timestamp.isoformat()))
def upsert_movie(self, imdb_id: str, path: str):
"""Insert or update movie record"""
@@ -324,7 +175,6 @@ class NFOGuardDatabase:
cursor = conn.cursor()
timestamp = datetime.utcnow()
if self.db_type == "postgresql":
cursor.execute("""
INSERT INTO movies (imdb_id, path, last_updated)
VALUES (%s, %s, %s)
@@ -332,21 +182,6 @@ class NFOGuardDatabase:
path = EXCLUDED.path,
last_updated = EXCLUDED.last_updated
""", (imdb_id, path, timestamp))
else:
try:
cursor.execute("""
INSERT OR REPLACE INTO movies (imdb_id, path, last_updated)
VALUES (?, ?, ?)
""", (imdb_id, path, timestamp.isoformat()))
except sqlite3.OperationalError as e:
if "no column named path" in str(e):
# Fallback for databases without path column - just insert imdb_id
cursor.execute("""
INSERT OR REPLACE INTO movies (imdb_id, last_updated)
VALUES (?, ?)
""", (imdb_id, timestamp.isoformat()))
else:
raise
def upsert_movie_dates(self, imdb_id: str, released: Optional[str],
dateadded: Optional[str], source: str, has_video_file: bool = False):
@@ -356,8 +191,6 @@ class NFOGuardDatabase:
cursor = conn.cursor()
timestamp = datetime.utcnow()
if self.db_type == "postgresql":
# PostgreSQL version with UPSERT
cursor.execute("""
INSERT INTO movies (imdb_id, path, released, dateadded, source, has_video_file, last_updated)
VALUES (%s, COALESCE((SELECT path FROM movies WHERE imdb_id = %s), 'unknown'), %s, %s, %s, %s, %s)
@@ -371,20 +204,6 @@ class NFOGuardDatabase:
# Debug: Check what was actually saved
cursor.execute("SELECT dateadded, source FROM movies WHERE imdb_id = %s", (imdb_id,))
else:
# SQLite version with INSERT OR REPLACE
cursor.execute("""
INSERT OR REPLACE INTO movies (imdb_id, path, released, dateadded, source, has_video_file, last_updated)
VALUES (
?,
COALESCE((SELECT path FROM movies WHERE imdb_id = ?), 'unknown'),
?, ?, ?, ?, ?
)
""", (imdb_id, imdb_id, released, dateadded, source, has_video_file, timestamp.isoformat()))
# Debug: Check what was actually saved
cursor.execute("SELECT dateadded, source FROM movies WHERE imdb_id = ?", (imdb_id,))
result = cursor.fetchone()
print(f"🔍 DATABASE VERIFY: After upsert, found dateadded={result[0] if result else 'NOT_FOUND'}, source={result[1] if result else 'NOT_FOUND'}")
@@ -393,16 +212,10 @@ class NFOGuardDatabase:
with self.get_connection() as conn:
cursor = conn.cursor()
if self.db_type == "postgresql":
query = "SELECT * FROM episodes WHERE imdb_id = %s"
params = [imdb_id]
if has_video_file_only:
query += " AND has_video_file = TRUE"
else:
query = "SELECT * FROM episodes WHERE imdb_id = ?"
params = [imdb_id]
if has_video_file_only:
query += " AND has_video_file = TRUE"
cursor.execute(query, params)
return [dict(row) for row in cursor.fetchall()]
@@ -411,10 +224,9 @@ class NFOGuardDatabase:
"""Get episode date record"""
with self.get_connection() as conn:
cursor = conn.cursor()
ph = self._get_placeholder()
cursor.execute(f"""
cursor.execute("""
SELECT * FROM episodes
WHERE imdb_id = {ph} AND season = {ph} AND episode = {ph}
WHERE imdb_id = %s AND season = %s AND episode = %s
""", (imdb_id, season, episode))
row = cursor.fetchone()
@@ -424,8 +236,7 @@ class NFOGuardDatabase:
"""Get movie date record"""
with self.get_connection() as conn:
cursor = conn.cursor()
ph = self._get_placeholder()
cursor.execute(f"SELECT * FROM movies WHERE imdb_id = {ph}", (imdb_id,))
cursor.execute("SELECT * FROM movies WHERE imdb_id = %s", (imdb_id,))
row = cursor.fetchone()
return dict(row) if row else None
@@ -434,21 +245,16 @@ class NFOGuardDatabase:
"""Add processing history entry"""
with self.get_connection() as conn:
cursor = conn.cursor()
placeholder = self._get_placeholder()
cursor.execute(f"""
cursor.execute("""
INSERT INTO processing_history (imdb_id, media_type, event_type, processed_at, details)
VALUES ({placeholder}, {placeholder}, {placeholder}, {placeholder}, {placeholder})
VALUES (%s, %s, %s, %s, %s)
""", (imdb_id, media_type, event_type, datetime.utcnow().isoformat(),
json.dumps(details) if details else None))
def get_stats(self) -> Dict[str, Any]:
"""Get database statistics"""
with self.get_connection() as conn:
# Use regular cursor for simple COUNT queries (not RealDictCursor)
if self.db_type == "postgresql":
cursor = conn.cursor() # Regular cursor for PostgreSQL
else:
cursor = conn.cursor() # SQLite cursor
# Series stats
cursor.execute("SELECT COUNT(*) FROM series")
@@ -472,15 +278,10 @@ class NFOGuardDatabase:
cursor.execute("SELECT COUNT(*) FROM processing_history")
history_count = self._get_first_value(cursor.fetchone())
# Database size calculation
if self.db_type == "postgresql":
# PostgreSQL: Query pg_database_size
# Database size calculation for PostgreSQL
cursor.execute("SELECT pg_database_size(%s)", (self.db_name,))
db_size_bytes = self._get_first_value(cursor.fetchone())
db_size_mb = round(db_size_bytes / 1024 / 1024, 2) if db_size_bytes else 0
else:
# SQLite: File size
db_size_mb = round(self.db_path.stat().st_size / 1024 / 1024, 2)
return {
"series_count": series_count,
@@ -490,7 +291,7 @@ class NFOGuardDatabase:
"movies_with_video": movies_with_video,
"processing_history_count": history_count,
"database_size_mb": db_size_mb,
"database_type": self.db_type
"database_type": "postgresql"
}
def close(self):
+39 -3
View File
@@ -921,8 +921,8 @@ class TVProcessor:
"""
Get episode date for webhook processing with correct priority:
1. Check existing NFOGuard database entry (preserve previous import dates)
2. If not found, use current time (webhook = new download)
3. Get aired date from Sonarr/TMDB for reference
2. Check Sonarr import history - if old history exists, prefer air date
3. If no history, use current time (webhook = new download)
"""
# Check if we already have this episode in our database (preserve existing dates)
existing = self.db.get_episode_date(imdb_id, season_num, episode_num)
@@ -938,7 +938,43 @@ class TVProcessor:
aired = episode_data.get('airDate')
_log("DEBUG", f"Got aired date from Sonarr for S{season_num:02d}E{episode_num:02d}: {aired}")
# For webhook processing, use current time as dateadded (new download)
# NEW: Check Sonarr import history to detect if this is an existing show
episode_id = None
if series_metadata and 'episodes' in series_metadata:
episode_data = series_metadata['episodes'].get((season_num, episode_num))
if episode_data:
episode_id = episode_data.get('id')
if episode_id and hasattr(self, 'sonarr'):
try:
import_history = self.sonarr.get_episode_import_history(episode_id)
if import_history:
from datetime import datetime as dt
from dateutil import parser
# Parse the import history date
history_date = parser.parse(import_history)
current_time = dt.now(history_date.tzinfo)
time_diff = current_time - history_date
# If import history is more than 24 hours old, this is likely a re-download/upgrade
# In this case, prefer air date over webhook date
if time_diff.days > 0:
_log("INFO", f"Found old import history for S{season_num:02d}E{episode_num:02d} ({import_history}), using air date instead of webhook date")
if aired:
# Use air date for shows with existing history
return aired, aired + "T20:00:00", "airdate"
else:
# Fallback to import history if no air date
return aired, import_history, "sonarr:import_history"
else:
# Recent import, use the import date
_log("DEBUG", f"Found recent import history for S{season_num:02d}E{episode_num:02d}: {import_history}")
return aired, import_history, "sonarr:import_history"
except Exception as e:
_log("DEBUG", f"Error checking Sonarr import history for S{season_num:02d}E{episode_num:02d}: {e}")
# For webhook processing with no history, use current time as dateadded (new download)
dateadded = datetime.now().isoformat()
source = "sonarr:webhook"