diff --git a/VERSION b/VERSION index aedc15b..fe16b34 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5.3 +2.5.4 diff --git a/core/database.py b/core/database.py index b64c2d8..ad4311e 100644 --- a/core/database.py +++ b/core/database.py @@ -1,126 +1,82 @@ #!/usr/bin/env python3 """ -Database management for NFOGuard -Handles SQLite and PostgreSQL database operations for tracking media dates and processing history +PostgreSQL database management for NFOGuard +Handles database operations for tracking media dates and processing history """ -import sqlite3 import json import threading -import os -from pathlib import Path from datetime import datetime -from typing import Optional, Dict, List, Any, Union +from typing import Optional, Dict, List, Any from contextlib import contextmanager -try: - import psycopg2 - import psycopg2.extras - POSTGRESQL_AVAILABLE = True -except ImportError: - psycopg2 = None - POSTGRESQL_AVAILABLE = False +import psycopg2 +import psycopg2.extras class NFOGuardDatabase: - """Manages NFOGuard database operations for both SQLite and PostgreSQL""" + """PostgreSQL database manager for NFOGuard media tracking and processing history""" - def __init__(self, config=None, db_path: Optional[Path] = None): + def __init__(self, config): """ - Initialize database connection + Initialize PostgreSQL database connection Args: config: Configuration object with database settings - db_path: Path for SQLite database (backward compatibility) """ - if config: - self.db_type = config.db_type - if self.db_type == "postgresql": - if not POSTGRESQL_AVAILABLE: - raise ImportError("psycopg2 is required for PostgreSQL support") - self.db_host = config.db_host - self.db_port = config.db_port - self.db_name = config.db_name - self.db_user = config.db_user - self.db_password = config.db_password - else: - self.db_path = Path(config.db_path) - self.db_path.parent.mkdir(parents=True, exist_ok=True) - else: - # Backward compatibility - self.db_type = "sqlite" - self.db_path = Path(db_path) if db_path else Path("/app/data/media_dates.db") - self.db_path.parent.mkdir(parents=True, exist_ok=True) + if not config: + raise ValueError("PostgreSQL configuration is required") + self.db_host = config.db_host + self.db_port = config.db_port + self.db_name = config.db_name + self.db_user = config.db_user + self.db_password = config.db_password self._local = threading.local() self._init_database() - def _get_placeholder(self) -> str: - """Get the correct parameter placeholder for this database type""" - return "%s" if self.db_type == "postgresql" else "?" - def _get_connection(self) -> Union[sqlite3.Connection, 'psycopg2.extensions.connection']: - """Get thread-local database connection""" + def _get_connection(self) -> 'psycopg2.extensions.connection': + """Get thread-local PostgreSQL database connection""" if not hasattr(self._local, 'connection'): - if self.db_type == "postgresql": - self._local.connection = psycopg2.connect( - host=self.db_host, - port=self.db_port, - database=self.db_name, - user=self.db_user, - password=self.db_password, - cursor_factory=psycopg2.extras.RealDictCursor - ) - self._local.connection.autocommit = True - else: - self._local.connection = sqlite3.connect( - self.db_path, - check_same_thread=False, - timeout=30.0 - ) - self._local.connection.row_factory = sqlite3.Row + self._local.connection = psycopg2.connect( + host=self.db_host, + port=self.db_port, + database=self.db_name, + user=self.db_user, + password=self.db_password, + cursor_factory=psycopg2.extras.RealDictCursor + ) + self._local.connection.autocommit = True return self._local.connection def _get_first_value(self, row): - """Get first value from row, handling both dict and tuple types""" - if self.db_type == "postgresql": - # RealDictCursor returns dict-like objects - return list(row.values())[0] if row else None - else: - # SQLite returns tuples - return row[0] if row else None + """Get first value from row from PostgreSQL RealDictCursor""" + # RealDictCursor returns dict-like objects + return list(row.values())[0] if row else None @contextmanager def get_connection(self): - """Context manager for database connections""" + """Context manager for PostgreSQL database connections""" conn = self._get_connection() try: yield conn - if self.db_type == "sqlite": - conn.commit() # PostgreSQL uses autocommit + # PostgreSQL uses autocommit - no manual commit needed except Exception: - if self.db_type == "sqlite": - conn.rollback() # PostgreSQL uses autocommit + # PostgreSQL uses autocommit - no manual rollback needed raise def _init_database(self): - """Initialize database tables with migration support""" + """Initialize PostgreSQL database tables""" with self.get_connection() as conn: cursor = conn.cursor() - - if self.db_type == "postgresql": - self._init_postgresql_tables(cursor) - else: - self._init_sqlite_tables(cursor) + self._init_postgresql_tables(cursor) # Test the connection works and verify autocommit cursor.execute("SELECT 1") - if self.db_type == "postgresql": - print(f"✅ Database initialized and connection verified: {self.db_type.upper()}") - print(f"🔍 PostgreSQL autocommit status: {conn.autocommit}") - else: - print(f"✅ Database initialized and connection verified: {self.db_type.upper()}") + print(f"✅ PostgreSQL database initialized and connection verified") + print(f"🔍 Autocommit status: {conn.autocommit}") def _init_postgresql_tables(self, cursor): - """Initialize PostgreSQL tables""" + """Initialize database tables""" # Series table cursor.execute(""" CREATE TABLE IF NOT EXISTS series ( @@ -177,118 +133,20 @@ class NFOGuardDatabase: cursor.execute("CREATE INDEX IF NOT EXISTS idx_episodes_video ON episodes(has_video_file)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_movies_video ON movies(has_video_file)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_history_imdb ON processing_history(imdb_id)") - - def _init_sqlite_tables(self, cursor): - """Initialize SQLite tables""" - # Series table - cursor.execute(""" - CREATE TABLE IF NOT EXISTS series ( - imdb_id TEXT PRIMARY KEY, - path TEXT NOT NULL, - last_updated TEXT NOT NULL, - metadata TEXT - ) - """) - - # Episodes table - cursor.execute(""" - CREATE TABLE IF NOT EXISTS episodes ( - imdb_id TEXT NOT NULL, - season INTEGER NOT NULL, - episode INTEGER NOT NULL, - aired TEXT, - dateadded TEXT, - source TEXT, - last_updated TEXT NOT NULL, - has_video_file BOOLEAN DEFAULT FALSE, - PRIMARY KEY (imdb_id, season, episode), - FOREIGN KEY (imdb_id) REFERENCES series(imdb_id) - ) - """) - - # Movies table - cursor.execute(""" - CREATE TABLE IF NOT EXISTS movies ( - imdb_id TEXT PRIMARY KEY, - path TEXT NOT NULL, - released TEXT, - dateadded TEXT, - source TEXT, - last_updated TEXT NOT NULL, - has_video_file BOOLEAN DEFAULT FALSE - ) - """) - - # Processing history table - cursor.execute(""" - CREATE TABLE IF NOT EXISTS processing_history ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - imdb_id TEXT NOT NULL, - media_type TEXT NOT NULL, - event_type TEXT NOT NULL, - processed_at TEXT NOT NULL, - details TEXT - ) - """) - - # Handle SQLite migrations for existing databases - self._handle_sqlite_migrations(cursor) - - # Create indexes for SQLite - cursor.execute("CREATE INDEX IF NOT EXISTS idx_episodes_imdb ON episodes(imdb_id)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_episodes_video ON episodes(has_video_file)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_movies_video ON movies(has_video_file)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_history_imdb ON processing_history(imdb_id)") - - def _handle_sqlite_migrations(self, cursor): - """Handle SQLite database migrations for existing databases""" - # Check current schema and add missing columns - cursor.execute("PRAGMA table_info(movies)") - movie_columns = [row[1] for row in cursor.fetchall()] - - cursor.execute("PRAGMA table_info(episodes)") - episode_columns = [row[1] for row in cursor.fetchall()] - - # Add missing columns to movies table - if 'path' not in movie_columns: - cursor.execute("ALTER TABLE movies ADD COLUMN path TEXT") - cursor.execute("UPDATE movies SET path = '/unknown/path/' || imdb_id WHERE path IS NULL") - - if 'has_video_file' not in movie_columns: - cursor.execute("ALTER TABLE movies ADD COLUMN has_video_file BOOLEAN DEFAULT FALSE") - - if 'last_updated' not in movie_columns: - cursor.execute("ALTER TABLE movies ADD COLUMN last_updated TEXT") - cursor.execute("UPDATE movies SET last_updated = datetime('now') WHERE last_updated IS NULL") - - # Add missing columns to episodes table - if 'has_video_file' not in episode_columns: - cursor.execute("ALTER TABLE episodes ADD COLUMN has_video_file BOOLEAN DEFAULT FALSE") - - if 'last_updated' not in episode_columns: - cursor.execute("ALTER TABLE episodes ADD COLUMN last_updated TEXT") - cursor.execute("UPDATE episodes SET last_updated = datetime('now') WHERE last_updated IS NULL") - def upsert_series(self, imdb_id: str, path: str, metadata: Optional[Dict] = None): """Insert or update series record""" with self.get_connection() as conn: cursor = conn.cursor() timestamp = datetime.utcnow() - if self.db_type == "postgresql": - cursor.execute(""" - INSERT INTO series (imdb_id, path, last_updated, metadata) - VALUES (%s, %s, %s, %s) - ON CONFLICT (imdb_id) DO UPDATE SET - path = EXCLUDED.path, - last_updated = EXCLUDED.last_updated, - metadata = EXCLUDED.metadata - """, (imdb_id, path, timestamp, json.dumps(metadata) if metadata else None)) - else: - cursor.execute(""" - INSERT OR REPLACE INTO series (imdb_id, path, last_updated, metadata) - VALUES (?, ?, ?, ?) - """, (imdb_id, path, timestamp.isoformat(), json.dumps(metadata) if metadata else None)) + cursor.execute(""" + INSERT INTO series (imdb_id, path, last_updated, metadata) + VALUES (%s, %s, %s, %s) + ON CONFLICT (imdb_id) DO UPDATE SET + path = EXCLUDED.path, + last_updated = EXCLUDED.last_updated, + metadata = EXCLUDED.metadata + """, (imdb_id, path, timestamp, json.dumps(metadata) if metadata else None)) def upsert_episode_date(self, imdb_id: str, season: int, episode: int, aired: Optional[str], dateadded: Optional[str], @@ -298,25 +156,18 @@ class NFOGuardDatabase: cursor = conn.cursor() timestamp = datetime.utcnow() - if self.db_type == "postgresql": - cursor.execute(""" - INSERT INTO episodes - (imdb_id, season, episode, aired, dateadded, source, has_video_file, last_updated) - VALUES (%s, %s, %s, %s, %s, %s, %s, %s) - ON CONFLICT (imdb_id, season, episode) DO UPDATE SET - aired = EXCLUDED.aired, - dateadded = EXCLUDED.dateadded, - source = EXCLUDED.source, - has_video_file = EXCLUDED.has_video_file, - last_updated = EXCLUDED.last_updated - """, (imdb_id, season, episode, aired, dateadded, source, has_video_file, timestamp)) - print(f"🔍 DEBUG: PostgreSQL upsert executed for {imdb_id} S{season:02d}E{episode:02d}, rows affected: {cursor.rowcount}") - else: - cursor.execute(""" - INSERT OR REPLACE INTO episodes - (imdb_id, season, episode, aired, dateadded, source, has_video_file, last_updated) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - """, (imdb_id, season, episode, aired, dateadded, source, has_video_file, timestamp.isoformat())) + cursor.execute(""" + INSERT INTO episodes + (imdb_id, season, episode, aired, dateadded, source, has_video_file, last_updated) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (imdb_id, season, episode) DO UPDATE SET + aired = EXCLUDED.aired, + dateadded = EXCLUDED.dateadded, + source = EXCLUDED.source, + has_video_file = EXCLUDED.has_video_file, + last_updated = EXCLUDED.last_updated + """, (imdb_id, season, episode, aired, dateadded, source, has_video_file, timestamp)) + print(f"🔍 DEBUG: PostgreSQL upsert executed for {imdb_id} S{season:02d}E{episode:02d}, rows affected: {cursor.rowcount}") def upsert_movie(self, imdb_id: str, path: str): """Insert or update movie record""" @@ -324,29 +175,13 @@ class NFOGuardDatabase: cursor = conn.cursor() timestamp = datetime.utcnow() - if self.db_type == "postgresql": - cursor.execute(""" - INSERT INTO movies (imdb_id, path, last_updated) - VALUES (%s, %s, %s) - ON CONFLICT (imdb_id) DO UPDATE SET - path = EXCLUDED.path, - last_updated = EXCLUDED.last_updated - """, (imdb_id, path, timestamp)) - else: - try: - cursor.execute(""" - INSERT OR REPLACE INTO movies (imdb_id, path, last_updated) - VALUES (?, ?, ?) - """, (imdb_id, path, timestamp.isoformat())) - except sqlite3.OperationalError as e: - if "no column named path" in str(e): - # Fallback for databases without path column - just insert imdb_id - cursor.execute(""" - INSERT OR REPLACE INTO movies (imdb_id, last_updated) - VALUES (?, ?) - """, (imdb_id, timestamp.isoformat())) - else: - raise + cursor.execute(""" + INSERT INTO movies (imdb_id, path, last_updated) + VALUES (%s, %s, %s) + ON CONFLICT (imdb_id) DO UPDATE SET + path = EXCLUDED.path, + last_updated = EXCLUDED.last_updated + """, (imdb_id, path, timestamp)) def upsert_movie_dates(self, imdb_id: str, released: Optional[str], dateadded: Optional[str], source: str, has_video_file: bool = False): @@ -356,35 +191,19 @@ class NFOGuardDatabase: cursor = conn.cursor() timestamp = datetime.utcnow() - if self.db_type == "postgresql": - # PostgreSQL version with UPSERT - cursor.execute(""" - INSERT INTO movies (imdb_id, path, released, dateadded, source, has_video_file, last_updated) - VALUES (%s, COALESCE((SELECT path FROM movies WHERE imdb_id = %s), 'unknown'), %s, %s, %s, %s, %s) - ON CONFLICT (imdb_id) DO UPDATE SET - released = EXCLUDED.released, - dateadded = EXCLUDED.dateadded, - source = EXCLUDED.source, - has_video_file = EXCLUDED.has_video_file, - last_updated = EXCLUDED.last_updated - """, (imdb_id, imdb_id, released, dateadded, source, has_video_file, timestamp)) - - # Debug: Check what was actually saved - cursor.execute("SELECT dateadded, source FROM movies WHERE imdb_id = %s", (imdb_id,)) - else: - # SQLite version with INSERT OR REPLACE - cursor.execute(""" - INSERT OR REPLACE INTO movies (imdb_id, path, released, dateadded, source, has_video_file, last_updated) - VALUES ( - ?, - COALESCE((SELECT path FROM movies WHERE imdb_id = ?), 'unknown'), - ?, ?, ?, ?, ? - ) - """, (imdb_id, imdb_id, released, dateadded, source, has_video_file, timestamp.isoformat())) - - # Debug: Check what was actually saved - cursor.execute("SELECT dateadded, source FROM movies WHERE imdb_id = ?", (imdb_id,)) + cursor.execute(""" + INSERT INTO movies (imdb_id, path, released, dateadded, source, has_video_file, last_updated) + VALUES (%s, COALESCE((SELECT path FROM movies WHERE imdb_id = %s), 'unknown'), %s, %s, %s, %s, %s) + ON CONFLICT (imdb_id) DO UPDATE SET + released = EXCLUDED.released, + dateadded = EXCLUDED.dateadded, + source = EXCLUDED.source, + has_video_file = EXCLUDED.has_video_file, + last_updated = EXCLUDED.last_updated + """, (imdb_id, imdb_id, released, dateadded, source, has_video_file, timestamp)) + # Debug: Check what was actually saved + cursor.execute("SELECT dateadded, source FROM movies WHERE imdb_id = %s", (imdb_id,)) result = cursor.fetchone() print(f"🔍 DATABASE VERIFY: After upsert, found dateadded={result[0] if result else 'NOT_FOUND'}, source={result[1] if result else 'NOT_FOUND'}") @@ -393,16 +212,10 @@ class NFOGuardDatabase: with self.get_connection() as conn: cursor = conn.cursor() - if self.db_type == "postgresql": - query = "SELECT * FROM episodes WHERE imdb_id = %s" - params = [imdb_id] - if has_video_file_only: - query += " AND has_video_file = TRUE" - else: - query = "SELECT * FROM episodes WHERE imdb_id = ?" - params = [imdb_id] - if has_video_file_only: - query += " AND has_video_file = TRUE" + query = "SELECT * FROM episodes WHERE imdb_id = %s" + params = [imdb_id] + if has_video_file_only: + query += " AND has_video_file = TRUE" cursor.execute(query, params) return [dict(row) for row in cursor.fetchall()] @@ -411,10 +224,9 @@ class NFOGuardDatabase: """Get episode date record""" with self.get_connection() as conn: cursor = conn.cursor() - ph = self._get_placeholder() - cursor.execute(f""" + cursor.execute(""" SELECT * FROM episodes - WHERE imdb_id = {ph} AND season = {ph} AND episode = {ph} + WHERE imdb_id = %s AND season = %s AND episode = %s """, (imdb_id, season, episode)) row = cursor.fetchone() @@ -424,8 +236,7 @@ class NFOGuardDatabase: """Get movie date record""" with self.get_connection() as conn: cursor = conn.cursor() - ph = self._get_placeholder() - cursor.execute(f"SELECT * FROM movies WHERE imdb_id = {ph}", (imdb_id,)) + cursor.execute("SELECT * FROM movies WHERE imdb_id = %s", (imdb_id,)) row = cursor.fetchone() return dict(row) if row else None @@ -434,21 +245,16 @@ class NFOGuardDatabase: """Add processing history entry""" with self.get_connection() as conn: cursor = conn.cursor() - placeholder = self._get_placeholder() - cursor.execute(f""" + cursor.execute(""" INSERT INTO processing_history (imdb_id, media_type, event_type, processed_at, details) - VALUES ({placeholder}, {placeholder}, {placeholder}, {placeholder}, {placeholder}) + VALUES (%s, %s, %s, %s, %s) """, (imdb_id, media_type, event_type, datetime.utcnow().isoformat(), json.dumps(details) if details else None)) def get_stats(self) -> Dict[str, Any]: """Get database statistics""" with self.get_connection() as conn: - # Use regular cursor for simple COUNT queries (not RealDictCursor) - if self.db_type == "postgresql": - cursor = conn.cursor() # Regular cursor for PostgreSQL - else: - cursor = conn.cursor() # SQLite cursor + cursor = conn.cursor() # Regular cursor for PostgreSQL # Series stats cursor.execute("SELECT COUNT(*) FROM series") @@ -472,15 +278,10 @@ class NFOGuardDatabase: cursor.execute("SELECT COUNT(*) FROM processing_history") history_count = self._get_first_value(cursor.fetchone()) - # Database size calculation - if self.db_type == "postgresql": - # PostgreSQL: Query pg_database_size - cursor.execute("SELECT pg_database_size(%s)", (self.db_name,)) - db_size_bytes = self._get_first_value(cursor.fetchone()) - db_size_mb = round(db_size_bytes / 1024 / 1024, 2) if db_size_bytes else 0 - else: - # SQLite: File size - db_size_mb = round(self.db_path.stat().st_size / 1024 / 1024, 2) + # Database size calculation for PostgreSQL + cursor.execute("SELECT pg_database_size(%s)", (self.db_name,)) + db_size_bytes = self._get_first_value(cursor.fetchone()) + db_size_mb = round(db_size_bytes / 1024 / 1024, 2) if db_size_bytes else 0 return { "series_count": series_count, @@ -490,7 +291,7 @@ class NFOGuardDatabase: "movies_with_video": movies_with_video, "processing_history_count": history_count, "database_size_mb": db_size_mb, - "database_type": self.db_type + "database_type": "postgresql" } def close(self): diff --git a/processors/tv_processor.py b/processors/tv_processor.py index edc837f..c35c728 100644 --- a/processors/tv_processor.py +++ b/processors/tv_processor.py @@ -921,8 +921,8 @@ class TVProcessor: """ Get episode date for webhook processing with correct priority: 1. Check existing NFOGuard database entry (preserve previous import dates) - 2. If not found, use current time (webhook = new download) - 3. Get aired date from Sonarr/TMDB for reference + 2. Check Sonarr import history - if old history exists, prefer air date + 3. If no history, use current time (webhook = new download) """ # Check if we already have this episode in our database (preserve existing dates) existing = self.db.get_episode_date(imdb_id, season_num, episode_num) @@ -938,7 +938,43 @@ class TVProcessor: aired = episode_data.get('airDate') _log("DEBUG", f"Got aired date from Sonarr for S{season_num:02d}E{episode_num:02d}: {aired}") - # For webhook processing, use current time as dateadded (new download) + # NEW: Check Sonarr import history to detect if this is an existing show + episode_id = None + if series_metadata and 'episodes' in series_metadata: + episode_data = series_metadata['episodes'].get((season_num, episode_num)) + if episode_data: + episode_id = episode_data.get('id') + + if episode_id and hasattr(self, 'sonarr'): + try: + import_history = self.sonarr.get_episode_import_history(episode_id) + if import_history: + from datetime import datetime as dt + from dateutil import parser + + # Parse the import history date + history_date = parser.parse(import_history) + current_time = dt.now(history_date.tzinfo) + time_diff = current_time - history_date + + # If import history is more than 24 hours old, this is likely a re-download/upgrade + # In this case, prefer air date over webhook date + if time_diff.days > 0: + _log("INFO", f"Found old import history for S{season_num:02d}E{episode_num:02d} ({import_history}), using air date instead of webhook date") + if aired: + # Use air date for shows with existing history + return aired, aired + "T20:00:00", "airdate" + else: + # Fallback to import history if no air date + return aired, import_history, "sonarr:import_history" + else: + # Recent import, use the import date + _log("DEBUG", f"Found recent import history for S{season_num:02d}E{episode_num:02d}: {import_history}") + return aired, import_history, "sonarr:import_history" + except Exception as e: + _log("DEBUG", f"Error checking Sonarr import history for S{season_num:02d}E{episode_num:02d}: {e}") + + # For webhook processing with no history, use current time as dateadded (new download) dateadded = datetime.now().isoformat() source = "sonarr:webhook"