From e89cbca125fff6b26b0180d120a18d0d13c6f96e Mon Sep 17 00:00:00 2001 From: sbcrumb Date: Fri, 17 Oct 2025 20:47:25 -0400 Subject: [PATCH] feat:update to postgress over sqllite --- VERSION | 2 +- config/settings.py | 23 ++- core/database.py | 459 +++++++++++++++++++++++++++++++-------------- main.py | 2 +- 4 files changed, 344 insertions(+), 142 deletions(-) diff --git a/VERSION b/VERSION index 05f68fc..276cbf9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.2.25 +2.3.0 diff --git a/config/settings.py b/config/settings.py index 6a0798b..8ae7c50 100644 --- a/config/settings.py +++ b/config/settings.py @@ -60,8 +60,17 @@ class NFOGuardConfig: self.max_concurrent = self._get_int_env("MAX_CONCURRENT_SERIES", 3, 1, 10) # Database + self.db_type = os.environ.get("DB_TYPE", "sqlite").lower() self.db_path = Path(os.environ.get("DB_PATH", "/app/data/media_dates.db")) + # PostgreSQL database settings + if self.db_type == "postgresql": + self.db_host = os.environ.get("DB_HOST", "localhost") + self.db_port = self._get_int_env("DB_PORT", 5432, 1, 65535) + self.db_name = os.environ.get("DB_NAME", "nfoguard") + self.db_user = os.environ.get("DB_USER", "nfoguard") + self.db_password = os.environ.get("DB_PASSWORD", "") + # External connections self._load_external_connections() @@ -222,15 +231,21 @@ class NFOGuardConfig: return { "tv_paths": [str(p) for p in self.tv_paths], "movie_paths": [str(p) for p in self.movie_paths], - "database_path": str(self.db_path), + "database": { + "type": self.db_type, + "path": str(self.db_path) if self.db_type == "sqlite" else None, + "host": getattr(self, 'db_host', None) if self.db_type == "postgresql" else None, + "port": getattr(self, 'db_port', None) if self.db_type == "postgresql" else None, + "name": getattr(self, 'db_name', None) if self.db_type == "postgresql" else None + }, "external_apis": { "radarr": bool(self.radarr_url), "sonarr": bool(self.sonarr_url), "jellyseerr": bool(self.jellyseerr_url) }, - "database_connection": { - "type": self.db_type, - "configured": bool(self.db_type and self.db_host) + "radarr_database": { + "type": getattr(self, 'radarr_db_type', None), + "configured": bool(getattr(self, 'radarr_db_type', None) and getattr(self, 'radarr_db_host', None)) }, "performance": { "batch_delay": self.batch_delay, diff --git a/core/database.py b/core/database.py index e8b5ec3..fd2d030 100644 --- a/core/database.py +++ b/core/database.py @@ -1,34 +1,82 @@ #!/usr/bin/env python3 """ Database management for NFOGuard -Handles SQLite database operations for tracking media dates and processing history +Handles SQLite and PostgreSQL database operations for tracking media dates and processing history """ import sqlite3 import json import threading +import os from pathlib import Path from datetime import datetime -from typing import Optional, Dict, List, Any +from typing import Optional, Dict, List, Any, Union from contextlib import contextmanager +try: + import psycopg2 + import psycopg2.extras + POSTGRESQL_AVAILABLE = True +except ImportError: + psycopg2 = None + POSTGRESQL_AVAILABLE = False + class NFOGuardDatabase: - """Manages NFOGuard SQLite database operations""" + """Manages NFOGuard database operations for both SQLite and PostgreSQL""" - def __init__(self, db_path: Path): - self.db_path = Path(db_path) - self.db_path.parent.mkdir(parents=True, exist_ok=True) + def __init__(self, config=None, db_path: Optional[Path] = None): + """ + Initialize database connection + + Args: + config: Configuration object with database settings + db_path: Path for SQLite database (backward compatibility) + """ + if config: + self.db_type = config.db_type + if self.db_type == "postgresql": + if not POSTGRESQL_AVAILABLE: + raise ImportError("psycopg2 is required for PostgreSQL support") + self.db_host = config.db_host + self.db_port = config.db_port + self.db_name = config.db_name + self.db_user = config.db_user + self.db_password = config.db_password + else: + self.db_path = Path(config.db_path) + self.db_path.parent.mkdir(parents=True, exist_ok=True) + else: + # Backward compatibility + self.db_type = "sqlite" + self.db_path = Path(db_path) if db_path else Path("/app/data/media_dates.db") + self.db_path.parent.mkdir(parents=True, exist_ok=True) + self._local = threading.local() self._init_database() - def _get_connection(self) -> sqlite3.Connection: + def _get_placeholder(self) -> str: + """Get the correct parameter placeholder for this database type""" + return "%s" if self.db_type == "postgresql" else "?" + + def _get_connection(self) -> Union[sqlite3.Connection, 'psycopg2.extensions.connection']: """Get thread-local database connection""" if not hasattr(self._local, 'connection'): - self._local.connection = sqlite3.connect( - self.db_path, - check_same_thread=False, - timeout=30.0 - ) - self._local.connection.row_factory = sqlite3.Row + if self.db_type == "postgresql": + self._local.connection = psycopg2.connect( + host=self.db_host, + port=self.db_port, + database=self.db_name, + user=self.db_user, + password=self.db_password, + cursor_factory=psycopg2.extras.RealDictCursor + ) + self._local.connection.autocommit = True + else: + self._local.connection = sqlite3.connect( + self.db_path, + check_same_thread=False, + timeout=30.0 + ) + self._local.connection.row_factory = sqlite3.Row return self._local.connection @contextmanager @@ -37,9 +85,11 @@ class NFOGuardDatabase: conn = self._get_connection() try: yield conn - conn.commit() + if self.db_type == "sqlite": + conn.commit() # PostgreSQL uses autocommit except Exception: - conn.rollback() + if self.db_type == "sqlite": + conn.rollback() # PostgreSQL uses autocommit raise def _init_database(self): @@ -47,97 +97,182 @@ class NFOGuardDatabase: with self.get_connection() as conn: cursor = conn.cursor() - # Series table - cursor.execute(""" - CREATE TABLE IF NOT EXISTS series ( - imdb_id TEXT PRIMARY KEY, - path TEXT NOT NULL, - last_updated TEXT NOT NULL, - metadata TEXT - ) - """) + if self.db_type == "postgresql": + self._init_postgresql_tables(cursor) + else: + self._init_sqlite_tables(cursor) - # Episodes table - cursor.execute(""" - CREATE TABLE IF NOT EXISTS episodes ( - imdb_id TEXT NOT NULL, - season INTEGER NOT NULL, - episode INTEGER NOT NULL, - aired TEXT, - dateadded TEXT, - source TEXT, - last_updated TEXT NOT NULL, - PRIMARY KEY (imdb_id, season, episode), - FOREIGN KEY (imdb_id) REFERENCES series(imdb_id) - ) - """) - - # Movies table - cursor.execute(""" - CREATE TABLE IF NOT EXISTS movies ( - imdb_id TEXT PRIMARY KEY, - path TEXT NOT NULL, - released TEXT, - dateadded TEXT, - source TEXT, - last_updated TEXT NOT NULL - ) - """) - - # Processing history table - cursor.execute(""" - CREATE TABLE IF NOT EXISTS processing_history ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - imdb_id TEXT NOT NULL, - media_type TEXT NOT NULL, - event_type TEXT NOT NULL, - processed_at TEXT NOT NULL, - details TEXT - ) - """) - - # Add missing columns if they don't exist (migration) - # Check current schema and add missing columns - cursor.execute("PRAGMA table_info(movies)") - movie_columns = [row[1] for row in cursor.fetchall()] - - cursor.execute("PRAGMA table_info(episodes)") - episode_columns = [row[1] for row in cursor.fetchall()] - - # Add missing columns to movies table - if 'path' not in movie_columns: - cursor.execute("ALTER TABLE movies ADD COLUMN path TEXT") - cursor.execute("UPDATE movies SET path = '/unknown/path/' || imdb_id WHERE path IS NULL") - - if 'has_video_file' not in movie_columns: - cursor.execute("ALTER TABLE movies ADD COLUMN has_video_file BOOLEAN DEFAULT FALSE") - - if 'last_updated' not in movie_columns: - cursor.execute("ALTER TABLE movies ADD COLUMN last_updated TEXT") - cursor.execute("UPDATE movies SET last_updated = datetime('now') WHERE last_updated IS NULL") - - # Add missing columns to episodes table - if 'has_video_file' not in episode_columns: - cursor.execute("ALTER TABLE episodes ADD COLUMN has_video_file BOOLEAN DEFAULT FALSE") - - if 'last_updated' not in episode_columns: - cursor.execute("ALTER TABLE episodes ADD COLUMN last_updated TEXT") - cursor.execute("UPDATE episodes SET last_updated = datetime('now') WHERE last_updated IS NULL") - - # Create indexes - cursor.execute("CREATE INDEX IF NOT EXISTS idx_episodes_imdb ON episodes(imdb_id)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_episodes_video ON episodes(has_video_file)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_movies_video ON movies(has_video_file)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_history_imdb ON processing_history(imdb_id)") + print(f"✅ Database initialized: {self.db_type.upper()}") + + def _init_postgresql_tables(self, cursor): + """Initialize PostgreSQL tables""" + # Series table + cursor.execute(""" + CREATE TABLE IF NOT EXISTS series ( + imdb_id VARCHAR(20) PRIMARY KEY, + path TEXT NOT NULL, + last_updated TIMESTAMP NOT NULL, + metadata JSONB + ) + """) + + # Episodes table + cursor.execute(""" + CREATE TABLE IF NOT EXISTS episodes ( + imdb_id VARCHAR(20) NOT NULL, + season INTEGER NOT NULL, + episode INTEGER NOT NULL, + aired DATE, + dateadded TIMESTAMP, + source VARCHAR(100), + last_updated TIMESTAMP NOT NULL, + PRIMARY KEY (imdb_id, season, episode), + FOREIGN KEY (imdb_id) REFERENCES series(imdb_id) + ) + """) + + # Movies table + cursor.execute(""" + CREATE TABLE IF NOT EXISTS movies ( + imdb_id VARCHAR(20) PRIMARY KEY, + path TEXT NOT NULL, + released DATE, + dateadded TIMESTAMP, + source VARCHAR(100), + last_updated TIMESTAMP NOT NULL, + has_video_file INTEGER DEFAULT 0 + ) + """) + + # Processing history table + cursor.execute(""" + CREATE TABLE IF NOT EXISTS processing_history ( + id SERIAL PRIMARY KEY, + imdb_id VARCHAR(20) NOT NULL, + media_type VARCHAR(20) NOT NULL, + event_type VARCHAR(50) NOT NULL, + processed_at TIMESTAMP NOT NULL, + details TEXT + ) + """) + + # Create indexes for PostgreSQL + cursor.execute("CREATE INDEX IF NOT EXISTS idx_episodes_imdb ON episodes(imdb_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_episodes_video ON episodes(has_video_file)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_movies_video ON movies(has_video_file)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_history_imdb ON processing_history(imdb_id)") + + def _init_sqlite_tables(self, cursor): + """Initialize SQLite tables""" + # Series table + cursor.execute(""" + CREATE TABLE IF NOT EXISTS series ( + imdb_id TEXT PRIMARY KEY, + path TEXT NOT NULL, + last_updated TEXT NOT NULL, + metadata TEXT + ) + """) + + # Episodes table + cursor.execute(""" + CREATE TABLE IF NOT EXISTS episodes ( + imdb_id TEXT NOT NULL, + season INTEGER NOT NULL, + episode INTEGER NOT NULL, + aired TEXT, + dateadded TEXT, + source TEXT, + last_updated TEXT NOT NULL, + has_video_file BOOLEAN DEFAULT FALSE, + PRIMARY KEY (imdb_id, season, episode), + FOREIGN KEY (imdb_id) REFERENCES series(imdb_id) + ) + """) + + # Movies table + cursor.execute(""" + CREATE TABLE IF NOT EXISTS movies ( + imdb_id TEXT PRIMARY KEY, + path TEXT NOT NULL, + released TEXT, + dateadded TEXT, + source TEXT, + last_updated TEXT NOT NULL, + has_video_file BOOLEAN DEFAULT FALSE + ) + """) + + # Processing history table + cursor.execute(""" + CREATE TABLE IF NOT EXISTS processing_history ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + imdb_id TEXT NOT NULL, + media_type TEXT NOT NULL, + event_type TEXT NOT NULL, + processed_at TEXT NOT NULL, + details TEXT + ) + """) + + # Handle SQLite migrations for existing databases + self._handle_sqlite_migrations(cursor) + + # Create indexes for SQLite + cursor.execute("CREATE INDEX IF NOT EXISTS idx_episodes_imdb ON episodes(imdb_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_episodes_video ON episodes(has_video_file)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_movies_video ON movies(has_video_file)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_history_imdb ON processing_history(imdb_id)") + + def _handle_sqlite_migrations(self, cursor): + """Handle SQLite database migrations for existing databases""" + # Check current schema and add missing columns + cursor.execute("PRAGMA table_info(movies)") + movie_columns = [row[1] for row in cursor.fetchall()] + + cursor.execute("PRAGMA table_info(episodes)") + episode_columns = [row[1] for row in cursor.fetchall()] + + # Add missing columns to movies table + if 'path' not in movie_columns: + cursor.execute("ALTER TABLE movies ADD COLUMN path TEXT") + cursor.execute("UPDATE movies SET path = '/unknown/path/' || imdb_id WHERE path IS NULL") + + if 'has_video_file' not in movie_columns: + cursor.execute("ALTER TABLE movies ADD COLUMN has_video_file BOOLEAN DEFAULT FALSE") + + if 'last_updated' not in movie_columns: + cursor.execute("ALTER TABLE movies ADD COLUMN last_updated TEXT") + cursor.execute("UPDATE movies SET last_updated = datetime('now') WHERE last_updated IS NULL") + + # Add missing columns to episodes table + if 'has_video_file' not in episode_columns: + cursor.execute("ALTER TABLE episodes ADD COLUMN has_video_file BOOLEAN DEFAULT FALSE") + + if 'last_updated' not in episode_columns: + cursor.execute("ALTER TABLE episodes ADD COLUMN last_updated TEXT") + cursor.execute("UPDATE episodes SET last_updated = datetime('now') WHERE last_updated IS NULL") def upsert_series(self, imdb_id: str, path: str, metadata: Optional[Dict] = None): """Insert or update series record""" with self.get_connection() as conn: cursor = conn.cursor() - cursor.execute(""" - INSERT OR REPLACE INTO series (imdb_id, path, last_updated, metadata) - VALUES (?, ?, ?, ?) - """, (imdb_id, path, datetime.utcnow().isoformat(), json.dumps(metadata) if metadata else None)) + timestamp = datetime.utcnow() + + if self.db_type == "postgresql": + cursor.execute(""" + INSERT INTO series (imdb_id, path, last_updated, metadata) + VALUES (%s, %s, %s, %s) + ON CONFLICT (imdb_id) DO UPDATE SET + path = EXCLUDED.path, + last_updated = EXCLUDED.last_updated, + metadata = EXCLUDED.metadata + """, (imdb_id, path, timestamp, json.dumps(metadata) if metadata else None)) + else: + cursor.execute(""" + INSERT OR REPLACE INTO series (imdb_id, path, last_updated, metadata) + VALUES (?, ?, ?, ?) + """, (imdb_id, path, timestamp.isoformat(), json.dumps(metadata) if metadata else None)) def upsert_episode_date(self, imdb_id: str, season: int, episode: int, aired: Optional[str], dateadded: Optional[str], @@ -145,30 +280,56 @@ class NFOGuardDatabase: """Insert or update episode date record""" with self.get_connection() as conn: cursor = conn.cursor() - cursor.execute(""" - INSERT OR REPLACE INTO episodes - (imdb_id, season, episode, aired, dateadded, source, has_video_file, last_updated) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - """, (imdb_id, season, episode, aired, dateadded, source, has_video_file, datetime.utcnow().isoformat())) + timestamp = datetime.utcnow() + + if self.db_type == "postgresql": + cursor.execute(""" + INSERT INTO episodes + (imdb_id, season, episode, aired, dateadded, source, has_video_file, last_updated) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (imdb_id, season, episode) DO UPDATE SET + aired = EXCLUDED.aired, + dateadded = EXCLUDED.dateadded, + source = EXCLUDED.source, + has_video_file = EXCLUDED.has_video_file, + last_updated = EXCLUDED.last_updated + """, (imdb_id, season, episode, aired, dateadded, source, has_video_file, timestamp)) + else: + cursor.execute(""" + INSERT OR REPLACE INTO episodes + (imdb_id, season, episode, aired, dateadded, source, has_video_file, last_updated) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, (imdb_id, season, episode, aired, dateadded, source, has_video_file, timestamp.isoformat())) def upsert_movie(self, imdb_id: str, path: str): """Insert or update movie record""" with self.get_connection() as conn: cursor = conn.cursor() - try: + timestamp = datetime.utcnow() + + if self.db_type == "postgresql": cursor.execute(""" - INSERT OR REPLACE INTO movies (imdb_id, path, last_updated) - VALUES (?, ?, ?) - """, (imdb_id, path, datetime.utcnow().isoformat())) - except sqlite3.OperationalError as e: - if "no column named path" in str(e): - # Fallback for databases without path column - just insert imdb_id + INSERT INTO movies (imdb_id, path, last_updated) + VALUES (%s, %s, %s) + ON CONFLICT (imdb_id) DO UPDATE SET + path = EXCLUDED.path, + last_updated = EXCLUDED.last_updated + """, (imdb_id, path, timestamp)) + else: + try: cursor.execute(""" - INSERT OR REPLACE INTO movies (imdb_id, last_updated) - VALUES (?, ?) - """, (imdb_id, datetime.utcnow().isoformat())) - else: - raise + INSERT OR REPLACE INTO movies (imdb_id, path, last_updated) + VALUES (?, ?, ?) + """, (imdb_id, path, timestamp.isoformat())) + except sqlite3.OperationalError as e: + if "no column named path" in str(e): + # Fallback for databases without path column - just insert imdb_id + cursor.execute(""" + INSERT OR REPLACE INTO movies (imdb_id, last_updated) + VALUES (?, ?) + """, (imdb_id, timestamp.isoformat())) + else: + raise def upsert_movie_dates(self, imdb_id: str, released: Optional[str], dateadded: Optional[str], source: str, has_video_file: bool = False): @@ -176,18 +337,37 @@ class NFOGuardDatabase: print(f"🔍 DATABASE UPSERT: imdb_id={imdb_id}, dateadded={dateadded}, source={source}") with self.get_connection() as conn: cursor = conn.cursor() - # Use INSERT OR REPLACE to ensure we always update the dates properly - cursor.execute(""" - INSERT OR REPLACE INTO movies (imdb_id, path, released, dateadded, source, has_video_file, last_updated) - VALUES ( - ?, - COALESCE((SELECT path FROM movies WHERE imdb_id = ?), 'unknown'), - ?, ?, ?, ?, ? - ) - """, (imdb_id, imdb_id, released, dateadded, source, has_video_file, datetime.utcnow().isoformat())) + timestamp = datetime.utcnow() + + if self.db_type == "postgresql": + # PostgreSQL version with UPSERT + cursor.execute(""" + INSERT INTO movies (imdb_id, path, released, dateadded, source, has_video_file, last_updated) + VALUES (%s, COALESCE((SELECT path FROM movies WHERE imdb_id = %s), 'unknown'), %s, %s, %s, %s, %s) + ON CONFLICT (imdb_id) DO UPDATE SET + released = EXCLUDED.released, + dateadded = EXCLUDED.dateadded, + source = EXCLUDED.source, + has_video_file = EXCLUDED.has_video_file, + last_updated = EXCLUDED.last_updated + """, (imdb_id, imdb_id, released, dateadded, source, has_video_file, timestamp)) + + # Debug: Check what was actually saved + cursor.execute("SELECT dateadded, source FROM movies WHERE imdb_id = %s", (imdb_id,)) + else: + # SQLite version with INSERT OR REPLACE + cursor.execute(""" + INSERT OR REPLACE INTO movies (imdb_id, path, released, dateadded, source, has_video_file, last_updated) + VALUES ( + ?, + COALESCE((SELECT path FROM movies WHERE imdb_id = ?), 'unknown'), + ?, ?, ?, ?, ? + ) + """, (imdb_id, imdb_id, released, dateadded, source, has_video_file, timestamp.isoformat())) + + # Debug: Check what was actually saved + cursor.execute("SELECT dateadded, source FROM movies WHERE imdb_id = ?", (imdb_id,)) - # Debug: Check what was actually saved - cursor.execute("SELECT dateadded, source FROM movies WHERE imdb_id = ?", (imdb_id,)) result = cursor.fetchone() print(f"🔍 DATABASE VERIFY: After upsert, found dateadded={result[0] if result else 'NOT_FOUND'}, source={result[1] if result else 'NOT_FOUND'}") @@ -196,11 +376,16 @@ class NFOGuardDatabase: with self.get_connection() as conn: cursor = conn.cursor() - query = "SELECT * FROM episodes WHERE imdb_id = ?" - params = [imdb_id] - - if has_video_file_only: - query += " AND has_video_file = TRUE" + if self.db_type == "postgresql": + query = "SELECT * FROM episodes WHERE imdb_id = %s" + params = [imdb_id] + if has_video_file_only: + query += " AND has_video_file = TRUE" + else: + query = "SELECT * FROM episodes WHERE imdb_id = ?" + params = [imdb_id] + if has_video_file_only: + query += " AND has_video_file = TRUE" cursor.execute(query, params) return [dict(row) for row in cursor.fetchall()] @@ -209,9 +394,10 @@ class NFOGuardDatabase: """Get episode date record""" with self.get_connection() as conn: cursor = conn.cursor() - cursor.execute(""" + ph = self._get_placeholder() + cursor.execute(f""" SELECT * FROM episodes - WHERE imdb_id = ? AND season = ? AND episode = ? + WHERE imdb_id = {ph} AND season = {ph} AND episode = {ph} """, (imdb_id, season, episode)) row = cursor.fetchone() @@ -221,7 +407,8 @@ class NFOGuardDatabase: """Get movie date record""" with self.get_connection() as conn: cursor = conn.cursor() - cursor.execute("SELECT * FROM movies WHERE imdb_id = ?", (imdb_id,)) + ph = self._get_placeholder() + cursor.execute(f"SELECT * FROM movies WHERE imdb_id = {ph}", (imdb_id,)) row = cursor.fetchone() return dict(row) if row else None diff --git a/main.py b/main.py index a6d91c1..8628760 100644 --- a/main.py +++ b/main.py @@ -86,7 +86,7 @@ def initialize_components(): start_time = datetime.now(timezone.utc) # Initialize core components - db = NFOGuardDatabase(config.db_path) + db = NFOGuardDatabase(config=config) nfo_manager = NFOManager(config.manager_brand, config.debug) path_mapper = PathMapper(config)