feat:update to postgress over sqllite
Local Docker Build (Dev) / build-dev (push) Successful in 3s

This commit is contained in:
2025-10-17 20:47:25 -04:00
parent 5ecf46df92
commit e89cbca125
4 changed files with 344 additions and 142 deletions
+323 -136
View File
@@ -1,34 +1,82 @@
#!/usr/bin/env python3
"""
Database management for NFOGuard
Handles SQLite database operations for tracking media dates and processing history
Handles SQLite and PostgreSQL database operations for tracking media dates and processing history
"""
import sqlite3
import json
import threading
import os
from pathlib import Path
from datetime import datetime
from typing import Optional, Dict, List, Any
from typing import Optional, Dict, List, Any, Union
from contextlib import contextmanager
try:
import psycopg2
import psycopg2.extras
POSTGRESQL_AVAILABLE = True
except ImportError:
psycopg2 = None
POSTGRESQL_AVAILABLE = False
class NFOGuardDatabase:
"""Manages NFOGuard SQLite database operations"""
"""Manages NFOGuard database operations for both SQLite and PostgreSQL"""
def __init__(self, db_path: Path):
self.db_path = Path(db_path)
self.db_path.parent.mkdir(parents=True, exist_ok=True)
def __init__(self, config=None, db_path: Optional[Path] = None):
"""
Initialize database connection
Args:
config: Configuration object with database settings
db_path: Path for SQLite database (backward compatibility)
"""
if config:
self.db_type = config.db_type
if self.db_type == "postgresql":
if not POSTGRESQL_AVAILABLE:
raise ImportError("psycopg2 is required for PostgreSQL support")
self.db_host = config.db_host
self.db_port = config.db_port
self.db_name = config.db_name
self.db_user = config.db_user
self.db_password = config.db_password
else:
self.db_path = Path(config.db_path)
self.db_path.parent.mkdir(parents=True, exist_ok=True)
else:
# Backward compatibility
self.db_type = "sqlite"
self.db_path = Path(db_path) if db_path else Path("/app/data/media_dates.db")
self.db_path.parent.mkdir(parents=True, exist_ok=True)
self._local = threading.local()
self._init_database()
def _get_connection(self) -> sqlite3.Connection:
def _get_placeholder(self) -> str:
"""Get the correct parameter placeholder for this database type"""
return "%s" if self.db_type == "postgresql" else "?"
def _get_connection(self) -> Union[sqlite3.Connection, 'psycopg2.extensions.connection']:
"""Get thread-local database connection"""
if not hasattr(self._local, 'connection'):
self._local.connection = sqlite3.connect(
self.db_path,
check_same_thread=False,
timeout=30.0
)
self._local.connection.row_factory = sqlite3.Row
if self.db_type == "postgresql":
self._local.connection = psycopg2.connect(
host=self.db_host,
port=self.db_port,
database=self.db_name,
user=self.db_user,
password=self.db_password,
cursor_factory=psycopg2.extras.RealDictCursor
)
self._local.connection.autocommit = True
else:
self._local.connection = sqlite3.connect(
self.db_path,
check_same_thread=False,
timeout=30.0
)
self._local.connection.row_factory = sqlite3.Row
return self._local.connection
@contextmanager
@@ -37,9 +85,11 @@ class NFOGuardDatabase:
conn = self._get_connection()
try:
yield conn
conn.commit()
if self.db_type == "sqlite":
conn.commit() # PostgreSQL uses autocommit
except Exception:
conn.rollback()
if self.db_type == "sqlite":
conn.rollback() # PostgreSQL uses autocommit
raise
def _init_database(self):
@@ -47,97 +97,182 @@ class NFOGuardDatabase:
with self.get_connection() as conn:
cursor = conn.cursor()
# Series table
cursor.execute("""
CREATE TABLE IF NOT EXISTS series (
imdb_id TEXT PRIMARY KEY,
path TEXT NOT NULL,
last_updated TEXT NOT NULL,
metadata TEXT
)
""")
if self.db_type == "postgresql":
self._init_postgresql_tables(cursor)
else:
self._init_sqlite_tables(cursor)
# Episodes table
cursor.execute("""
CREATE TABLE IF NOT EXISTS episodes (
imdb_id TEXT NOT NULL,
season INTEGER NOT NULL,
episode INTEGER NOT NULL,
aired TEXT,
dateadded TEXT,
source TEXT,
last_updated TEXT NOT NULL,
PRIMARY KEY (imdb_id, season, episode),
FOREIGN KEY (imdb_id) REFERENCES series(imdb_id)
)
""")
# Movies table
cursor.execute("""
CREATE TABLE IF NOT EXISTS movies (
imdb_id TEXT PRIMARY KEY,
path TEXT NOT NULL,
released TEXT,
dateadded TEXT,
source TEXT,
last_updated TEXT NOT NULL
)
""")
# Processing history table
cursor.execute("""
CREATE TABLE IF NOT EXISTS processing_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
imdb_id TEXT NOT NULL,
media_type TEXT NOT NULL,
event_type TEXT NOT NULL,
processed_at TEXT NOT NULL,
details TEXT
)
""")
# Add missing columns if they don't exist (migration)
# Check current schema and add missing columns
cursor.execute("PRAGMA table_info(movies)")
movie_columns = [row[1] for row in cursor.fetchall()]
cursor.execute("PRAGMA table_info(episodes)")
episode_columns = [row[1] for row in cursor.fetchall()]
# Add missing columns to movies table
if 'path' not in movie_columns:
cursor.execute("ALTER TABLE movies ADD COLUMN path TEXT")
cursor.execute("UPDATE movies SET path = '/unknown/path/' || imdb_id WHERE path IS NULL")
if 'has_video_file' not in movie_columns:
cursor.execute("ALTER TABLE movies ADD COLUMN has_video_file BOOLEAN DEFAULT FALSE")
if 'last_updated' not in movie_columns:
cursor.execute("ALTER TABLE movies ADD COLUMN last_updated TEXT")
cursor.execute("UPDATE movies SET last_updated = datetime('now') WHERE last_updated IS NULL")
# Add missing columns to episodes table
if 'has_video_file' not in episode_columns:
cursor.execute("ALTER TABLE episodes ADD COLUMN has_video_file BOOLEAN DEFAULT FALSE")
if 'last_updated' not in episode_columns:
cursor.execute("ALTER TABLE episodes ADD COLUMN last_updated TEXT")
cursor.execute("UPDATE episodes SET last_updated = datetime('now') WHERE last_updated IS NULL")
# Create indexes
cursor.execute("CREATE INDEX IF NOT EXISTS idx_episodes_imdb ON episodes(imdb_id)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_episodes_video ON episodes(has_video_file)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_movies_video ON movies(has_video_file)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_history_imdb ON processing_history(imdb_id)")
print(f"✅ Database initialized: {self.db_type.upper()}")
def _init_postgresql_tables(self, cursor):
"""Initialize PostgreSQL tables"""
# Series table
cursor.execute("""
CREATE TABLE IF NOT EXISTS series (
imdb_id VARCHAR(20) PRIMARY KEY,
path TEXT NOT NULL,
last_updated TIMESTAMP NOT NULL,
metadata JSONB
)
""")
# Episodes table
cursor.execute("""
CREATE TABLE IF NOT EXISTS episodes (
imdb_id VARCHAR(20) NOT NULL,
season INTEGER NOT NULL,
episode INTEGER NOT NULL,
aired DATE,
dateadded TIMESTAMP,
source VARCHAR(100),
last_updated TIMESTAMP NOT NULL,
PRIMARY KEY (imdb_id, season, episode),
FOREIGN KEY (imdb_id) REFERENCES series(imdb_id)
)
""")
# Movies table
cursor.execute("""
CREATE TABLE IF NOT EXISTS movies (
imdb_id VARCHAR(20) PRIMARY KEY,
path TEXT NOT NULL,
released DATE,
dateadded TIMESTAMP,
source VARCHAR(100),
last_updated TIMESTAMP NOT NULL,
has_video_file INTEGER DEFAULT 0
)
""")
# Processing history table
cursor.execute("""
CREATE TABLE IF NOT EXISTS processing_history (
id SERIAL PRIMARY KEY,
imdb_id VARCHAR(20) NOT NULL,
media_type VARCHAR(20) NOT NULL,
event_type VARCHAR(50) NOT NULL,
processed_at TIMESTAMP NOT NULL,
details TEXT
)
""")
# Create indexes for PostgreSQL
cursor.execute("CREATE INDEX IF NOT EXISTS idx_episodes_imdb ON episodes(imdb_id)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_episodes_video ON episodes(has_video_file)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_movies_video ON movies(has_video_file)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_history_imdb ON processing_history(imdb_id)")
def _init_sqlite_tables(self, cursor):
"""Initialize SQLite tables"""
# Series table
cursor.execute("""
CREATE TABLE IF NOT EXISTS series (
imdb_id TEXT PRIMARY KEY,
path TEXT NOT NULL,
last_updated TEXT NOT NULL,
metadata TEXT
)
""")
# Episodes table
cursor.execute("""
CREATE TABLE IF NOT EXISTS episodes (
imdb_id TEXT NOT NULL,
season INTEGER NOT NULL,
episode INTEGER NOT NULL,
aired TEXT,
dateadded TEXT,
source TEXT,
last_updated TEXT NOT NULL,
has_video_file BOOLEAN DEFAULT FALSE,
PRIMARY KEY (imdb_id, season, episode),
FOREIGN KEY (imdb_id) REFERENCES series(imdb_id)
)
""")
# Movies table
cursor.execute("""
CREATE TABLE IF NOT EXISTS movies (
imdb_id TEXT PRIMARY KEY,
path TEXT NOT NULL,
released TEXT,
dateadded TEXT,
source TEXT,
last_updated TEXT NOT NULL,
has_video_file BOOLEAN DEFAULT FALSE
)
""")
# Processing history table
cursor.execute("""
CREATE TABLE IF NOT EXISTS processing_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
imdb_id TEXT NOT NULL,
media_type TEXT NOT NULL,
event_type TEXT NOT NULL,
processed_at TEXT NOT NULL,
details TEXT
)
""")
# Handle SQLite migrations for existing databases
self._handle_sqlite_migrations(cursor)
# Create indexes for SQLite
cursor.execute("CREATE INDEX IF NOT EXISTS idx_episodes_imdb ON episodes(imdb_id)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_episodes_video ON episodes(has_video_file)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_movies_video ON movies(has_video_file)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_history_imdb ON processing_history(imdb_id)")
def _handle_sqlite_migrations(self, cursor):
"""Handle SQLite database migrations for existing databases"""
# Check current schema and add missing columns
cursor.execute("PRAGMA table_info(movies)")
movie_columns = [row[1] for row in cursor.fetchall()]
cursor.execute("PRAGMA table_info(episodes)")
episode_columns = [row[1] for row in cursor.fetchall()]
# Add missing columns to movies table
if 'path' not in movie_columns:
cursor.execute("ALTER TABLE movies ADD COLUMN path TEXT")
cursor.execute("UPDATE movies SET path = '/unknown/path/' || imdb_id WHERE path IS NULL")
if 'has_video_file' not in movie_columns:
cursor.execute("ALTER TABLE movies ADD COLUMN has_video_file BOOLEAN DEFAULT FALSE")
if 'last_updated' not in movie_columns:
cursor.execute("ALTER TABLE movies ADD COLUMN last_updated TEXT")
cursor.execute("UPDATE movies SET last_updated = datetime('now') WHERE last_updated IS NULL")
# Add missing columns to episodes table
if 'has_video_file' not in episode_columns:
cursor.execute("ALTER TABLE episodes ADD COLUMN has_video_file BOOLEAN DEFAULT FALSE")
if 'last_updated' not in episode_columns:
cursor.execute("ALTER TABLE episodes ADD COLUMN last_updated TEXT")
cursor.execute("UPDATE episodes SET last_updated = datetime('now') WHERE last_updated IS NULL")
def upsert_series(self, imdb_id: str, path: str, metadata: Optional[Dict] = None):
"""Insert or update series record"""
with self.get_connection() as conn:
cursor = conn.cursor()
cursor.execute("""
INSERT OR REPLACE INTO series (imdb_id, path, last_updated, metadata)
VALUES (?, ?, ?, ?)
""", (imdb_id, path, datetime.utcnow().isoformat(), json.dumps(metadata) if metadata else None))
timestamp = datetime.utcnow()
if self.db_type == "postgresql":
cursor.execute("""
INSERT INTO series (imdb_id, path, last_updated, metadata)
VALUES (%s, %s, %s, %s)
ON CONFLICT (imdb_id) DO UPDATE SET
path = EXCLUDED.path,
last_updated = EXCLUDED.last_updated,
metadata = EXCLUDED.metadata
""", (imdb_id, path, timestamp, json.dumps(metadata) if metadata else None))
else:
cursor.execute("""
INSERT OR REPLACE INTO series (imdb_id, path, last_updated, metadata)
VALUES (?, ?, ?, ?)
""", (imdb_id, path, timestamp.isoformat(), json.dumps(metadata) if metadata else None))
def upsert_episode_date(self, imdb_id: str, season: int, episode: int,
aired: Optional[str], dateadded: Optional[str],
@@ -145,30 +280,56 @@ class NFOGuardDatabase:
"""Insert or update episode date record"""
with self.get_connection() as conn:
cursor = conn.cursor()
cursor.execute("""
INSERT OR REPLACE INTO episodes
(imdb_id, season, episode, aired, dateadded, source, has_video_file, last_updated)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""", (imdb_id, season, episode, aired, dateadded, source, has_video_file, datetime.utcnow().isoformat()))
timestamp = datetime.utcnow()
if self.db_type == "postgresql":
cursor.execute("""
INSERT INTO episodes
(imdb_id, season, episode, aired, dateadded, source, has_video_file, last_updated)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (imdb_id, season, episode) DO UPDATE SET
aired = EXCLUDED.aired,
dateadded = EXCLUDED.dateadded,
source = EXCLUDED.source,
has_video_file = EXCLUDED.has_video_file,
last_updated = EXCLUDED.last_updated
""", (imdb_id, season, episode, aired, dateadded, source, has_video_file, timestamp))
else:
cursor.execute("""
INSERT OR REPLACE INTO episodes
(imdb_id, season, episode, aired, dateadded, source, has_video_file, last_updated)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""", (imdb_id, season, episode, aired, dateadded, source, has_video_file, timestamp.isoformat()))
def upsert_movie(self, imdb_id: str, path: str):
"""Insert or update movie record"""
with self.get_connection() as conn:
cursor = conn.cursor()
try:
timestamp = datetime.utcnow()
if self.db_type == "postgresql":
cursor.execute("""
INSERT OR REPLACE INTO movies (imdb_id, path, last_updated)
VALUES (?, ?, ?)
""", (imdb_id, path, datetime.utcnow().isoformat()))
except sqlite3.OperationalError as e:
if "no column named path" in str(e):
# Fallback for databases without path column - just insert imdb_id
INSERT INTO movies (imdb_id, path, last_updated)
VALUES (%s, %s, %s)
ON CONFLICT (imdb_id) DO UPDATE SET
path = EXCLUDED.path,
last_updated = EXCLUDED.last_updated
""", (imdb_id, path, timestamp))
else:
try:
cursor.execute("""
INSERT OR REPLACE INTO movies (imdb_id, last_updated)
VALUES (?, ?)
""", (imdb_id, datetime.utcnow().isoformat()))
else:
raise
INSERT OR REPLACE INTO movies (imdb_id, path, last_updated)
VALUES (?, ?, ?)
""", (imdb_id, path, timestamp.isoformat()))
except sqlite3.OperationalError as e:
if "no column named path" in str(e):
# Fallback for databases without path column - just insert imdb_id
cursor.execute("""
INSERT OR REPLACE INTO movies (imdb_id, last_updated)
VALUES (?, ?)
""", (imdb_id, timestamp.isoformat()))
else:
raise
def upsert_movie_dates(self, imdb_id: str, released: Optional[str],
dateadded: Optional[str], source: str, has_video_file: bool = False):
@@ -176,18 +337,37 @@ class NFOGuardDatabase:
print(f"🔍 DATABASE UPSERT: imdb_id={imdb_id}, dateadded={dateadded}, source={source}")
with self.get_connection() as conn:
cursor = conn.cursor()
# Use INSERT OR REPLACE to ensure we always update the dates properly
cursor.execute("""
INSERT OR REPLACE INTO movies (imdb_id, path, released, dateadded, source, has_video_file, last_updated)
VALUES (
?,
COALESCE((SELECT path FROM movies WHERE imdb_id = ?), 'unknown'),
?, ?, ?, ?, ?
)
""", (imdb_id, imdb_id, released, dateadded, source, has_video_file, datetime.utcnow().isoformat()))
timestamp = datetime.utcnow()
if self.db_type == "postgresql":
# PostgreSQL version with UPSERT
cursor.execute("""
INSERT INTO movies (imdb_id, path, released, dateadded, source, has_video_file, last_updated)
VALUES (%s, COALESCE((SELECT path FROM movies WHERE imdb_id = %s), 'unknown'), %s, %s, %s, %s, %s)
ON CONFLICT (imdb_id) DO UPDATE SET
released = EXCLUDED.released,
dateadded = EXCLUDED.dateadded,
source = EXCLUDED.source,
has_video_file = EXCLUDED.has_video_file,
last_updated = EXCLUDED.last_updated
""", (imdb_id, imdb_id, released, dateadded, source, has_video_file, timestamp))
# Debug: Check what was actually saved
cursor.execute("SELECT dateadded, source FROM movies WHERE imdb_id = %s", (imdb_id,))
else:
# SQLite version with INSERT OR REPLACE
cursor.execute("""
INSERT OR REPLACE INTO movies (imdb_id, path, released, dateadded, source, has_video_file, last_updated)
VALUES (
?,
COALESCE((SELECT path FROM movies WHERE imdb_id = ?), 'unknown'),
?, ?, ?, ?, ?
)
""", (imdb_id, imdb_id, released, dateadded, source, has_video_file, timestamp.isoformat()))
# Debug: Check what was actually saved
cursor.execute("SELECT dateadded, source FROM movies WHERE imdb_id = ?", (imdb_id,))
# Debug: Check what was actually saved
cursor.execute("SELECT dateadded, source FROM movies WHERE imdb_id = ?", (imdb_id,))
result = cursor.fetchone()
print(f"🔍 DATABASE VERIFY: After upsert, found dateadded={result[0] if result else 'NOT_FOUND'}, source={result[1] if result else 'NOT_FOUND'}")
@@ -196,11 +376,16 @@ class NFOGuardDatabase:
with self.get_connection() as conn:
cursor = conn.cursor()
query = "SELECT * FROM episodes WHERE imdb_id = ?"
params = [imdb_id]
if has_video_file_only:
query += " AND has_video_file = TRUE"
if self.db_type == "postgresql":
query = "SELECT * FROM episodes WHERE imdb_id = %s"
params = [imdb_id]
if has_video_file_only:
query += " AND has_video_file = TRUE"
else:
query = "SELECT * FROM episodes WHERE imdb_id = ?"
params = [imdb_id]
if has_video_file_only:
query += " AND has_video_file = TRUE"
cursor.execute(query, params)
return [dict(row) for row in cursor.fetchall()]
@@ -209,9 +394,10 @@ class NFOGuardDatabase:
"""Get episode date record"""
with self.get_connection() as conn:
cursor = conn.cursor()
cursor.execute("""
ph = self._get_placeholder()
cursor.execute(f"""
SELECT * FROM episodes
WHERE imdb_id = ? AND season = ? AND episode = ?
WHERE imdb_id = {ph} AND season = {ph} AND episode = {ph}
""", (imdb_id, season, episode))
row = cursor.fetchone()
@@ -221,7 +407,8 @@ class NFOGuardDatabase:
"""Get movie date record"""
with self.get_connection() as conn:
cursor = conn.cursor()
cursor.execute("SELECT * FROM movies WHERE imdb_id = ?", (imdb_id,))
ph = self._get_placeholder()
cursor.execute(f"SELECT * FROM movies WHERE imdb_id = {ph}", (imdb_id,))
row = cursor.fetchone()
return dict(row) if row else None