This commit is contained in:
+25
-224
@@ -1,66 +1,42 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Database management for NFOGuard
|
||||
Handles SQLite and PostgreSQL database operations for tracking media dates and processing history
|
||||
PostgreSQL database management for NFOGuard
|
||||
Handles database operations for tracking media dates and processing history
|
||||
"""
|
||||
import sqlite3
|
||||
import json
|
||||
import threading
|
||||
import os
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import Optional, Dict, List, Any, Union
|
||||
from typing import Optional, Dict, List, Any
|
||||
from contextlib import contextmanager
|
||||
|
||||
try:
|
||||
import psycopg2
|
||||
import psycopg2.extras
|
||||
POSTGRESQL_AVAILABLE = True
|
||||
except ImportError:
|
||||
psycopg2 = None
|
||||
POSTGRESQL_AVAILABLE = False
|
||||
|
||||
class NFOGuardDatabase:
|
||||
"""Manages NFOGuard database operations for both SQLite and PostgreSQL"""
|
||||
"""PostgreSQL database manager for NFOGuard media tracking and processing history"""
|
||||
|
||||
def __init__(self, config=None, db_path: Optional[Path] = None):
|
||||
def __init__(self, config):
|
||||
"""
|
||||
Initialize database connection
|
||||
Initialize PostgreSQL database connection
|
||||
|
||||
Args:
|
||||
config: Configuration object with database settings
|
||||
db_path: Path for SQLite database (backward compatibility)
|
||||
"""
|
||||
if config:
|
||||
self.db_type = config.db_type
|
||||
if self.db_type == "postgresql":
|
||||
if not POSTGRESQL_AVAILABLE:
|
||||
raise ImportError("psycopg2 is required for PostgreSQL support")
|
||||
if not config:
|
||||
raise ValueError("PostgreSQL configuration is required")
|
||||
self.db_host = config.db_host
|
||||
self.db_port = config.db_port
|
||||
self.db_name = config.db_name
|
||||
self.db_user = config.db_user
|
||||
self.db_password = config.db_password
|
||||
else:
|
||||
self.db_path = Path(config.db_path)
|
||||
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
else:
|
||||
# Backward compatibility
|
||||
self.db_type = "sqlite"
|
||||
self.db_path = Path(db_path) if db_path else Path("/app/data/media_dates.db")
|
||||
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self._local = threading.local()
|
||||
self._init_database()
|
||||
|
||||
def _get_placeholder(self) -> str:
|
||||
"""Get the correct parameter placeholder for this database type"""
|
||||
return "%s" if self.db_type == "postgresql" else "?"
|
||||
|
||||
def _get_connection(self) -> Union[sqlite3.Connection, 'psycopg2.extensions.connection']:
|
||||
"""Get thread-local database connection"""
|
||||
def _get_connection(self) -> 'psycopg2.extensions.connection':
|
||||
"""Get thread-local PostgreSQL database connection"""
|
||||
if not hasattr(self._local, 'connection'):
|
||||
if self.db_type == "postgresql":
|
||||
self._local.connection = psycopg2.connect(
|
||||
host=self.db_host,
|
||||
port=self.db_port,
|
||||
@@ -70,57 +46,37 @@ class NFOGuardDatabase:
|
||||
cursor_factory=psycopg2.extras.RealDictCursor
|
||||
)
|
||||
self._local.connection.autocommit = True
|
||||
else:
|
||||
self._local.connection = sqlite3.connect(
|
||||
self.db_path,
|
||||
check_same_thread=False,
|
||||
timeout=30.0
|
||||
)
|
||||
self._local.connection.row_factory = sqlite3.Row
|
||||
return self._local.connection
|
||||
|
||||
def _get_first_value(self, row):
|
||||
"""Get first value from row, handling both dict and tuple types"""
|
||||
if self.db_type == "postgresql":
|
||||
"""Get first value from row from PostgreSQL RealDictCursor"""
|
||||
# RealDictCursor returns dict-like objects
|
||||
return list(row.values())[0] if row else None
|
||||
else:
|
||||
# SQLite returns tuples
|
||||
return row[0] if row else None
|
||||
|
||||
@contextmanager
|
||||
def get_connection(self):
|
||||
"""Context manager for database connections"""
|
||||
"""Context manager for PostgreSQL database connections"""
|
||||
conn = self._get_connection()
|
||||
try:
|
||||
yield conn
|
||||
if self.db_type == "sqlite":
|
||||
conn.commit() # PostgreSQL uses autocommit
|
||||
# PostgreSQL uses autocommit - no manual commit needed
|
||||
except Exception:
|
||||
if self.db_type == "sqlite":
|
||||
conn.rollback() # PostgreSQL uses autocommit
|
||||
# PostgreSQL uses autocommit - no manual rollback needed
|
||||
raise
|
||||
|
||||
def _init_database(self):
|
||||
"""Initialize database tables with migration support"""
|
||||
"""Initialize PostgreSQL database tables"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
if self.db_type == "postgresql":
|
||||
self._init_postgresql_tables(cursor)
|
||||
else:
|
||||
self._init_sqlite_tables(cursor)
|
||||
|
||||
# Test the connection works and verify autocommit
|
||||
cursor.execute("SELECT 1")
|
||||
if self.db_type == "postgresql":
|
||||
print(f"✅ Database initialized and connection verified: {self.db_type.upper()}")
|
||||
print(f"🔍 PostgreSQL autocommit status: {conn.autocommit}")
|
||||
else:
|
||||
print(f"✅ Database initialized and connection verified: {self.db_type.upper()}")
|
||||
print(f"✅ PostgreSQL database initialized and connection verified")
|
||||
print(f"🔍 Autocommit status: {conn.autocommit}")
|
||||
|
||||
def _init_postgresql_tables(self, cursor):
|
||||
"""Initialize PostgreSQL tables"""
|
||||
"""Initialize database tables"""
|
||||
# Series table
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS series (
|
||||
@@ -177,105 +133,12 @@ class NFOGuardDatabase:
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_episodes_video ON episodes(has_video_file)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_movies_video ON movies(has_video_file)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_history_imdb ON processing_history(imdb_id)")
|
||||
|
||||
def _init_sqlite_tables(self, cursor):
|
||||
"""Initialize SQLite tables"""
|
||||
# Series table
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS series (
|
||||
imdb_id TEXT PRIMARY KEY,
|
||||
path TEXT NOT NULL,
|
||||
last_updated TEXT NOT NULL,
|
||||
metadata TEXT
|
||||
)
|
||||
""")
|
||||
|
||||
# Episodes table
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS episodes (
|
||||
imdb_id TEXT NOT NULL,
|
||||
season INTEGER NOT NULL,
|
||||
episode INTEGER NOT NULL,
|
||||
aired TEXT,
|
||||
dateadded TEXT,
|
||||
source TEXT,
|
||||
last_updated TEXT NOT NULL,
|
||||
has_video_file BOOLEAN DEFAULT FALSE,
|
||||
PRIMARY KEY (imdb_id, season, episode),
|
||||
FOREIGN KEY (imdb_id) REFERENCES series(imdb_id)
|
||||
)
|
||||
""")
|
||||
|
||||
# Movies table
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS movies (
|
||||
imdb_id TEXT PRIMARY KEY,
|
||||
path TEXT NOT NULL,
|
||||
released TEXT,
|
||||
dateadded TEXT,
|
||||
source TEXT,
|
||||
last_updated TEXT NOT NULL,
|
||||
has_video_file BOOLEAN DEFAULT FALSE
|
||||
)
|
||||
""")
|
||||
|
||||
# Processing history table
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS processing_history (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
imdb_id TEXT NOT NULL,
|
||||
media_type TEXT NOT NULL,
|
||||
event_type TEXT NOT NULL,
|
||||
processed_at TEXT NOT NULL,
|
||||
details TEXT
|
||||
)
|
||||
""")
|
||||
|
||||
# Handle SQLite migrations for existing databases
|
||||
self._handle_sqlite_migrations(cursor)
|
||||
|
||||
# Create indexes for SQLite
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_episodes_imdb ON episodes(imdb_id)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_episodes_video ON episodes(has_video_file)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_movies_video ON movies(has_video_file)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_history_imdb ON processing_history(imdb_id)")
|
||||
|
||||
def _handle_sqlite_migrations(self, cursor):
|
||||
"""Handle SQLite database migrations for existing databases"""
|
||||
# Check current schema and add missing columns
|
||||
cursor.execute("PRAGMA table_info(movies)")
|
||||
movie_columns = [row[1] for row in cursor.fetchall()]
|
||||
|
||||
cursor.execute("PRAGMA table_info(episodes)")
|
||||
episode_columns = [row[1] for row in cursor.fetchall()]
|
||||
|
||||
# Add missing columns to movies table
|
||||
if 'path' not in movie_columns:
|
||||
cursor.execute("ALTER TABLE movies ADD COLUMN path TEXT")
|
||||
cursor.execute("UPDATE movies SET path = '/unknown/path/' || imdb_id WHERE path IS NULL")
|
||||
|
||||
if 'has_video_file' not in movie_columns:
|
||||
cursor.execute("ALTER TABLE movies ADD COLUMN has_video_file BOOLEAN DEFAULT FALSE")
|
||||
|
||||
if 'last_updated' not in movie_columns:
|
||||
cursor.execute("ALTER TABLE movies ADD COLUMN last_updated TEXT")
|
||||
cursor.execute("UPDATE movies SET last_updated = datetime('now') WHERE last_updated IS NULL")
|
||||
|
||||
# Add missing columns to episodes table
|
||||
if 'has_video_file' not in episode_columns:
|
||||
cursor.execute("ALTER TABLE episodes ADD COLUMN has_video_file BOOLEAN DEFAULT FALSE")
|
||||
|
||||
if 'last_updated' not in episode_columns:
|
||||
cursor.execute("ALTER TABLE episodes ADD COLUMN last_updated TEXT")
|
||||
cursor.execute("UPDATE episodes SET last_updated = datetime('now') WHERE last_updated IS NULL")
|
||||
|
||||
def upsert_series(self, imdb_id: str, path: str, metadata: Optional[Dict] = None):
|
||||
"""Insert or update series record"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
timestamp = datetime.utcnow()
|
||||
|
||||
if self.db_type == "postgresql":
|
||||
cursor.execute("""
|
||||
INSERT INTO series (imdb_id, path, last_updated, metadata)
|
||||
VALUES (%s, %s, %s, %s)
|
||||
@@ -284,11 +147,6 @@ class NFOGuardDatabase:
|
||||
last_updated = EXCLUDED.last_updated,
|
||||
metadata = EXCLUDED.metadata
|
||||
""", (imdb_id, path, timestamp, json.dumps(metadata) if metadata else None))
|
||||
else:
|
||||
cursor.execute("""
|
||||
INSERT OR REPLACE INTO series (imdb_id, path, last_updated, metadata)
|
||||
VALUES (?, ?, ?, ?)
|
||||
""", (imdb_id, path, timestamp.isoformat(), json.dumps(metadata) if metadata else None))
|
||||
|
||||
def upsert_episode_date(self, imdb_id: str, season: int, episode: int,
|
||||
aired: Optional[str], dateadded: Optional[str],
|
||||
@@ -298,7 +156,6 @@ class NFOGuardDatabase:
|
||||
cursor = conn.cursor()
|
||||
timestamp = datetime.utcnow()
|
||||
|
||||
if self.db_type == "postgresql":
|
||||
cursor.execute("""
|
||||
INSERT INTO episodes
|
||||
(imdb_id, season, episode, aired, dateadded, source, has_video_file, last_updated)
|
||||
@@ -311,12 +168,6 @@ class NFOGuardDatabase:
|
||||
last_updated = EXCLUDED.last_updated
|
||||
""", (imdb_id, season, episode, aired, dateadded, source, has_video_file, timestamp))
|
||||
print(f"🔍 DEBUG: PostgreSQL upsert executed for {imdb_id} S{season:02d}E{episode:02d}, rows affected: {cursor.rowcount}")
|
||||
else:
|
||||
cursor.execute("""
|
||||
INSERT OR REPLACE INTO episodes
|
||||
(imdb_id, season, episode, aired, dateadded, source, has_video_file, last_updated)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (imdb_id, season, episode, aired, dateadded, source, has_video_file, timestamp.isoformat()))
|
||||
|
||||
def upsert_movie(self, imdb_id: str, path: str):
|
||||
"""Insert or update movie record"""
|
||||
@@ -324,7 +175,6 @@ class NFOGuardDatabase:
|
||||
cursor = conn.cursor()
|
||||
timestamp = datetime.utcnow()
|
||||
|
||||
if self.db_type == "postgresql":
|
||||
cursor.execute("""
|
||||
INSERT INTO movies (imdb_id, path, last_updated)
|
||||
VALUES (%s, %s, %s)
|
||||
@@ -332,21 +182,6 @@ class NFOGuardDatabase:
|
||||
path = EXCLUDED.path,
|
||||
last_updated = EXCLUDED.last_updated
|
||||
""", (imdb_id, path, timestamp))
|
||||
else:
|
||||
try:
|
||||
cursor.execute("""
|
||||
INSERT OR REPLACE INTO movies (imdb_id, path, last_updated)
|
||||
VALUES (?, ?, ?)
|
||||
""", (imdb_id, path, timestamp.isoformat()))
|
||||
except sqlite3.OperationalError as e:
|
||||
if "no column named path" in str(e):
|
||||
# Fallback for databases without path column - just insert imdb_id
|
||||
cursor.execute("""
|
||||
INSERT OR REPLACE INTO movies (imdb_id, last_updated)
|
||||
VALUES (?, ?)
|
||||
""", (imdb_id, timestamp.isoformat()))
|
||||
else:
|
||||
raise
|
||||
|
||||
def upsert_movie_dates(self, imdb_id: str, released: Optional[str],
|
||||
dateadded: Optional[str], source: str, has_video_file: bool = False):
|
||||
@@ -356,8 +191,6 @@ class NFOGuardDatabase:
|
||||
cursor = conn.cursor()
|
||||
timestamp = datetime.utcnow()
|
||||
|
||||
if self.db_type == "postgresql":
|
||||
# PostgreSQL version with UPSERT
|
||||
cursor.execute("""
|
||||
INSERT INTO movies (imdb_id, path, released, dateadded, source, has_video_file, last_updated)
|
||||
VALUES (%s, COALESCE((SELECT path FROM movies WHERE imdb_id = %s), 'unknown'), %s, %s, %s, %s, %s)
|
||||
@@ -371,20 +204,6 @@ class NFOGuardDatabase:
|
||||
|
||||
# Debug: Check what was actually saved
|
||||
cursor.execute("SELECT dateadded, source FROM movies WHERE imdb_id = %s", (imdb_id,))
|
||||
else:
|
||||
# SQLite version with INSERT OR REPLACE
|
||||
cursor.execute("""
|
||||
INSERT OR REPLACE INTO movies (imdb_id, path, released, dateadded, source, has_video_file, last_updated)
|
||||
VALUES (
|
||||
?,
|
||||
COALESCE((SELECT path FROM movies WHERE imdb_id = ?), 'unknown'),
|
||||
?, ?, ?, ?, ?
|
||||
)
|
||||
""", (imdb_id, imdb_id, released, dateadded, source, has_video_file, timestamp.isoformat()))
|
||||
|
||||
# Debug: Check what was actually saved
|
||||
cursor.execute("SELECT dateadded, source FROM movies WHERE imdb_id = ?", (imdb_id,))
|
||||
|
||||
result = cursor.fetchone()
|
||||
print(f"🔍 DATABASE VERIFY: After upsert, found dateadded={result[0] if result else 'NOT_FOUND'}, source={result[1] if result else 'NOT_FOUND'}")
|
||||
|
||||
@@ -393,16 +212,10 @@ class NFOGuardDatabase:
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
if self.db_type == "postgresql":
|
||||
query = "SELECT * FROM episodes WHERE imdb_id = %s"
|
||||
params = [imdb_id]
|
||||
if has_video_file_only:
|
||||
query += " AND has_video_file = TRUE"
|
||||
else:
|
||||
query = "SELECT * FROM episodes WHERE imdb_id = ?"
|
||||
params = [imdb_id]
|
||||
if has_video_file_only:
|
||||
query += " AND has_video_file = TRUE"
|
||||
|
||||
cursor.execute(query, params)
|
||||
return [dict(row) for row in cursor.fetchall()]
|
||||
@@ -411,10 +224,9 @@ class NFOGuardDatabase:
|
||||
"""Get episode date record"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
ph = self._get_placeholder()
|
||||
cursor.execute(f"""
|
||||
cursor.execute("""
|
||||
SELECT * FROM episodes
|
||||
WHERE imdb_id = {ph} AND season = {ph} AND episode = {ph}
|
||||
WHERE imdb_id = %s AND season = %s AND episode = %s
|
||||
""", (imdb_id, season, episode))
|
||||
|
||||
row = cursor.fetchone()
|
||||
@@ -424,8 +236,7 @@ class NFOGuardDatabase:
|
||||
"""Get movie date record"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
ph = self._get_placeholder()
|
||||
cursor.execute(f"SELECT * FROM movies WHERE imdb_id = {ph}", (imdb_id,))
|
||||
cursor.execute("SELECT * FROM movies WHERE imdb_id = %s", (imdb_id,))
|
||||
|
||||
row = cursor.fetchone()
|
||||
return dict(row) if row else None
|
||||
@@ -434,21 +245,16 @@ class NFOGuardDatabase:
|
||||
"""Add processing history entry"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
placeholder = self._get_placeholder()
|
||||
cursor.execute(f"""
|
||||
cursor.execute("""
|
||||
INSERT INTO processing_history (imdb_id, media_type, event_type, processed_at, details)
|
||||
VALUES ({placeholder}, {placeholder}, {placeholder}, {placeholder}, {placeholder})
|
||||
VALUES (%s, %s, %s, %s, %s)
|
||||
""", (imdb_id, media_type, event_type, datetime.utcnow().isoformat(),
|
||||
json.dumps(details) if details else None))
|
||||
|
||||
def get_stats(self) -> Dict[str, Any]:
|
||||
"""Get database statistics"""
|
||||
with self.get_connection() as conn:
|
||||
# Use regular cursor for simple COUNT queries (not RealDictCursor)
|
||||
if self.db_type == "postgresql":
|
||||
cursor = conn.cursor() # Regular cursor for PostgreSQL
|
||||
else:
|
||||
cursor = conn.cursor() # SQLite cursor
|
||||
|
||||
# Series stats
|
||||
cursor.execute("SELECT COUNT(*) FROM series")
|
||||
@@ -472,15 +278,10 @@ class NFOGuardDatabase:
|
||||
cursor.execute("SELECT COUNT(*) FROM processing_history")
|
||||
history_count = self._get_first_value(cursor.fetchone())
|
||||
|
||||
# Database size calculation
|
||||
if self.db_type == "postgresql":
|
||||
# PostgreSQL: Query pg_database_size
|
||||
# Database size calculation for PostgreSQL
|
||||
cursor.execute("SELECT pg_database_size(%s)", (self.db_name,))
|
||||
db_size_bytes = self._get_first_value(cursor.fetchone())
|
||||
db_size_mb = round(db_size_bytes / 1024 / 1024, 2) if db_size_bytes else 0
|
||||
else:
|
||||
# SQLite: File size
|
||||
db_size_mb = round(self.db_path.stat().st_size / 1024 / 1024, 2)
|
||||
|
||||
return {
|
||||
"series_count": series_count,
|
||||
@@ -490,7 +291,7 @@ class NFOGuardDatabase:
|
||||
"movies_with_video": movies_with_video,
|
||||
"processing_history_count": history_count,
|
||||
"database_size_mb": db_size_mb,
|
||||
"database_type": self.db_type
|
||||
"database_type": "postgresql"
|
||||
}
|
||||
|
||||
def close(self):
|
||||
|
||||
@@ -921,8 +921,8 @@ class TVProcessor:
|
||||
"""
|
||||
Get episode date for webhook processing with correct priority:
|
||||
1. Check existing NFOGuard database entry (preserve previous import dates)
|
||||
2. If not found, use current time (webhook = new download)
|
||||
3. Get aired date from Sonarr/TMDB for reference
|
||||
2. Check Sonarr import history - if old history exists, prefer air date
|
||||
3. If no history, use current time (webhook = new download)
|
||||
"""
|
||||
# Check if we already have this episode in our database (preserve existing dates)
|
||||
existing = self.db.get_episode_date(imdb_id, season_num, episode_num)
|
||||
@@ -938,7 +938,43 @@ class TVProcessor:
|
||||
aired = episode_data.get('airDate')
|
||||
_log("DEBUG", f"Got aired date from Sonarr for S{season_num:02d}E{episode_num:02d}: {aired}")
|
||||
|
||||
# For webhook processing, use current time as dateadded (new download)
|
||||
# NEW: Check Sonarr import history to detect if this is an existing show
|
||||
episode_id = None
|
||||
if series_metadata and 'episodes' in series_metadata:
|
||||
episode_data = series_metadata['episodes'].get((season_num, episode_num))
|
||||
if episode_data:
|
||||
episode_id = episode_data.get('id')
|
||||
|
||||
if episode_id and hasattr(self, 'sonarr'):
|
||||
try:
|
||||
import_history = self.sonarr.get_episode_import_history(episode_id)
|
||||
if import_history:
|
||||
from datetime import datetime as dt
|
||||
from dateutil import parser
|
||||
|
||||
# Parse the import history date
|
||||
history_date = parser.parse(import_history)
|
||||
current_time = dt.now(history_date.tzinfo)
|
||||
time_diff = current_time - history_date
|
||||
|
||||
# If import history is more than 24 hours old, this is likely a re-download/upgrade
|
||||
# In this case, prefer air date over webhook date
|
||||
if time_diff.days > 0:
|
||||
_log("INFO", f"Found old import history for S{season_num:02d}E{episode_num:02d} ({import_history}), using air date instead of webhook date")
|
||||
if aired:
|
||||
# Use air date for shows with existing history
|
||||
return aired, aired + "T20:00:00", "airdate"
|
||||
else:
|
||||
# Fallback to import history if no air date
|
||||
return aired, import_history, "sonarr:import_history"
|
||||
else:
|
||||
# Recent import, use the import date
|
||||
_log("DEBUG", f"Found recent import history for S{season_num:02d}E{episode_num:02d}: {import_history}")
|
||||
return aired, import_history, "sonarr:import_history"
|
||||
except Exception as e:
|
||||
_log("DEBUG", f"Error checking Sonarr import history for S{season_num:02d}E{episode_num:02d}: {e}")
|
||||
|
||||
# For webhook processing with no history, use current time as dateadded (new download)
|
||||
dateadded = datetime.now().isoformat()
|
||||
source = "sonarr:webhook"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user