From 97a9f3431ae5861b400d154e2af74f9673806689 Mon Sep 17 00:00:00 2001 From: SBCrumb Date: Sun, 2 Nov 2025 13:43:28 -0500 Subject: [PATCH] refactor: update and remove anything to do with NFO files moving to an all DB approach since radarr overwrites .nfo files --- .env.example | 20 +- api/routes.py | 117 +++++ core/async_nfo_manager.py | 473 ------------------ core/database_populator.py | 267 ++++++++++ core/episode_nfo_manager.py | 276 ----------- core/nfo_manager.py | 890 ---------------------------------- main.py | 22 +- processors/movie_processor.py | 361 ++------------ processors/tv_processor.py | 279 ++--------- static/index.html | 33 ++ static/js/app.js | 184 +++++++ utils/imdb_utils.py | 85 ++++ webhooks/webhook_batcher.py | 82 ++-- 13 files changed, 837 insertions(+), 2252 deletions(-) delete mode 100644 core/async_nfo_manager.py create mode 100644 core/database_populator.py delete mode 100644 core/episode_nfo_manager.py delete mode 100644 core/nfo_manager.py create mode 100644 utils/imdb_utils.py diff --git a/.env.example b/.env.example index 1422453..bf84f7a 100644 --- a/.env.example +++ b/.env.example @@ -97,18 +97,22 @@ ALLOW_FILE_DATE_FALLBACK=false TMDB_COUNTRY=US # =========================================== -# NFO FILE MANAGEMENT +# NFO FILE MANAGEMENT (DEPRECATED - Phase 1 Migration) # =========================================== -# Create/update .nfo files -MANAGE_NFO=true +# NFO file operations have been removed in favor of database-only architecture +# These settings are preserved for backward compatibility but no longer have effect +# The PostgreSQL database is now the single source of truth for all metadata -# Update file modification times to match import dates -FIX_DIR_MTIMES=true +# Create/update .nfo files (DEPRECATED - no longer used) +MANAGE_NFO=false -# Add lockdata tags to prevent metadata overwrites -LOCK_METADATA=true +# Update file modification times to match import dates (DEPRECATED - no longer used) +FIX_DIR_MTIMES=false -# Brand name in NFO comments +# Add lockdata tags to prevent metadata overwrites (DEPRECATED - no longer used) +LOCK_METADATA=false + +# Brand name in NFO comments (DEPRECATED - no longer used) MANAGER_BRAND=NFOGuard # =========================================== diff --git a/api/routes.py b/api/routes.py index cd3e519..7b0690a 100644 --- a/api/routes.py +++ b/api/routes.py @@ -2538,6 +2538,115 @@ async def lookup_movie(imdb_id: str, dependencies: dict): raise HTTPException(status_code=500, detail=f"Movie lookup failed: {str(e)}") +async def populate_database(background_tasks: BackgroundTasks, media_type: str = "both", dependencies: dict = None): + """ + Populate NFOGuard database from Radarr/Sonarr sources + + Args: + background_tasks: FastAPI background tasks + media_type: Type of media to populate ("movies", "tv", or "both") + dependencies: Dictionary with db, radarr_client, sonarr_client + + Returns: + Status message indicating population has started + """ + from core.database_populator import DatabasePopulator + + db = dependencies["db"] + config = dependencies["config"] + + # Get Radarr and Sonarr clients + from clients.radarr_client import RadarrClient + from clients.sonarr_client import SonarrClient + + radarr_client = RadarrClient(config) + sonarr_client = SonarrClient(config) + + if media_type not in ["both", "movies", "tv"]: + raise HTTPException(status_code=400, detail="media_type must be 'both', 'movies', or 'tv'") + + # Create global status tracking + populate_status = { + "running": True, + "media_type": media_type, + "start_time": datetime.now().isoformat(), + "movies": {"status": "pending", "stats": None}, + "tv": {"status": "pending", "stats": None}, + "completed": False, + "error": None + } + + # Store status globally so it can be queried + global _populate_status + _populate_status = populate_status + + async def run_population(): + """Background task to populate the database""" + try: + populator = DatabasePopulator(db, radarr_client, sonarr_client) + + _log("INFO", f"Starting database population: {media_type}") + + if media_type == "movies": + populate_status["movies"]["status"] = "running" + movie_stats = populator.populate_movies() + populate_status["movies"]["status"] = "completed" + populate_status["movies"]["stats"] = movie_stats + _log("INFO", f"Movie population completed: {movie_stats}") + + elif media_type == "tv": + populate_status["tv"]["status"] = "running" + tv_stats = populator.populate_tv_episodes() + populate_status["tv"]["status"] = "completed" + populate_status["tv"]["stats"] = tv_stats + _log("INFO", f"TV population completed: {tv_stats}") + + elif media_type == "both": + populate_status["movies"]["status"] = "running" + movie_stats = populator.populate_movies() + populate_status["movies"]["status"] = "completed" + populate_status["movies"]["stats"] = movie_stats + _log("INFO", f"Movie population completed: {movie_stats}") + + populate_status["tv"]["status"] = "running" + tv_stats = populator.populate_tv_episodes() + populate_status["tv"]["status"] = "completed" + populate_status["tv"]["stats"] = tv_stats + _log("INFO", f"TV population completed: {tv_stats}") + + populate_status["completed"] = True + populate_status["running"] = False + _log("INFO", "Database population completed successfully") + + except Exception as e: + _log("ERROR", f"Database population failed: {e}") + populate_status["error"] = str(e) + populate_status["running"] = False + populate_status["completed"] = True + + # Add task to background + background_tasks.add_task(run_population) + + _log("INFO", f"Database population started for: {media_type}") + return { + "status": "started", + "media_type": media_type, + "message": f"Database population started for {media_type}" + } + + +async def get_populate_status(): + """Get the current status of database population""" + global _populate_status + if '_populate_status' not in globals(): + return {"running": False, "completed": False} + return _populate_status + + +# Initialize global populate status +_populate_status = {"running": False, "completed": False} + + def register_routes(app, dependencies: dict): """ Register all routes with the FastAPI app @@ -2641,6 +2750,14 @@ def register_routes(app, dependencies: dict): async def _scan_status(): return await get_scan_status() + @app.post("/admin/populate-database") + async def _populate_database(background_tasks: BackgroundTasks, media_type: str = "both"): + return await populate_database(background_tasks, media_type, dependencies) + + @app.get("/api/populate/status") + async def _populate_status(): + return await get_populate_status() + @app.post("/tv/scan-season") async def _scan_tv_season(background_tasks: BackgroundTasks, request: TVSeasonRequest): return await scan_tv_season(background_tasks, request, dependencies) diff --git a/core/async_nfo_manager.py b/core/async_nfo_manager.py deleted file mode 100644 index d0ab52f..0000000 --- a/core/async_nfo_manager.py +++ /dev/null @@ -1,473 +0,0 @@ -""" -Async NFO Manager for NFOGuard -High-performance async NFO file operations with concurrent processing -""" -import asyncio -from pathlib import Path -from typing import Optional, List, Dict, Any, Tuple -from datetime import datetime -import xml.etree.ElementTree as ET - -from utils.logging import _log -from utils.async_file_utils import ( - async_read_nfo_file, - async_write_nfo_file, - async_file_exists, - async_set_file_mtime, - async_batch_nfo_operations, - async_concurrent_episode_processing, - aiofiles -) -from utils.nfo_patterns import ( - create_basic_nfo_structure, - extract_imdb_from_nfo_content, - extract_dates_from_nfo, - extract_imdb_id_from_text -) -from utils.validation import validate_date_string -from utils.file_utils import VIDEO_EXTENSIONS - - -class AsyncNFOManager: - """Async NFO file manager with concurrent processing capabilities""" - - def __init__(self, manager_brand: str = "NFOGuard", debug: bool = False): - self.manager_brand = manager_brand - self.debug = debug - - async def _async_get_target_nfo_path(self, season_dir: Path, season: int, episode: int) -> Path: - """ - Get the target NFO path using video filename matching to prevent concatenation - - Args: - season_dir: Path to season directory - season: Season number - episode: Episode number - - Returns: - Path to target NFO file (video-matching preferred, generic fallback) - """ - try: - # Find video files in the season directory - video_files = [] - if await async_file_exists(season_dir): - import re - try: - entries = await aiofiles.os.listdir(season_dir) - for entry in entries: - entry_path = season_dir / entry - if await aiofiles.os.path.isfile(entry_path): - if entry_path.suffix.lower() in VIDEO_EXTENSIONS: - # Parse episode info from filename - match = re.search(r'[Ss](\d{1,2})[Ee](\d{1,2})', entry) - if match: - s, e = int(match.group(1)), int(match.group(2)) - if s == season and e == episode: - # Found matching video file - use its name for NFO - target_nfo = season_dir / f"{entry_path.stem}.nfo" - if self.debug: - _log("DEBUG", f"Video-matching NFO path: {target_nfo.name}") - return target_nfo - except Exception as e: - if self.debug: - _log("WARNING", f"Failed to scan season directory {season_dir}: {e}") - - # Fallback to generic filename if no matching video found - target_nfo = season_dir / f"S{season:02d}E{episode:02d}.nfo" - if self.debug: - _log("WARNING", f"No video file found for S{season:02d}E{episode:02d}, using generic: {target_nfo.name}") - return target_nfo - - except Exception as e: - _log("ERROR", f"Failed to determine NFO path for S{season:02d}E{episode:02d}: {e}") - # Emergency fallback - return season_dir / f"S{season:02d}E{episode:02d}.nfo" - - async def async_parse_imdb_from_path(self, path: Path) -> Optional[str]: - """ - Async extract IMDb ID from directory path or filename - - Args: - path: Path to examine - - Returns: - IMDb ID if found, None otherwise - """ - # Use the sync version since it's just string processing - return extract_imdb_id_from_text(str(path)) - - async def async_parse_imdb_from_nfo(self, nfo_path: Path) -> Optional[str]: - """ - Async extract IMDb ID from NFO file content - - Args: - nfo_path: Path to NFO file - - Returns: - IMDb ID if found, None otherwise - """ - root = await async_read_nfo_file(nfo_path) - if root is None: - return None - - return extract_imdb_from_nfo_content(root) - - async def async_find_movie_imdb_id(self, movie_dir: Path) -> Optional[str]: - """ - Async find IMDb ID from directory name, filenames, or NFO file - - Args: - movie_dir: Path to movie directory - - Returns: - IMDb ID if found, None otherwise - """ - if self.debug: - _log("DEBUG", f"Async searching for IMDb ID in: {movie_dir.name}") - - # First try directory name - imdb_id = await self.async_parse_imdb_from_path(movie_dir) - if imdb_id: - if self.debug: - _log("DEBUG", f"Found IMDb ID in directory name: {imdb_id}") - return imdb_id - - # Try all files in the directory concurrently - if await async_file_exists(movie_dir): - try: - from utils.async_file_utils import aiofiles - entries = await aiofiles.os.listdir(movie_dir) - - # Create tasks to check all files - async def check_file(filename: str) -> Optional[str]: - file_path = movie_dir / filename - if await aiofiles.os.path.isfile(file_path): - return await self.async_parse_imdb_from_path(file_path) - return None - - # Check all files concurrently - results = await asyncio.gather( - *[check_file(filename) for filename in entries], - return_exceptions=True - ) - - # Find first valid IMDb ID - for result in results: - if isinstance(result, str) and result: - if self.debug: - _log("DEBUG", f"Found IMDb ID in filename: {result}") - return result - - except Exception as e: - _log("WARNING", f"Failed to scan directory {movie_dir}: {e}") - - # Finally, try NFO file content - nfo_path = movie_dir / "movie.nfo" - imdb_id = await self.async_parse_imdb_from_nfo(nfo_path) - if imdb_id: - if self.debug: - _log("DEBUG", f"Found IMDb ID in NFO file: {imdb_id}") - return imdb_id - - if self.debug: - _log("DEBUG", f"No IMDb ID found for: {movie_dir.name}") - return None - - async def async_create_movie_nfo( - self, - movie_dir: Path, - imdb_id: str, - dateadded: str, - premiered: Optional[str] = None, - lock_metadata: bool = True - ) -> bool: - """ - Async create movie NFO file - - Args: - movie_dir: Path to movie directory - imdb_id: IMDb ID - dateadded: Date added - premiered: Optional premiere date - lock_metadata: Whether to lock metadata - - Returns: - True if successful, False otherwise - """ - try: - nfo_path = movie_dir / "movie.nfo" - - # Prepare dates - dates = {"dateadded": dateadded} - if premiered and validate_date_string(premiered): - dates["premiered"] = premiered - - # Create NFO structure - root = create_basic_nfo_structure( - media_type="movie", - title=movie_dir.name, - imdb_id=imdb_id, - dates=dates, - additional_fields={"source": self.manager_brand} - ) - - # Write NFO file asynchronously - success = await async_write_nfo_file(nfo_path, root, lock_metadata) - - if success and self.debug: - _log("DEBUG", f"Created movie NFO: {nfo_path}") - - return success - - except Exception as e: - _log("ERROR", f"Failed to create movie NFO for {movie_dir}: {e}") - return False - - async def async_create_episode_nfo( - self, - season_dir: Path, - season: int, - episode: int, - aired: Optional[str] = None, - dateadded: Optional[str] = None, - source: str = "unknown", - lock_metadata: bool = True - ) -> bool: - """ - Async create episode NFO file with proper video filename matching - - Args: - season_dir: Path to season directory - season: Season number - episode: Episode number - aired: Optional air date - dateadded: Optional date added - source: Source of the data - lock_metadata: Whether to lock metadata - - Returns: - True if successful, False otherwise - """ - try: - # Use proper video filename matching to prevent NFO concatenation - nfo_path = await self._async_get_target_nfo_path(season_dir, season, episode) - - # Prepare dates - dates = {} - if aired and validate_date_string(aired): - dates["aired"] = aired - if dateadded and validate_date_string(dateadded): - dates["dateadded"] = dateadded - - # Create NFO structure - root = create_basic_nfo_structure( - media_type="episodedetails", - title=f"S{season:02d}E{episode:02d}", - dates=dates, - additional_fields={ - "season": str(season), - "episode": str(episode), - "source": source - } - ) - - # Write NFO file asynchronously - success = await async_write_nfo_file(nfo_path, root, lock_metadata) - - if success and self.debug: - _log("DEBUG", f"Created episode NFO: {nfo_path}") - - return success - - except Exception as e: - _log("ERROR", f"Failed to create episode NFO S{season:02d}E{episode:02d}: {e}") - return False - - async def async_batch_create_episode_nfos( - self, - episode_data_list: List[Dict[str, Any]], - max_concurrent: int = 5 - ) -> List[bool]: - """ - Batch create multiple episode NFOs concurrently - - Args: - episode_data_list: List of episode data dictionaries - max_concurrent: Maximum concurrent NFO operations - - Returns: - List of success/failure results - """ - async def _create_episode_nfo(episode_data: Dict[str, Any]) -> bool: - return await self.async_create_episode_nfo( - season_dir=episode_data.get('season_dir'), - season=episode_data.get('season'), - episode=episode_data.get('episode'), - aired=episode_data.get('aired'), - dateadded=episode_data.get('dateadded'), - source=episode_data.get('source', 'unknown'), - lock_metadata=episode_data.get('lock_metadata', True) - ) - - return await async_concurrent_episode_processing( - episode_data_list, - _create_episode_nfo, - max_concurrent - ) - - async def async_set_file_mtime(self, file_path: Path, date_str: str) -> bool: - """ - Async set file modification time from date string - - Args: - file_path: Path to file - date_str: Date string in ISO format - - Returns: - True if successful, False otherwise - """ - try: - if not validate_date_string(date_str): - _log("WARNING", f"Invalid date format for mtime: {date_str}") - return False - - # Parse date string to timestamp - dt = datetime.fromisoformat(date_str.replace('Z', '+00:00')) - timestamp = dt.timestamp() - - # Set file mtime asynchronously - success = await async_set_file_mtime(file_path, timestamp) - - if success and self.debug: - _log("DEBUG", f"Set mtime for {file_path}: {date_str}") - - return success - - except Exception as e: - _log("ERROR", f"Failed to set mtime for {file_path}: {e}") - return False - - async def async_batch_set_file_mtimes( - self, - file_mtime_pairs: List[Tuple[Path, str]], - max_concurrent: int = 10 - ) -> List[bool]: - """ - Batch set file modification times concurrently - - Args: - file_mtime_pairs: List of (file_path, date_str) tuples - max_concurrent: Maximum concurrent operations - - Returns: - List of success/failure results - """ - async def _set_single_mtime(file_path: Path, date_str: str) -> bool: - return await self.async_set_file_mtime(file_path, date_str) - - # Create tasks for all mtime operations - semaphore = asyncio.Semaphore(max_concurrent) - - async def _set_mtime_with_semaphore(file_path: Path, date_str: str) -> bool: - async with semaphore: - return await _set_single_mtime(file_path, date_str) - - tasks = [ - _set_mtime_with_semaphore(file_path, date_str) - for file_path, date_str in file_mtime_pairs - ] - - results = await asyncio.gather(*tasks, return_exceptions=True) - - return [result if isinstance(result, bool) else False for result in results] - - async def async_validate_nfo_integrity( - self, - nfo_paths: List[Path], - max_concurrent: int = 10 - ) -> Dict[str, Any]: - """ - Async validate integrity of multiple NFO files - - Args: - nfo_paths: List of NFO file paths to validate - max_concurrent: Maximum concurrent validations - - Returns: - Dictionary with validation results and statistics - """ - results = { - 'total_files': len(nfo_paths), - 'valid_files': 0, - 'invalid_files': 0, - 'missing_files': 0, - 'validation_errors': [], - 'file_results': {} - } - - async def _validate_single_nfo(nfo_path: Path) -> Dict[str, Any]: - file_result = { - 'path': str(nfo_path), - 'exists': False, - 'valid_xml': False, - 'has_imdb_id': False, - 'has_dates': False, - 'error': None - } - - try: - # Check if file exists - if not await async_file_exists(nfo_path): - file_result['error'] = 'File does not exist' - return file_result - - file_result['exists'] = True - - # Try to parse NFO - root = await async_read_nfo_file(nfo_path) - if root is None: - file_result['error'] = 'Failed to parse XML' - return file_result - - file_result['valid_xml'] = True - - # Check for IMDb ID - imdb_id = extract_imdb_from_nfo_content(root) - file_result['has_imdb_id'] = bool(imdb_id) - - # Check for dates - dates = extract_dates_from_nfo(root) - file_result['has_dates'] = any(dates.values()) - - except Exception as e: - file_result['error'] = str(e) - - return file_result - - # Validate all files concurrently - semaphore = asyncio.Semaphore(max_concurrent) - - async def _validate_with_semaphore(nfo_path: Path) -> Dict[str, Any]: - async with semaphore: - return await _validate_single_nfo(nfo_path) - - tasks = [_validate_with_semaphore(nfo_path) for nfo_path in nfo_paths] - file_results = await asyncio.gather(*tasks, return_exceptions=True) - - # Process results - for i, result in enumerate(file_results): - if isinstance(result, dict): - path_str = str(nfo_paths[i]) - results['file_results'][path_str] = result - - if not result['exists']: - results['missing_files'] += 1 - elif result['valid_xml']: - results['valid_files'] += 1 - else: - results['invalid_files'] += 1 - - if result.get('error'): - results['validation_errors'].append(f"{path_str}: {result['error']}") - - return results \ No newline at end of file diff --git a/core/database_populator.py b/core/database_populator.py new file mode 100644 index 0000000..3d55f41 --- /dev/null +++ b/core/database_populator.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python3 +""" +Database Populator for NFOGuard +Bulk populates the NFOGuard database from Radarr/Sonarr +Phase 4: Replace NFO-based initial population with direct DB/API queries +""" +import time +from typing import Dict, List, Optional, Tuple +from datetime import datetime + +from core.database import NFOGuardDatabase +from clients.radarr_client import RadarrClient +from clients.sonarr_client import SonarrClient +from utils.logging import _log + + +class DatabasePopulator: + """Populates NFOGuard database from Radarr/Sonarr sources""" + + def __init__(self, db: NFOGuardDatabase, radarr_client: RadarrClient, sonarr_client: SonarrClient): + self.db = db + self.radarr = radarr_client + self.sonarr = sonarr_client + + def populate_movies(self) -> Dict[str, any]: + """ + Populate movies from Radarr database/API + + Returns: + Dictionary with statistics: { + 'total': int, + 'added': int, + 'updated': int, + 'skipped': int, + 'errors': int, + 'duration': float + } + """ + _log("INFO", "Starting movie population from Radarr") + start_time = time.time() + + stats = { + 'total': 0, + 'added': 0, + 'updated': 0, + 'skipped': 0, + 'errors': 0, + 'duration': 0.0 + } + + try: + # Get all movies from Radarr + movies = self.radarr.get_all_movies() + if not movies: + _log("WARNING", "No movies found in Radarr") + return stats + + stats['total'] = len(movies) + _log("INFO", f"Found {stats['total']} movies in Radarr") + + # Process each movie + for movie in movies: + try: + imdb_id = movie.get('imdbId') + if not imdb_id: + _log("DEBUG", f"Skipping movie without IMDb ID: {movie.get('title')}") + stats['skipped'] += 1 + continue + + # Check if movie already exists in database + existing = self.db.get_movie_dates(imdb_id) + if existing and existing.get('dateadded'): + _log("DEBUG", f"Movie {imdb_id} already in database, skipping") + stats['skipped'] += 1 + continue + + # Get movie path + path = movie.get('path', '') + + # Get release date + released = None + if movie.get('digitalRelease'): + released = movie.get('digitalRelease') + source_type = 'radarr:digital' + elif movie.get('physicalRelease'): + released = movie.get('physicalRelease') + source_type = 'radarr:physical' + elif movie.get('inCinemas'): + released = movie.get('inCinemas') + source_type = 'radarr:theatrical' + else: + source_type = 'radarr:unknown' + + # Get import date from Radarr history + import_date = self.radarr.get_movie_import_date(imdb_id) + if import_date: + dateadded = import_date + source = 'radarr:import_history' + elif released: + # Use release date as fallback + dateadded = released + source = f'{source_type}_fallback' + else: + _log("DEBUG", f"No date available for movie {imdb_id}, skipping") + stats['skipped'] += 1 + continue + + # Insert into database + self.db.upsert_movie_dates(imdb_id, released, dateadded, source, has_video_file=True) + stats['added'] += 1 + _log("DEBUG", f"Added movie {imdb_id}: {movie.get('title')} (source: {source})") + + except Exception as e: + _log("ERROR", f"Error processing movie {movie.get('title', 'unknown')}: {e}") + stats['errors'] += 1 + continue + + except Exception as e: + _log("ERROR", f"Error during movie population: {e}") + stats['errors'] += 1 + + stats['duration'] = time.time() - start_time + _log("INFO", f"Movie population complete: {stats['added']} added, {stats['skipped']} skipped, {stats['errors']} errors in {stats['duration']:.2f}s") + return stats + + def populate_tv_episodes(self) -> Dict[str, any]: + """ + Populate TV episodes from Sonarr API + + Returns: + Dictionary with statistics: { + 'total_series': int, + 'total_episodes': int, + 'added': int, + 'updated': int, + 'skipped': int, + 'errors': int, + 'duration': float + } + """ + _log("INFO", "Starting TV episode population from Sonarr") + start_time = time.time() + + stats = { + 'total_series': 0, + 'total_episodes': 0, + 'added': 0, + 'updated': 0, + 'skipped': 0, + 'errors': 0, + 'duration': 0.0 + } + + try: + # Get all series from Sonarr + all_series = self.sonarr.get_all_series() + if not all_series: + _log("WARNING", "No series found in Sonarr") + return stats + + stats['total_series'] = len(all_series) + _log("INFO", f"Found {stats['total_series']} series in Sonarr") + + # Process each series + for series in all_series: + try: + imdb_id = series.get('imdbId') + if not imdb_id: + _log("DEBUG", f"Skipping series without IMDb ID: {series.get('title')}") + continue + + series_id = series.get('id') + series_path = series.get('path', '') + series_title = series.get('title', 'Unknown') + + # Update series record + self.db.upsert_series(imdb_id, series_path) + + # Get all episodes for this series + episodes = self.sonarr.episodes_for_series(series_id) + if not episodes: + continue + + _log("DEBUG", f"Processing {len(episodes)} episodes for {series_title}") + + # Process each episode + for episode in episodes: + try: + season_num = episode.get('seasonNumber', 0) + episode_num = episode.get('episodeNumber', 0) + + if season_num < 0 or episode_num <= 0: + continue + + stats['total_episodes'] += 1 + + # Check if episode already exists + existing = self.db.get_episode_date(imdb_id, season_num, episode_num) + if existing and existing.get('dateadded'): + stats['skipped'] += 1 + continue + + # Get air date + aired = episode.get('airDate') + + # Get import date from history + dateadded = None + episode_id = episode.get('id') + if episode_id and episode.get('hasFile'): + import_date = self.sonarr.get_episode_import_history(episode_id) + if import_date: + dateadded = import_date + source = 'sonarr:import_history' + + # Fallback to air date if no import date + if not dateadded and aired: + dateadded = aired + source = 'sonarr:aired_fallback' + elif not dateadded: + # No date available + stats['skipped'] += 1 + continue + + # Insert into database + has_file = episode.get('hasFile', False) + self.db.upsert_episode_date(imdb_id, season_num, episode_num, aired, dateadded, source, has_file) + stats['added'] += 1 + + except Exception as e: + _log("ERROR", f"Error processing episode S{season_num:02d}E{episode_num:02d} of {series_title}: {e}") + stats['errors'] += 1 + continue + + except Exception as e: + _log("ERROR", f"Error processing series {series.get('title', 'unknown')}: {e}") + stats['errors'] += 1 + continue + + except Exception as e: + _log("ERROR", f"Error during TV episode population: {e}") + stats['errors'] += 1 + + stats['duration'] = time.time() - start_time + _log("INFO", f"TV episode population complete: {stats['added']} added, {stats['skipped']} skipped, {stats['errors']} errors in {stats['duration']:.2f}s") + return stats + + def populate_all(self) -> Dict[str, any]: + """ + Populate both movies and TV episodes + + Returns: + Combined statistics dictionary + """ + _log("INFO", "Starting full database population") + start_time = time.time() + + movie_stats = self.populate_movies() + tv_stats = self.populate_tv_episodes() + + combined_stats = { + 'movies': movie_stats, + 'tv': tv_stats, + 'total_duration': time.time() - start_time + } + + _log("INFO", f"Full database population complete in {combined_stats['total_duration']:.2f}s") + return combined_stats diff --git a/core/episode_nfo_manager.py b/core/episode_nfo_manager.py deleted file mode 100644 index 24b1858..0000000 --- a/core/episode_nfo_manager.py +++ /dev/null @@ -1,276 +0,0 @@ -#!/usr/bin/env python3 -""" -Episode NFO Manager - Handles TV episode NFO creation with video filename matching -Core principle: NFO filenames should match video filenames -""" -import xml.etree.ElementTree as ET -from pathlib import Path -from typing import Optional, Dict, Any, List, Tuple -import re -from .logging import _log - - -class EpisodeNFOManager: - """Manages episode NFO files with video filename matching""" - - def __init__(self, manager_brand: str = "NFOGuard"): - self.manager_brand = manager_brand - - def find_video_files_for_season(self, season_dir: Path) -> Dict[Tuple[int, int], List[Path]]: - """Find all video files in season directory, grouped by (season, episode)""" - if not season_dir.exists(): - _log("DEBUG", f"Season directory does not exist: {season_dir}") - return {} - - video_extensions = [".mkv", ".mp4", ".avi", ".mov", ".m4v"] - episodes = {} - - _log("DEBUG", f"Scanning video files in: {season_dir}") - for video_file in season_dir.iterdir(): - _log("DEBUG", f"Checking file: {video_file.name} (is_file: {video_file.is_file()}, suffix: {video_file.suffix.lower()})") - if (video_file.is_file() and - video_file.suffix.lower() in video_extensions): - - episode_info = self._parse_episode_from_filename(video_file.name) - _log("DEBUG", f"Episode parsing for '{video_file.name}': {episode_info}") - if episode_info: - season_num, episode_num = episode_info - key = (season_num, episode_num) - if key not in episodes: - episodes[key] = [] - episodes[key].append(video_file) - _log("DEBUG", f"Added video file: S{season_num:02d}E{episode_num:02d} → {video_file.name}") - - _log("DEBUG", f"Total video files found: {len(episodes)} episodes") - return episodes - - def _parse_episode_from_filename(self, filename: str) -> Optional[Tuple[int, int]]: - """Extract season and episode numbers from filename""" - # Try S##E## format first (most common) - match = re.search(r'[Ss](\d{1,2})[Ee](\d{1,2})', filename) - if match: - return int(match.group(1)), int(match.group(2)) - - # Try ##x## format - match = re.search(r'(\d{1,2})x(\d{1,2})', filename) - if match: - return int(match.group(1)), int(match.group(2)) - - return None - - def find_nfo_for_episode(self, season_dir: Path, season_num: int, episode_num: int) -> Optional[Path]: - """Find existing NFO file for episode (prefer video-matching filename)""" - if not season_dir.exists(): - return None - - # First, look for NFO files that match video filenames - video_files = self.find_video_files_for_season(season_dir) - key = (season_num, episode_num) - - if key in video_files: - for video_file in video_files[key]: - potential_nfo = season_dir / f"{video_file.stem}.nfo" - if potential_nfo.exists(): - _log("DEBUG", f"Found video-matching NFO: {potential_nfo.name}") - return potential_nfo - - # Fallback: look for short name NFO - short_nfo = season_dir / f"S{season_num:02d}E{episode_num:02d}.nfo" - if short_nfo.exists(): - _log("DEBUG", f"Found short-name NFO: {short_nfo.name}") - return short_nfo - - # Last resort: search all NFO files for matching season/episode data - for nfo_file in season_dir.glob("*.nfo"): - if self._nfo_matches_episode(nfo_file, season_num, episode_num): - _log("DEBUG", f"Found matching NFO by content: {nfo_file.name}") - return nfo_file - - return None - - def _nfo_matches_episode(self, nfo_path: Path, season_num: int, episode_num: int) -> bool: - """Check if NFO file contains the specified season/episode""" - try: - tree = ET.parse(nfo_path) - root = tree.getroot() - - if root.tag == "episodedetails": - season_elem = root.find("season") - episode_elem = root.find("episode") - - if (season_elem is not None and episode_elem is not None and - season_elem.text and episode_elem.text): - try: - file_season = int(season_elem.text) - file_episode = int(episode_elem.text) - return file_season == season_num and file_episode == episode_num - except ValueError: - pass - except (ET.ParseError, Exception): - pass - - return False - - def get_target_nfo_path(self, season_dir: Path, season_num: int, episode_num: int) -> Path: - """Get the target NFO path (prefer video filename, fallback to short name)""" - video_files = self.find_video_files_for_season(season_dir) - key = (season_num, episode_num) - - if key in video_files and video_files[key]: - # Use the first video file found (handle multiple files gracefully) - video_file = video_files[key][0] - target_nfo = season_dir / f"{video_file.stem}.nfo" - _log("DEBUG", f"Target NFO will match video: {target_nfo.name}") - return target_nfo - else: - # Fallback to short name if no video file found - target_nfo = season_dir / f"S{season_num:02d}E{episode_num:02d}.nfo" - _log("WARNING", f"No video file found for S{season_num:02d}E{episode_num:02d}, using short name: {target_nfo.name}") - return target_nfo - - def migrate_nfo_to_video_filename(self, season_dir: Path, season_num: int, episode_num: int) -> bool: - """If short-name NFO exists, rename it to match video filename""" - existing_nfo = self.find_nfo_for_episode(season_dir, season_num, episode_num) - target_nfo = self.get_target_nfo_path(season_dir, season_num, episode_num) - - # If we already have the right filename, nothing to do - if existing_nfo and existing_nfo == target_nfo: - return True - - # If we have an NFO but it doesn't match target, rename it - if existing_nfo and existing_nfo != target_nfo: - try: - _log("INFO", f"Migrating NFO filename: {existing_nfo.name} -> {target_nfo.name}") - existing_nfo.rename(target_nfo) - return True - except Exception as e: - _log("ERROR", f"Failed to rename NFO: {e}") - return False - - return False - - def create_episode_nfo(self, season_dir: Path, season_num: int, episode_num: int, - aired: Optional[str], dateadded: Optional[str], source: str, - title: Optional[str] = None, plot: Optional[str] = None) -> bool: - """Create or update episode NFO with video filename matching""" - - # Get the target NFO path (matching video filename) - nfo_path = self.get_target_nfo_path(season_dir, season_num, episode_num) - - # Migrate existing NFO if needed - self.migrate_nfo_to_video_filename(season_dir, season_num, episode_num) - - try: - # Load existing NFO if it exists - episode_elem = None - if nfo_path.exists(): - try: - tree = ET.parse(nfo_path) - episode_elem = tree.getroot() - - if episode_elem.tag != "episodedetails": - raise ValueError("Root element is not ") - - # Remove NFOGuard-managed fields (we'll re-add them) - for tag in ["season", "episode", "aired", "premiered", "dateadded", "lockdata"]: - existing = episode_elem.find(tag) - if existing is not None: - episode_elem.remove(existing) - - _log("DEBUG", f"Loaded existing NFO content for {nfo_path.name}") - - except (ET.ParseError, ValueError) as e: - _log("WARNING", f"Corrupted NFO file {nfo_path.name}: {e}. Creating new one.") - episode_elem = None - - # Create new structure if needed - if episode_elem is None: - episode_elem = ET.Element("episodedetails") - - # Add title if provided and not already present - if title and not episode_elem.find("title"): - title_elem = ET.SubElement(episode_elem, "title") - title_elem.text = title - - # Add plot if provided and not already present - if plot and not episode_elem.find("plot"): - plot_elem = ET.SubElement(episode_elem, "plot") - plot_elem.text = plot - - # Add NFOGuard fields at the end - season_elem = ET.SubElement(episode_elem, "season") - season_elem.text = str(season_num) - - episode_num_elem = ET.SubElement(episode_elem, "episode") - episode_num_elem.text = str(episode_num) - - if aired: - aired_elem = ET.SubElement(episode_elem, "aired") - aired_elem.text = aired[:10] if len(aired) >= 10 else aired - - # Also add premiered for compatibility - premiered_elem = ET.SubElement(episode_elem, "premiered") - premiered_elem.text = aired[:10] if len(aired) >= 10 else aired - - if dateadded: - dateadded_elem = ET.SubElement(episode_elem, "dateadded") - dateadded_elem.text = dateadded - - # Add lockdata - lockdata_elem = ET.SubElement(episode_elem, "lockdata") - lockdata_elem.text = "true" - - # Add comment with source at the bottom - comment = ET.Comment(f" Created by {self.manager_brand} - Source: {source} ") - episode_elem.append(comment) - - # Write the NFO file - tree = ET.ElementTree(episode_elem) - ET.indent(tree, space=" ", level=0) - tree.write(nfo_path, encoding='utf-8', xml_declaration=True) - - _log("INFO", f"✅ Created/updated episode NFO: {nfo_path.name}") - _log("INFO", f" S{season_num:02d}E{episode_num:02d}, Aired: {aired}, DateAdded: {dateadded}, Source: {source}") - - return True - - except Exception as e: - _log("ERROR", f"Failed to create episode NFO {nfo_path}: {e}") - return False - - def extract_nfoguard_data(self, nfo_path: Path) -> Optional[Dict[str, str]]: - """Extract NFOGuard-managed data from existing NFO""" - if not nfo_path.exists(): - return None - - try: - tree = ET.parse(nfo_path) - root = tree.getroot() - - if root.tag != "episodedetails": - return None - - # Look for NFOGuard fields - dateadded_elem = root.find("dateadded") - aired_elem = root.find("aired") - lockdata_elem = root.find("lockdata") - - # Only consider it NFOGuard-managed if it has dateadded and lockdata - if (dateadded_elem is not None and dateadded_elem.text and - lockdata_elem is not None and lockdata_elem.text == "true"): - - result = { - "dateadded": dateadded_elem.text.strip(), - "source": "existing_nfo" - } - - if aired_elem is not None and aired_elem.text: - result["aired"] = aired_elem.text.strip() - - _log("DEBUG", f"Found NFOGuard data in {nfo_path.name}: {result}") - return result - - except (ET.ParseError, Exception) as e: - _log("WARNING", f"Error parsing NFO {nfo_path.name}: {e}") - - return None \ No newline at end of file diff --git a/core/nfo_manager.py b/core/nfo_manager.py deleted file mode 100644 index 473c771..0000000 --- a/core/nfo_manager.py +++ /dev/null @@ -1,890 +0,0 @@ -#!/usr/bin/env python3 -""" -NFO Manager for creating and managing metadata files -Handles NFO creation for movies, TV shows, seasons, and episodes -""" -import os -import xml.etree.ElementTree as ET -from pathlib import Path -from datetime import datetime -from typing import Optional, Dict, Any, Tuple -import re - - -class NFOManager: - """Manages NFO file creation and updates""" - - def __init__(self, manager_brand: str = "NFOGuard", debug: bool = False): - self.manager_brand = manager_brand - self.debug = debug - - def parse_imdb_from_path(self, path: Path) -> Optional[str]: - """Extract IMDb ID from directory path or filename""" - # Look for various IMDb patterns in both directory and file names - path_str = str(path).lower() - - # Try [imdb-ttXXXXXXX] format first (most explicit) - match = re.search(r'\[imdb-?(tt\d+)\]', path_str) - if match: - return match.group(1) - - # Try standalone [ttXXXXXXX] format in brackets - match = re.search(r'\[(tt\d+)\]', path_str) - if match: - return match.group(1) - - # Try {imdb-ttXXXXXXX} format with curly braces - match = re.search(r'\{imdb-?(tt\d+)\}', path_str) - if match: - return match.group(1) - - # Try (imdb-ttXXXXXXX) format with parentheses - match = re.search(r'\(imdb-?(tt\d+)\)', path_str) - if match: - return match.group(1) - - # Try ttXXXXXXX at end of filename/dirname (common pattern) - match = re.search(r'[-_\s](tt\d+)$', path_str) - if match: - return match.group(1) - - return None - - def parse_imdb_from_path_with_nfo_fallback(self, path: Path, sonarr_client=None, shutdown_event=None) -> Optional[str]: - """ - Enhanced IMDb detection that fallback to Sonarr ID lookup from NFO files - - 1. First try to parse IMDb ID from directory path/name - 2. If not found, scan NFO files in directory for Sonarr series ID - 3. Use Sonarr API to lookup series and extract IMDb ID - """ - # Primary: Try directory name first - imdb_id = self.parse_imdb_from_path(path) - if imdb_id: - return imdb_id - - # Check for shutdown signal before expensive operations - if shutdown_event and shutdown_event.is_set(): - return None - - # Fallback: Check NFO files for Sonarr series ID - if not sonarr_client or not sonarr_client.enabled: - return None - - try: - # Look for episode NFO files in the directory (and subdirectories) - nfo_files = [] - if path.is_dir(): - # Check current directory and season subdirectories - nfo_files.extend(list(path.glob("*.nfo"))) - for subdir in path.iterdir(): - if subdir.is_dir() and subdir.name.lower().startswith('season'): - nfo_files.extend(list(subdir.glob("*.nfo"))) - - # Extract Sonarr series ID from any NFO file - for nfo_file in nfo_files: - # Check for shutdown signal during NFO processing - if shutdown_event and shutdown_event.is_set(): - return None - - try: - tree = ET.parse(nfo_file) - root = tree.getroot() - - # Look for Sonarr series ID - for uniqueid in root.findall('.//uniqueid[@type="sonarr"]'): - sonarr_id = uniqueid.text - if sonarr_id and sonarr_id.isdigit(): - # Check for shutdown signal before API call - if shutdown_event and shutdown_event.is_set(): - return None - - # Look up series in Sonarr to get IMDb ID - series_data = sonarr_client.get_series_by_id(int(sonarr_id)) - if series_data: - imdb_id = series_data.get('imdbId') - if imdb_id: - return imdb_id.replace('tt', '') if imdb_id.startswith('tt') else imdb_id - - except Exception as e: - continue # Skip this NFO file and try the next one - - except Exception as e: - pass # Fallback failed, return None - - return None - - def parse_imdb_from_nfo(self, nfo_path: Path) -> Optional[str]: - """Extract IMDb ID from NFO file content""" - if not nfo_path.exists(): - return None - - try: - root = self._parse_nfo_with_tolerance(nfo_path) - - # Check for ttXXXXXX - imdb_uniqueid = root.find('.//uniqueid[@type="imdb"]') - if imdb_uniqueid is not None and imdb_uniqueid.text: - imdb_id = imdb_uniqueid.text.strip() - if imdb_id.startswith('tt'): - return imdb_id - - # Check for legacy ttXXXXXX - imdbid_elem = root.find('.//imdbid') - if imdbid_elem is not None and imdbid_elem.text: - imdb_id = imdbid_elem.text.strip() - if imdb_id.startswith('tt'): - return imdb_id - - # Check for legacy ttXXXXXX - imdb_elem = root.find('.//imdb') - if imdb_elem is not None and imdb_elem.text: - imdb_id = imdb_elem.text.strip() - if imdb_id.startswith('tt'): - return imdb_id - - # Last resort: Check for TMDB ID as fallback identifier - # This handles movies that only have TMDB IDs in NFO files - tmdb_uniqueid = root.find('.//uniqueid[@type="tmdb"]') - if tmdb_uniqueid is not None and tmdb_uniqueid.text: - tmdb_id = tmdb_uniqueid.text.strip() - if tmdb_id.isdigit(): - print(f"⚠ī¸ Found TMDB ID {tmdb_id} but no IMDb ID - using TMDB ID as fallback") - # Return TMDB ID with prefix to distinguish from IMDb IDs - return f"tmdb-{tmdb_id}" - - except (ET.ParseError, Exception): - # Skip corrupted or non-XML files - pass - - return None - - def find_movie_imdb_id(self, movie_dir: Path) -> Optional[str]: - """Find IMDb ID from directory name, filenames, or NFO file""" - # First try directory name - imdb_id = self.parse_imdb_from_path(movie_dir) - if imdb_id: - return imdb_id - - # Try all files in the directory for IMDb ID patterns - for file_path in movie_dir.iterdir(): - if file_path.is_file(): - imdb_id = self.parse_imdb_from_path(file_path) - if imdb_id: - return imdb_id - - # Finally, try NFO file content (including TMDB fallback) - nfo_path = movie_dir / "movie.nfo" - imdb_id = self.parse_imdb_from_nfo(nfo_path) - if imdb_id: - return imdb_id - - return None - - def find_series_imdb_id(self, series_dir: Path) -> Optional[str]: - """Find IMDb ID from TV series directory name, filenames, or tvshow.nfo file""" - # First try directory name - imdb_id = self.parse_imdb_from_path(series_dir) - if imdb_id: - return imdb_id - - # Try all files in the directory for IMDb ID patterns - for file_path in series_dir.iterdir(): - if file_path.is_file(): - imdb_id = self.parse_imdb_from_path(file_path) - if imdb_id: - return imdb_id - - # Finally, try tvshow.nfo file content - nfo_path = series_dir / "tvshow.nfo" - imdb_id = self.parse_imdb_from_nfo(nfo_path) - if imdb_id: - return imdb_id - - return None - - def extract_nfoguard_dates_from_nfo(self, nfo_path: Path) -> Optional[Dict[str, str]]: - """Extract NFOGuard-managed dates from existing NFO file""" - if not nfo_path.exists(): - return None - - try: - tree = ET.parse(nfo_path) - root = tree.getroot() - - # Look for NFOGuard fields - dateadded_elem = root.find('.//dateadded') - premiered_elem = root.find('.//premiered') - aired_elem = root.find('.//aired') # For TV episodes - lockdata_elem = root.find('.//lockdata') - - # Consider it NFOGuard-managed ONLY if it has BOTH lockdata=true AND dateadded - # This prevents incomplete NFO files from being marked as "complete" - if (lockdata_elem is not None and lockdata_elem.text == "true" and - dateadded_elem is not None and dateadded_elem.text): - # Extract original source from NFOGuard comment, default to nfo_file_existing - source = "nfo_file_existing" - - # Parse XML content to find NFOGuard comment with source - nfo_content = nfo_path.read_text(encoding='utf-8') - import re - source_match = re.search(r'', nfo_content) - if source_match: - source = source_match.group(1).strip() - print(f"🔍 Extracted original source from NFO comment: {source}") - - result = { - "source": source - } - - if dateadded_elem is not None and dateadded_elem.text: - result["dateadded"] = dateadded_elem.text.strip() - - if premiered_elem is not None and premiered_elem.text: - result["released"] = premiered_elem.text.strip() - - if aired_elem is not None and aired_elem.text: - result["aired"] = aired_elem.text.strip() - - print(f"✅ Found NFOGuard data in NFO: dateadded={result.get('dateadded', 'None')}, source={source}, released={result.get('released', 'None')}, aired={result.get('aired', 'None')}") - return result - - except ET.ParseError as e: - # Handle malformed XML files gracefully (common with external NFO files) - print(f"🔍 NFO XML parse error (will try text fallback): {nfo_path.name}") - # Try text-based fallback for extracting NFOGuard data - return self._extract_nfoguard_data_text_fallback(nfo_path) - except Exception as e: - print(f"⚠ī¸ Unexpected error reading NFO file {nfo_path.name}: {e}") - pass - - return None - - def _extract_nfoguard_data_text_fallback(self, nfo_path: Path) -> Optional[Dict[str, str]]: - """Text-based fallback for extracting NFOGuard data from malformed XML files""" - try: - content = nfo_path.read_text(encoding='utf-8', errors='ignore') - - # Look for NFOGuard elements using regex - import re - - # Check for lockdata=true first (NFOGuard marker) - if not re.search(r'true', content, re.IGNORECASE): - return None - - # Extract dateadded - dateadded_match = re.search(r'([^<]+)', content, re.IGNORECASE) - if not dateadded_match: - return None - - dateadded = dateadded_match.group(1).strip() - if not dateadded: - return None - - result = { - "dateadded": dateadded, - "source": "nfo_file_existing" # Default source - } - - # Extract optional fields - premiered_match = re.search(r'([^<]+)', content, re.IGNORECASE) - if premiered_match: - result["released"] = premiered_match.group(1).strip() - - aired_match = re.search(r'([^<]+)', content, re.IGNORECASE) - if aired_match: - result["aired"] = aired_match.group(1).strip() - - # Extract source from NFOGuard comment - source_match = re.search(r'', content) - if source_match: - result["source"] = source_match.group(1).strip() - - print(f"✅ Extracted NFOGuard data via text fallback: dateadded={result.get('dateadded', 'None')}, source={result['source']}") - return result - - except Exception as e: - print(f"⚠ī¸ Text fallback extraction failed for {nfo_path.name}: {e}") - return None - - def extract_nfoguard_dates_from_episode_nfo(self, season_path: Path, season_num: int, episode_num: int) -> Optional[Dict[str, str]]: - """Extract NFOGuard-managed dates from existing episode NFO file""" - nfo_filename = f"s{season_num:02d}e{episode_num:02d}.nfo" - nfo_path = season_path / nfo_filename - - if not nfo_path.exists(): - return None - - try: - tree = ET.parse(nfo_path) - root = tree.getroot() - - # Look for NFOGuard fields in episode NFO - dateadded_elem = root.find('.//dateadded') - aired_elem = root.find('.//aired') - lockdata_elem = root.find('.//lockdata') - - # Consider it NFOGuard-managed ONLY if it has BOTH lockdata=true AND dateadded - # This prevents incomplete episode NFO files from being marked as "complete" - if (lockdata_elem is not None and lockdata_elem.text == "true" and - dateadded_elem is not None and dateadded_elem.text): - # Extract original source from NFOGuard comment, default to episode_nfo_existing - source = "episode_nfo_existing" - - # Parse XML content to find NFOGuard comment with source - nfo_content = nfo_path.read_text(encoding='utf-8') - import re - source_match = re.search(r'', nfo_content) - if source_match: - source = source_match.group(1).strip() - print(f"🔍 Extracted original source from episode NFO comment: {source}") - - result = { - "source": source - } - - if dateadded_elem is not None and dateadded_elem.text: - result["dateadded"] = dateadded_elem.text.strip() - - if aired_elem is not None and aired_elem.text: - result["aired"] = aired_elem.text.strip() - - print(f"✅ Found NFOGuard data in episode NFO S{season_num:02d}E{episode_num:02d}: dateadded={result.get('dateadded', 'None')}, source={source}, aired={result.get('aired', 'None')}") - return result - - except (ET.ParseError, Exception) as e: - print(f"⚠ī¸ Error parsing episode NFO for NFOGuard data: {e}") - pass - - return None - - def _parse_nfo_with_tolerance(self, nfo_path: Path): - """Parse NFO file with tolerance for URLs appended after XML""" - try: - # First try normal parsing - tree = ET.parse(nfo_path) - return tree.getroot() - except ET.ParseError as e: - # If parsing fails, try to extract just the XML part - try: - with open(nfo_path, 'r', encoding='utf-8') as f: - content = f.read() - - # Find the last tag and truncate after it - last_movie_end = content.rfind('') - if last_movie_end != -1: - xml_content = content[:last_movie_end + 8] # +8 for - - # Try parsing the truncated content - root = ET.fromstring(xml_content) - print(f"✅ Successfully parsed NFO after removing trailing content: {nfo_path.name}") - return root - else: - # Re-raise original error if we can't find - raise e - except Exception: - # Re-raise original error if our fix attempt fails - raise e - - def create_movie_nfo(self, movie_dir: Path, imdb_id: str, dateadded: str, - released: Optional[str] = None, source: str = "unknown", - lock_metadata: bool = True) -> None: - """Create or update movie.nfo file preserving existing content""" - nfo_path = movie_dir / "movie.nfo" - - # Debug output only if DEBUG=true in environment - import os - if os.environ.get("DEBUG", "false").lower() == "true": - print(f"🔍 create_movie_nfo called: imdb_id={imdb_id}, dateadded={dateadded}, released={released}, source={source}") - print(f"🔍 NFO path: {nfo_path}") - print(f"🔍 NFO exists: {nfo_path.exists()}") - - try: - # Try to load existing NFO file - if nfo_path.exists(): - try: - # Try to parse the XML, handling URLs appended after - movie = self._parse_nfo_with_tolerance(nfo_path) - - # Ensure root element is - if movie.tag != "movie": - raise ValueError("Root element is not ") - - # Only remove elements that are clearly NFOGuard-managed - # Look for elements that have NFOGuard characteristics - elements_to_remove = [] - - # Remove lockdata=true (this is definitely NFOGuard) - for lockdata in movie.findall("lockdata"): - if lockdata.text == "true": - elements_to_remove.append(lockdata) - - # Remove IMDb uniqueids that we manage - for uniqueid in movie.findall("uniqueid[@type='imdb']"): - elements_to_remove.append(uniqueid) - - # For dateadded/premiered/year, only remove if they appear to be NFOGuard-managed - # (i.e., if lockdata=true exists, these are likely ours) - has_nfoguard_lockdata = any(ld.text == "true" for ld in movie.findall("lockdata")) - - if has_nfoguard_lockdata: - # This NFO was managed by NFOGuard, safe to remove our fields - for tag in ["dateadded", "premiered", "year"]: - existing = movie.find(tag) - if existing is not None: - # Store the value before removing (for premiered/year) - if tag == "premiered" and not released: - print(f"🔍 Preserving existing premiered date: {existing.text}") - released = existing.text - elements_to_remove.append(existing) - else: - # No NFOGuard lockdata found, be more conservative - # Only remove dateadded if it looks like NFOGuard format (ISO timestamp) - dateadded_elem = movie.find("dateadded") - if dateadded_elem is not None and dateadded_elem.text: - # NFOGuard uses ISO format like "2025-10-12 16:26:02" - if re.match(r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}', dateadded_elem.text.strip()): - elements_to_remove.append(dateadded_elem) - - # Remove all identified elements - for elem in elements_to_remove: - movie.remove(elem) - - except (ET.ParseError, ValueError) as e: - print(f"⚠ī¸ Corrupted NFO detected: {nfo_path} - {str(e)[:100]}...") - print(f" Creating new clean NFO file to replace corrupted one") - movie = ET.Element("movie") - else: - # Create new NFO structure - movie = ET.Element("movie") - - # Create all NFOGuard elements first, then append them in correct order - # This ensures they appear as a group at the bottom of the file - nfoguard_elements = [] - - # Add IMDb uniqueid - uniqueid = ET.Element("uniqueid", type="imdb", default="true") - uniqueid.text = imdb_id - nfoguard_elements.append(uniqueid) - - # Add premiered date if we have it - if released: - premiered_elem = ET.Element("premiered") - premiered_elem.text = released[:10] if len(released) >= 10 else released - nfoguard_elements.append(premiered_elem) - - # Extract year from premiered date for consistency - try: - year_value = released[:4] if len(released) >= 4 else None - if year_value and year_value.isdigit(): - year_elem = ET.Element("year") - year_elem.text = year_value - nfoguard_elements.append(year_elem) - except: - pass # Skip year if we can't extract it - - # Add dateadded - THIS IS CRITICAL FOR EMBY PLUGIN - if os.environ.get("DEBUG", "false").lower() == "true": - print(f"🔍 About to add dateadded: {dateadded} (type: {type(dateadded)})") - if dateadded: - dateadded_elem = ET.Element("dateadded") - dateadded_elem.text = dateadded - nfoguard_elements.append(dateadded_elem) - print(f"✅ Adding dateadded to NFO: {dateadded}") - else: - print(f"❌ dateadded is empty/None, not adding to NFO") - - # Add lockdata at the very end - if lock_metadata: - lockdata = ET.Element("lockdata") - lockdata.text = "true" - nfoguard_elements.append(lockdata) - - # Add NFOGuard comment as the very last element (appears at bottom) - nfoguard_comment = ET.Comment(f" NFOGuard - Source: {source} ") - nfoguard_elements.append(nfoguard_comment) - - # Now append all NFOGuard elements to the movie in one batch - # This ensures they appear as a contiguous block at the bottom - for elem in nfoguard_elements: - movie.append(elem) - - print(f"✅ Added {len(nfoguard_elements)} NFOGuard elements to bottom of NFO") - - # Write file with proper formatting - tree = ET.ElementTree(movie) - ET.indent(tree, space=" ", level=0) - - # Write directly to file (comment is already embedded in XML at bottom) - with open(nfo_path, 'w', encoding='utf-8') as f: - f.write('\n') - tree.write(f, encoding='unicode', xml_declaration=False) - - print(f"✅ Successfully created/updated movie NFO: {nfo_path}") - print(f" IMDb ID: {imdb_id}, Date Added: {dateadded}, Source: {source}") - - except Exception as e: - print(f"❌ Error creating/updating movie NFO {nfo_path}: {e}") - - def create_tvshow_nfo(self, series_dir: Path, imdb_id: str, tvdb_id: Optional[str] = None) -> None: - """Create or update tvshow.nfo file preserving existing content""" - nfo_path = series_dir / "tvshow.nfo" - - try: - # Try to load existing NFO file - if nfo_path.exists(): - try: - tree = ET.parse(nfo_path) - tvshow = tree.getroot() - - # Ensure root element is - if tvshow.tag != "tvshow": - raise ValueError("Root element is not ") - - # Remove existing NFOGuard-managed elements to avoid duplicates - # These will be re-added at the bottom - for tag in ["lockdata"]: - existing = tvshow.find(tag) - if existing is not None: - tvshow.remove(existing) - - # Remove ALL existing uniqueid with type="imdb" regardless of attributes - for uniqueid in tvshow.findall("uniqueid[@type='imdb']"): - tvshow.remove(uniqueid) - - except (ET.ParseError, ValueError) as e: - print(f"⚠ī¸ Corrupted TV show NFO detected: {nfo_path} - {str(e)[:100]}...") - print(f" Creating new clean tvshow.nfo file to replace corrupted one") - tvshow = ET.Element("tvshow") - else: - # Create new NFO structure - tvshow = ET.Element("tvshow") - - # Add NFOGuard fields at the bottom - - # Add IMDb uniqueid at the end - imdb_uniqueid = ET.SubElement(tvshow, "uniqueid", type="imdb", default="true") - imdb_uniqueid.text = imdb_id - - # Add TVDB ID if available (preserve existing or add new) - if tvdb_id and not tvshow.find("uniqueid[@type='tvdb']"): - tvdb_uniqueid = ET.SubElement(tvshow, "uniqueid", type="tvdb") - tvdb_uniqueid.text = tvdb_id - - # Add lockdata at the very end - lockdata = ET.SubElement(tvshow, "lockdata") - lockdata.text = "true" - - # Add NFOGuard comment at the bottom - comment = ET.Comment(f" Created by {self.manager_brand} ") - tvshow.append(comment) - - # Write file with proper formatting - tree = ET.ElementTree(tvshow) - ET.indent(tree, space=" ", level=0) - tree.write(nfo_path, encoding='utf-8', xml_declaration=True) - - print(f"✅ Successfully created/updated TV show NFO: {nfo_path}") - print(f" IMDb ID: {imdb_id}" + (f", TVDB ID: {tvdb_id}" if tvdb_id else "")) - - except Exception as e: - print(f"❌ Error creating/updating tvshow NFO {nfo_path}: {e}") - - def create_season_nfo(self, season_dir: Path, season_number: int) -> None: - """Create or update season.nfo file preserving existing content""" - nfo_path = season_dir / "season.nfo" - - try: - season_dir.mkdir(exist_ok=True) - - # Try to load existing NFO file - if nfo_path.exists(): - try: - tree = ET.parse(nfo_path) - season = tree.getroot() - - # Ensure root element is - if season.tag != "season": - raise ValueError("Root element is not ") - - # Remove existing NFOGuard-managed elements - for tag in ["seasonnumber", "lockdata"]: - existing = season.find(tag) - if existing is not None: - season.remove(existing) - - except (ET.ParseError, ValueError) as e: - print(f"⚠ī¸ Corrupted season NFO detected: {nfo_path} - {str(e)[:100]}...") - print(f" Creating new clean season.nfo file to replace corrupted one") - season = ET.Element("season") - else: - # Create new NFO structure - season = ET.Element("season") - - # Add NFOGuard fields at the bottom - seasonnumber = ET.SubElement(season, "seasonnumber") - seasonnumber.text = str(season_number) - - # Add lockdata at the end - lockdata = ET.SubElement(season, "lockdata") - lockdata.text = "true" - - # Add NFOGuard comment at the bottom - comment = ET.Comment(f" Created by {self.manager_brand} ") - season.append(comment) - - # Write file with proper formatting - tree = ET.ElementTree(season) - ET.indent(tree, space=" ", level=0) - tree.write(nfo_path, encoding='utf-8', xml_declaration=True) - - print(f"✅ Successfully created/updated season NFO: {nfo_path}") - print(f" Season: {season_number}") - - except Exception as e: - print(f"❌ Error creating/updating season NFO {nfo_path}: {e}") - - def find_existing_episode_nfo(self, season_dir: Path, season_num: int, episode_num: int) -> Optional[Path]: - """Find any existing episode NFO file that matches season/episode""" - if not season_dir.exists(): - return None - - # Look for NFO files in the season directory - for nfo_file in season_dir.glob("*.nfo"): - - # Check if this NFO contains the right season/episode - try: - tree = ET.parse(nfo_file) - root = tree.getroot() - - if root.tag == "episodedetails": - # Check for season/episode elements - season_elem = root.find("season") - episode_elem = root.find("episode") - - if (season_elem is not None and episode_elem is not None and - season_elem.text and episode_elem.text): - try: - file_season = int(season_elem.text) - file_episode = int(episode_elem.text) - - if file_season == season_num and file_episode == episode_num: - print(f"🔍 Found existing episode NFO: {nfo_file.name}") - return nfo_file - except ValueError: - continue - - except (ET.ParseError, Exception): - # Skip corrupted or non-XML files - continue - - return None - - def _get_target_episode_nfo_name(self, season_dir: Path, season_num: int, episode_num: int) -> str: - """Get target NFO filename - prefer existing NFO, then matching video file, fallback to short name""" - if not season_dir.exists(): - return f"S{season_num:02d}E{episode_num:02d}.nfo" - - # First check if an NFO already exists for this episode - existing_nfo = self.find_existing_episode_nfo(season_dir, season_num, episode_num) - if existing_nfo: - print(f"📂 Existing NFO found, will preserve filename: {existing_nfo.name}") - return existing_nfo.name - - # Look for video files with matching season/episode - video_extensions = [".mkv", ".mp4", ".avi", ".mov", ".m4v", ".wmv", ".flv", ".webm"] - - for video_file in season_dir.iterdir(): - if (video_file.is_file() and - video_file.suffix.lower() in video_extensions): - - # Parse episode info from video filename - episode_info = self._parse_episode_from_filename(video_file.name) - if episode_info and episode_info == (season_num, episode_num): - # Found matching video file - use its name for NFO - target_nfo_name = f"{video_file.stem}.nfo" - print(f"đŸŽ¯ Target NFO will match video: {target_nfo_name}") - return target_nfo_name - - # Fallback to short name if no matching video found - short_name = f"S{season_num:02d}E{episode_num:02d}.nfo" - print(f"⚠ī¸ No matching video file found, using short name: {short_name}") - return short_name - - def _parse_episode_from_filename(self, filename: str) -> Optional[Tuple[int, int]]: - """Parse season and episode numbers from filename""" - # Try S##E## format first - match = re.search(r'[Ss](\d{1,2})[Ee](\d{1,2})', filename) - if match: - return int(match.group(1)), int(match.group(2)) - - # Try ##x## format - match = re.search(r'(\d{1,2})x(\d{1,2})', filename) - if match: - return int(match.group(1)), int(match.group(2)) - - return None - - def create_episode_nfo(self, season_dir: Path, season_num: int, episode_num: int, - aired: Optional[str], dateadded: Optional[str], source: str, - lock_metadata: bool = True, enhanced_metadata: Optional[Dict[str, Any]] = None) -> None: - """Create or update episode NFO file preserving existing content""" - # Get target NFO filename (prefer long name matching video file) - target_nfo_name = self._get_target_episode_nfo_name(season_dir, season_num, episode_num) - nfo_path = season_dir / target_nfo_name - - try: - # Check for existing NFO file at target location - source_nfo_path = nfo_path if nfo_path.exists() else None - - if source_nfo_path: - try: - tree = ET.parse(source_nfo_path) - episode = tree.getroot() - - # Ensure root element is - if episode.tag != "episodedetails": - raise ValueError("Root element is not ") - - print(f"📝 Updating existing NFO: {nfo_path.name}") - - # Preserve existing content fields and store NFOGuard-managed fields for re-adding - preserved_values = {} - - # Extract and preserve NFOGuard-managed fields (we'll re-add these at the bottom) - nfoguard_fields = ["aired", "dateadded", "lockdata", "season", "episode"] - for tag in nfoguard_fields: - existing = episode.find(tag) - if existing is not None: - # Store the value before removing - if tag == "aired" and not aired: - aired = existing.text # Preserve existing aired date - preserved_values[tag] = existing.text - episode.remove(existing) - - # Important: DO NOT remove content fields like title, plot, runtime, premiered, etc. - # These should be preserved from the long-named NFO files - - # Debug: Show what fields are preserved after removing NFOGuard fields (if DEBUG enabled) - if self.debug: - preserved_fields = [elem.tag for elem in episode] - if preserved_fields: - print(f" 🔍 Content preserved after cleanup: {', '.join(preserved_fields)}") - else: - print(f" ℹī¸ NFO contains only NFOGuard metadata (no additional content fields)") - - except (ET.ParseError, ValueError) as e: - print(f"⚠ī¸ Corrupted episode NFO detected: {nfo_path} - {str(e)[:100]}...") - print(f" Creating new clean episode NFO file to replace corrupted one") - episode = ET.Element("episodedetails") - else: - # Create new NFO structure - episode = ET.Element("episodedetails") - - # Add enhanced metadata only if not already present (preserve existing from long-named NFO) - if enhanced_metadata: - if enhanced_metadata.get("title") and not episode.find("title"): - title_elem = ET.SubElement(episode, "title") - title_elem.text = enhanced_metadata["title"] - - if enhanced_metadata.get("overview") and not episode.find("plot"): - plot_elem = ET.SubElement(episode, "plot") - plot_elem.text = enhanced_metadata["overview"] - - if enhanced_metadata.get("runtime") and not episode.find("runtime"): - runtime_elem = ET.SubElement(episode, "runtime") - runtime_elem.text = str(enhanced_metadata["runtime"]) - - # Add NFOGuard fields at the bottom - - # Basic episode info at the end - season_elem = ET.SubElement(episode, "season") - season_elem.text = str(season_num) - - episode_elem = ET.SubElement(episode, "episode") - episode_elem.text = str(episode_num) - - # Dates at the end - if aired: - aired_elem = ET.SubElement(episode, "aired") - # Convert datetime objects to strings - aired_str = str(aired) - aired_elem.text = aired_str[:10] if len(aired_str) >= 10 else aired_str - - # Debug logging for dateadded - print(f"🔍 DEBUG: dateadded value: {repr(dateadded)} (type: {type(dateadded)})") - if dateadded: - dateadded_elem = ET.SubElement(episode, "dateadded") - # Convert datetime objects to strings - dateadded_elem.text = str(dateadded) - print(f"✅ Added dateadded to episode NFO: {dateadded}") - else: - print(f"❌ dateadded is empty/None, not adding to episode NFO") - - # Add lockdata at the very end - if lock_metadata: - lockdata = ET.SubElement(episode, "lockdata") - lockdata.text = "true" - - # Add NFOGuard comment at the bottom - comment = ET.Comment(f" Created by {self.manager_brand} - Source: {source} ") - episode.append(comment) - - # Write file with proper formatting - tree = ET.ElementTree(episode) - ET.indent(tree, space=" ", level=0) - tree.write(nfo_path, encoding='utf-8', xml_declaration=True) - - print(f"✅ Successfully created/updated episode NFO: {nfo_path}") - print(f" S{season_num:02d}E{episode_num:02d}, Aired: {aired}, Date Added: {dateadded}") - - # NFO file created/updated successfully - pass - except Exception as e: - print(f"❌ Error creating/updating episode NFO {nfo_path}: {e}") - - def set_file_mtime(self, file_path: Path, iso_timestamp) -> None: - """Set file modification time to match import date""" - try: - # Convert datetime objects to strings first - if hasattr(iso_timestamp, 'isoformat'): - iso_timestamp = iso_timestamp.isoformat() - elif not isinstance(iso_timestamp, str): - iso_timestamp = str(iso_timestamp) - - # Parse ISO timestamp - if iso_timestamp.endswith('Z'): - dt = datetime.fromisoformat(iso_timestamp.replace('Z', '+00:00')) - elif '+' in iso_timestamp or 'T' in iso_timestamp: - dt = datetime.fromisoformat(iso_timestamp) - elif ' ' in iso_timestamp: - # Handle space-separated datetime format (e.g., "2025-10-16 20:31:22") - dt = datetime.fromisoformat(iso_timestamp.replace(' ', 'T')) - else: - # Assume it's a simple date (e.g., "2025-10-16") - dt = datetime.fromisoformat(iso_timestamp + 'T00:00:00') - - # Convert to timestamp - timestamp = dt.timestamp() - - # Set both access and modification times - os.utime(file_path, (timestamp, timestamp)) - print(f"✅ Updated file timestamp: {file_path.name} -> {iso_timestamp}") - - except Exception as e: - print(f"❌ Error setting mtime for {file_path}: {e}") - - def update_movie_files_mtime(self, movie_dir: Path, iso_timestamp: str) -> None: - """Update modification times for all video files in movie directory""" - video_exts = (".mkv", ".mp4", ".avi", ".mov", ".m4v") - updated_files = [] - - for file_path in movie_dir.iterdir(): - if file_path.is_file() and file_path.suffix.lower() in video_exts: - self.set_file_mtime(file_path, iso_timestamp) - updated_files.append(file_path.name) - - if updated_files: - print(f"✅ Updated {len(updated_files)} video file timestamps in {movie_dir.name}") - else: - print(f"⚠ī¸ No video files found to update in {movie_dir.name}") \ No newline at end of file diff --git a/main.py b/main.py index 97eb35a..92749e4 100644 --- a/main.py +++ b/main.py @@ -22,7 +22,7 @@ from utils.logging import _log # Import core components from core.database import NFOGuardDatabase -from core.nfo_manager import NFOManager +# from core.nfo_manager import NFOManager # Phase 3: Removed - no longer needed from core.path_mapper import PathMapper # Import clients @@ -93,20 +93,20 @@ def initialize_components(): # Initialize core components db = NFOGuardDatabase(config=config) - nfo_manager = NFOManager(config.manager_brand, config.debug) + # nfo_manager = NFOManager(config.manager_brand, config.debug) # Phase 3: Removed path_mapper = PathMapper(config) - - # Initialize processors - tv_processor = TVProcessor(db, nfo_manager, path_mapper) - movie_processor = MovieProcessor(db, nfo_manager, path_mapper) - - # Initialize webhook batcher with nfo_manager for comprehensive IMDb detection - batcher = WebhookBatcher(nfo_manager) + + # Initialize processors (nfo_manager=None for backward compatibility) + tv_processor = TVProcessor(db, None, path_mapper) + movie_processor = MovieProcessor(db, None, path_mapper) + + # Initialize webhook batcher (no longer needs nfo_manager - Phase 3) + batcher = WebhookBatcher(nfo_manager=None) batcher.set_processors(tv_processor, movie_processor) - + return { "db": db, - "nfo_manager": nfo_manager, + # "nfo_manager": nfo_manager, # Phase 3: Removed "path_mapper": path_mapper, "tv_processor": tv_processor, "movie_processor": movie_processor, diff --git a/processors/movie_processor.py b/processors/movie_processor.py index af18f13..c6e766f 100644 --- a/processors/movie_processor.py +++ b/processors/movie_processor.py @@ -11,12 +11,12 @@ from datetime import datetime, timezone from zoneinfo import ZoneInfo from core.database import NFOGuardDatabase -from core.nfo_manager import NFOManager from core.path_mapper import PathMapper from clients.radarr_client import RadarrClient from clients.external_clients import ExternalClientManager from config.settings import config from utils.logging import _log +from utils.imdb_utils import find_imdb_in_directory # Phase 3: Replaced NFOManager from utils.file_utils import find_media_path_by_imdb_and_title @@ -68,9 +68,9 @@ def convert_utc_to_local(utc_iso_string: str) -> str: class MovieProcessor: """Handles movie processing""" - def __init__(self, db: NFOGuardDatabase, nfo_manager: NFOManager, path_mapper: PathMapper): + def __init__(self, db: NFOGuardDatabase, nfo_manager, path_mapper: PathMapper): + # nfo_manager parameter kept for backward compatibility but no longer used (Phase 3) self.db = db - self.nfo_manager = nfo_manager self.path_mapper = path_mapper self.radarr = RadarrClient( os.environ.get("RADARR_URL", ""), @@ -153,7 +153,7 @@ class MovieProcessor: def process_movie(self, movie_path: Path, webhook_mode: bool = False, force_scan: bool = False, scan_mode: str = "smart", shutdown_event=None) -> str: """Process a movie directory""" - imdb_id = self.nfo_manager.find_movie_imdb_id(movie_path) + imdb_id = find_imdb_in_directory(movie_path) # Phase 3: Using imdb_utils instead of NFOManager if not imdb_id: _log("ERROR", f"No IMDb ID found in movie directory, filenames, or NFO file: {movie_path}") return "error" @@ -225,106 +225,16 @@ class MovieProcessor: if released and hasattr(released, 'isoformat'): released = released.isoformat() - # Create NFO with existing data and update files - if config.manage_nfo: - self.nfo_manager.create_movie_nfo( - movie_path, imdb_id, dateadded, released, source, config.lock_metadata - ) - - if config.fix_dir_mtimes and dateadded: - self.nfo_manager.update_movie_files_mtime(movie_path, dateadded) + # NFO file operations removed - database is now the single source of truth + # (Phase 1: Remove NFO file write operations) _log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [database-cached]") return "processed" else: _log("INFO", f"🔍 TIER 1 SKIP - {imdb_id}: Database incomplete, proceeding to Tier 2") - # TIER 2: Check if NFO file has NFOGuard data and cache it in database - nfo_path = movie_path / "movie.nfo" - _log("INFO", f"🔍 TIER 2 - Checking NFO file: {nfo_path}") - _log("INFO", f"🔍 TIER 2 - NFO exists: {nfo_path.exists()}") - - nfo_data = self.nfo_manager.extract_nfoguard_dates_from_nfo(nfo_path) - _log("INFO", f"🔍 TIER 2 - NFOGuard data extracted: {nfo_data}") - - if nfo_data: - _log("INFO", f"🚀 TIER 2 - Found NFOGuard data in NFO file: {nfo_data['dateadded']} (source: {nfo_data['source']})") - dateadded = nfo_data["dateadded"] - source = nfo_data["source"] - released = nfo_data.get("released") - - # Cache NFO data in database for future lookups - # Fixed parameter order: imdb_id, released, dateadded, source - self.db.upsert_movie_dates(imdb_id, released, dateadded, source, True) - _log("INFO", f"✅ Cached NFO data in database for {imdb_id}") - - # Update file mtimes if enabled (NFO is already correct) - if config.fix_dir_mtimes and dateadded: - self.nfo_manager.update_movie_files_mtime(movie_path, dateadded) - - _log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [nfo-cached]") - return "processed" - - # TIER 2.5: Check for any existing valid date data in NFO (even without lockdata marker) - # Only use NFO dates if prioritize_nfo is enabled, otherwise check external APIs first - existing_nfo_data = self._extract_any_valid_dates_from_nfo(nfo_path) - if existing_nfo_data and config.manual_scan_prioritize_nfo: - _log("INFO", f"🔍 TIER 2.5 - Found existing date data in NFO (no lockdata): {existing_nfo_data['dateadded']} (source: {existing_nfo_data['source']})") - _log("INFO", f"⚡ MANUAL_SCAN_PRIORITIZE_NFO=True - Using NFO date for speed") - dateadded = existing_nfo_data["dateadded"] - source = existing_nfo_data["source"] - released = existing_nfo_data.get("released") - - # Cache existing data in database and add proper NFOGuard formatting - # Fixed parameter order: imdb_id, released, dateadded, source - self.db.upsert_movie_dates(imdb_id, released, dateadded, source, True) - _log("INFO", f"✅ Cached existing NFO data in database for {imdb_id}") - - # Update NFO file to add NFOGuard formatting (lockdata, comment) - if config.manage_nfo: - self.nfo_manager.create_movie_nfo( - movie_path, imdb_id, dateadded, released, source, config.lock_metadata - ) - _log("INFO", f"✅ Added NFOGuard formatting to existing NFO for {imdb_id}") - - # Update file mtimes if enabled - if config.fix_dir_mtimes and dateadded: - self.nfo_manager.update_movie_files_mtime(movie_path, dateadded) - - _log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [existing-nfo-enhanced]") - return "processed" - elif existing_nfo_data: - _log("INFO", f"🔍 TIER 2.5 - Found existing date data in NFO (no lockdata): {existing_nfo_data['dateadded']} (source: {existing_nfo_data['source']})") - _log("INFO", f"đŸŽ¯ MANUAL_SCAN_PRIORITIZE_NFO=False - Will verify against external APIs first") - # Store NFO data as fallback but continue to TIER 3 to check external APIs - nfo_fallback_data = existing_nfo_data - - # TIER 1.5: Special handling for TMDB-only movies - extract dates from existing NFO - if is_tmdb_fallback: - tmdb_nfo_data = self._extract_dates_from_tmdb_nfo(nfo_path) - if tmdb_nfo_data: - _log("INFO", f"đŸŽŦ Using TMDB data from existing NFO file: {tmdb_nfo_data['dateadded']} (source: {tmdb_nfo_data['source']})") - dateadded = tmdb_nfo_data["dateadded"] - source = tmdb_nfo_data["source"] - released = tmdb_nfo_data.get("released") - - # Create NFO with NFOGuard fields added - if config.manage_nfo: - self.nfo_manager.create_movie_nfo( - movie_path, imdb_id, dateadded, released, source, config.lock_metadata - ) - - # Update file mtimes if enabled - if config.fix_dir_mtimes and dateadded: - self.nfo_manager.update_movie_files_mtime(movie_path, dateadded) - - # Save to database - self.db.upsert_movie_dates(imdb_id, released, dateadded, source, True) - - _log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [tmdb-nfo]") - return "processed" - - # TIER 3: No cached data found - proceed with API lookups and verification + # TIER 2: Query external APIs directly (NFO layer removed in Phase 2) + _log("INFO", f"🔍 TIER 2 - No database cache, querying external APIs") # Check for shutdown signal before expensive API operations if shutdown_event and shutdown_event.is_set(): @@ -362,18 +272,8 @@ class MovieProcessor: final_source = f"{source}_as_dateadded" if source else "release_date_fallback" _log("INFO", f"Using release date as dateadded: {final_dateadded} (source: {final_source})") - # Create NFO regardless of date availability (preserves existing metadata) - if config.debug: - print(f"🔍 TIER3 - config.manage_nfo: {config.manage_nfo}") - if config.manage_nfo: - if config.debug: - print(f"🔍 TIER3 - Calling create_movie_nfo with final_dateadded: {final_dateadded}") - self.nfo_manager.create_movie_nfo( - movie_path, imdb_id, final_dateadded, released, final_source, config.lock_metadata - ) - else: - if config.debug: - print(f"❌ TIER3 - manage_nfo is disabled, skipping NFO creation") + # NFO file operations removed - database is now the single source of truth + # (Phase 1: Remove NFO file write operations) # Skip remaining processing if no valid date found and file dates disabled if final_dateadded is None: @@ -387,9 +287,8 @@ class MovieProcessor: _log("DEBUG", f"Movie {movie_path.name} proceeding to save: dateadded={dateadded}, source={source}") - # Update file mtimes (only if we have a valid date) - if config.fix_dir_mtimes and dateadded and dateadded != "MANUAL_REVIEW_NEEDED": - self.nfo_manager.update_movie_files_mtime(movie_path, dateadded) + # File mtime operations removed - database is now the single source of truth + # (Phase 1: Remove NFO file write operations) _log("DEBUG", f"Movie processing reached file mtime section: fix_dir_mtimes={config.fix_dir_mtimes}, dateadded={dateadded}") @@ -407,76 +306,34 @@ class MovieProcessor: return "processed" def _process_movie_nfo_first(self, movie_path: Path, imdb_id: str, shutdown_event=None) -> str: - """Process movie for incomplete mode: Check NFO files first for missing dateadded elements""" - _log("INFO", f"🔍 NFO-FIRST MODE: Checking movie for missing dateadded in NFO file: {movie_path.name}") - + """Process movie for incomplete mode: Database-first then API (NFO checks removed in Phase 2)""" + _log("INFO", f"🔍 INCOMPLETE MODE: Checking movie for missing data: {movie_path.name}") + # Check for shutdown signal if shutdown_event and shutdown_event.is_set(): - _log("INFO", f"⚠ī¸ SHUTDOWN SIGNAL RECEIVED - Stopping movie NFO-first processing: {movie_path.name}") + _log("INFO", f"⚠ī¸ SHUTDOWN SIGNAL RECEIVED - Stopping movie processing: {movie_path.name}") return "shutdown" - - nfo_path = movie_path / "movie.nfo" - - # STEP 1: Check if NFO file exists and has dateadded - _log("DEBUG", f"STEP 1 - Checking NFO file for missing dateadded: {nfo_path}") - _log("INFO", f"🔍 NFO exists: {nfo_path.exists()}") - - if nfo_path.exists(): - nfo_data = self.nfo_manager.extract_nfoguard_dates_from_nfo(nfo_path) - _log("INFO", f"🔍 NFOGuard data extracted: {nfo_data}") - - if nfo_data and nfo_data.get('dateadded'): - # NFO has dateadded - this movie is complete - _log("INFO", f"✅ NFO has dateadded={nfo_data['dateadded']}, movie marked as complete") - dateadded = nfo_data["dateadded"] - source = nfo_data["source"] - released = nfo_data.get("released") - - # Cache NFO data in database for future lookups - self.db.upsert_movie_dates(imdb_id, released, dateadded, source, True) - - # Update file mtimes if needed - if config.fix_dir_mtimes and dateadded: - self.nfo_manager.update_movie_files_mtime(movie_path, dateadded) - - _log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [nfo-complete]") - return "processed" - else: - # NFO exists but missing dateadded - _log("INFO", f"🔍 NFO exists but missing dateadded - needs DB/API lookup") - else: - # No NFO file found - _log("INFO", f"🔍 No NFO file found - needs DB/API lookup") - - # STEP 2: For movies missing dateadded in NFO, check database - _log("DEBUG", f"STEP 2 - Checking database for missing dateadded") + + # STEP 1: Check database for existing data (Phase 2: NFO check removed) + _log("DEBUG", f"STEP 1 - Checking database for existing data") existing = self.db.get_movie_dates(imdb_id) - + if existing and existing.get("dateadded") and existing.get("source") != "no_valid_date_source": - # Found in database - use cached data and will add to NFO - _log("INFO", f"✅ Database has dateadded={existing['dateadded']} - will add to NFO") + # Found in database - data is complete + _log("INFO", f"✅ Database has dateadded={existing['dateadded']}") dateadded, source, released = existing["dateadded"], existing["source"], existing.get("released") - - # Convert datetime objects to strings for NFO manager + + # Convert datetime objects to strings if hasattr(dateadded, 'isoformat'): dateadded = dateadded.isoformat() if released and hasattr(released, 'isoformat'): released = released.isoformat() - - # Create NFO with database data - if config.manage_nfo: - self.nfo_manager.create_movie_nfo( - movie_path, imdb_id, dateadded, released, source, config.lock_metadata - ) - - if config.fix_dir_mtimes and dateadded: - self.nfo_manager.update_movie_files_mtime(movie_path, dateadded) - - _log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [database-to-nfo]") + + _log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [database-cached]") return "processed" - - # STEP 3: For movies still missing dateadded, query APIs - _log("DEBUG", f"STEP 3 - Querying APIs for missing dateadded") + + # STEP 2: Database incomplete or missing, query APIs + _log("DEBUG", f"STEP 2 - Querying APIs for missing data") # Check for shutdown signal before API calls if shutdown_event and shutdown_event.is_set(): @@ -487,19 +344,10 @@ class MovieProcessor: is_tmdb_fallback = imdb_id.startswith("tmdb-") if is_tmdb_fallback: - # TMDB fallback processing + # TMDB fallback processing - use file modification time _log("INFO", f"🔍 TMDB fallback processing for {imdb_id}") - - # Check for existing TMDB NFO date data - tmdb_data = self._extract_dates_from_tmdb_nfo(nfo_path) - if tmdb_data and tmdb_data.get('dateadded'): - dateadded = tmdb_data['dateadded'] - released = tmdb_data.get('released') - source = "tmdb_nfo" - else: - # Use file modification time as fallback for TMDB - dateadded, source, released = self._get_file_mtime_date(movie_path) - _log("INFO", f"Using file mtime as fallback for TMDB movie: {dateadded}") + dateadded, source, released = self._get_file_mtime_date(movie_path) + _log("INFO", f"Using file mtime for TMDB movie: {dateadded}") else: # Standard IMDb processing # Try to get digital release date from external APIs @@ -511,116 +359,22 @@ class MovieProcessor: released = digital_date # For movies, digital release is often the key date _log("INFO", f"Got digital release date from APIs: {dateadded} (source: {source})") else: - # Check Radarr NFO for premiered date - radarr_premiered = self._get_radarr_nfo_premiered_date(movie_path) - if radarr_premiered: - dateadded = radarr_premiered - source = "radarr_nfo" - released = radarr_premiered - _log("INFO", f"Got premiered date from Radarr NFO: {dateadded}") - else: - # Last resort: file modification time - dateadded, source, released = self._get_file_mtime_date(movie_path) - _log("INFO", f"Using file mtime as last resort: {dateadded}") + # Last resort: file modification time + dateadded, source, released = self._get_file_mtime_date(movie_path) + _log("INFO", f"Using file mtime as fallback: {dateadded}") - # Save to database and create NFO + # Save to database only (NFO operations removed in Phase 1) if dateadded: self.db.upsert_movie_dates(imdb_id, released, dateadded, source, True) - if config.manage_nfo: - self.nfo_manager.create_movie_nfo( - movie_path, imdb_id, dateadded, released, source, config.lock_metadata - ) - - if config.fix_dir_mtimes and dateadded: - self.nfo_manager.update_movie_files_mtime(movie_path, dateadded) - - _log("INFO", f"🔍 NFO-FIRST COMPLETE: {movie_path.name} (source: {source})") + _log("INFO", f"🔍 INCOMPLETE MODE COMPLETE: {movie_path.name} (source: {source})") return "processed" else: _log("WARNING", f"Could not determine dateadded for movie: {movie_path.name}") return "error" - def _extract_dates_from_tmdb_nfo(self, nfo_path: Path) -> Optional[Dict[str, str]]: - """Extract date information from TMDB-based NFO file""" - if not nfo_path.exists(): - return None - - try: - root = self.nfo_manager._parse_nfo_with_tolerance(nfo_path) - - # Look for premiered date (from TMDB) - premiered_elem = root.find('.//premiered') - if premiered_elem is not None and premiered_elem.text: - premiered_date = premiered_elem.text.strip() - print(f"✅ Found TMDB premiered date: {premiered_date}") - - return { - "dateadded": premiered_date, - "source": "tmdb:premiered_from_nfo", - "released": premiered_date - } - - except (ET.ParseError, Exception) as e: - print(f"⚠ī¸ Error parsing TMDB NFO for dates: {e}") - - return None - - def _extract_any_valid_dates_from_nfo(self, nfo_path: Path) -> Optional[Dict[str, str]]: - """Extract any valid date information from NFO file, even without NFOGuard markers""" - if not nfo_path.exists(): - return None - - try: - import xml.etree.ElementTree as ET - tree = ET.parse(nfo_path) - root = tree.getroot() - - # Look for dateadded element (indicates previously processed by NFOGuard or similar) - dateadded_elem = root.find('.//dateadded') - premiered_elem = root.find('.//premiered') - - if dateadded_elem is not None and dateadded_elem.text: - dateadded = dateadded_elem.text.strip() - - # Try to determine source from NFOGuard comment if present - source = "existing_nfo_data" - try: - nfo_content = nfo_path.read_text(encoding='utf-8') - import re - # Look for NFOGuard comment pattern - source_match = re.search(r'', nfo_content, re.DOTALL | re.IGNORECASE) - if source_match: - source = source_match.group(1).strip() - _log("DEBUG", f"Found source in NFOGuard comment: {source}") - else: - # Try to infer source from dateadded format/content - if "tmdb" in nfo_content.lower() or (premiered_elem and premiered_elem.text): - source = "tmdb:digital" - elif "radarr" in nfo_content.lower(): - source = "radarr:db.history.import" - else: - source = "existing_nfo_data" - _log("DEBUG", f"Inferred source from NFO content: {source}") - except Exception as e: - _log("DEBUG", f"Could not determine source from NFO content: {e}") - - result = { - "dateadded": dateadded, - "source": source - } - - if premiered_elem is not None and premiered_elem.text: - result["released"] = premiered_elem.text.strip() - - _log("INFO", f"✅ Found existing date data in NFO: dateadded={dateadded}, source={source}") - return result - - except (ET.ParseError, Exception) as e: - _log("DEBUG", f"Error parsing NFO for existing date data: {e}") - - return None - + # NFO helper methods removed in Phase 2 - database is the single source of truth + def _decide_movie_dates(self, imdb_id: str, movie_path: Path, should_query: bool, existing: Optional[Dict]) -> Tuple[str, str, Optional[str]]: """Decide movie dates based on configuration and available data""" _log("DEBUG", f"_decide_movie_dates for {imdb_id}: should_query={should_query}, existing={existing}") @@ -687,14 +441,7 @@ class MovieProcessor: # When using digital release date, store it as both dateadded and released return digital_date, digital_source, digital_date else: - _log("WARNING", f"⚠ī¸ Movie {imdb_id}: No import date OR digital release date found - trying additional fallbacks") - - # Try Radarr's own NFO premiered date as fallback - radarr_premiered = self._get_radarr_nfo_premiered_date(movie_path) - if radarr_premiered: - _log("INFO", f"✅ Movie {imdb_id}: Using Radarr NFO premiered date {radarr_premiered}") - # When using Radarr NFO premiered date, store it as both dateadded and released - return radarr_premiered, "radarr:nfo.premiered", radarr_premiered + _log("WARNING", f"⚠ī¸ Movie {imdb_id}: No import date OR digital release date found") else: # digital_then_import # Try digital release first @@ -753,32 +500,8 @@ class MovieProcessor: _log("ERROR", f"External clients error for {imdb_id}: {e}") return None, f"release:error:{str(e)}" - def _get_radarr_nfo_premiered_date(self, movie_path: Path) -> Optional[str]: - """Extract premiered date from Radarr's existing movie.nfo file""" - try: - nfo_path = movie_path / "movie.nfo" - if not nfo_path.exists(): - _log("DEBUG", f"No existing NFO file found at {nfo_path}") - return None - - nfo_content = nfo_path.read_text(encoding='utf-8') - - # Look for YYYY-MM-DD - match = re.search(r'(\d{4}-\d{2}-\d{2})', nfo_content) - if match: - premiered_date = match.group(1) - # Convert to ISO format with timezone - iso_date = f"{premiered_date}T00:00:00+00:00" - _log("INFO", f"✅ Found Radarr NFO premiered date: {premiered_date}") - return iso_date - else: - _log("DEBUG", f"No tag found in existing NFO") - return None - - except Exception as e: - _log("ERROR", f"Error reading Radarr NFO file: {e}") - return None - + # _get_radarr_nfo_premiered_date() removed in Phase 2 - no longer reading NFO files + def _log_failed_movie(self, movie_path: Path, imdb_id: str, reason: str, available_countries: List[str] = None): """Log movies that failed to get valid dates to a debug file""" try: diff --git a/processors/tv_processor.py b/processors/tv_processor.py index c6e81e6..2c3f661 100644 --- a/processors/tv_processor.py +++ b/processors/tv_processor.py @@ -11,13 +11,12 @@ from typing import Optional, Dict, List, Set, Tuple, Any from datetime import datetime from core.database import NFOGuardDatabase -from core.nfo_manager import NFOManager -from core.async_nfo_manager import AsyncNFOManager from core.path_mapper import PathMapper from clients.sonarr_client import SonarrClient from clients.external_clients import ExternalClientManager from config.settings import config from utils.logging import _log +from utils.imdb_utils import parse_imdb_from_path # Phase 3: Replaced NFOManager from utils.file_utils import ( find_media_path_by_imdb_and_title, find_episodes_on_disk, @@ -32,10 +31,9 @@ from utils.async_file_utils import ( class TVProcessor: """Handles TV series processing""" - def __init__(self, db: NFOGuardDatabase, nfo_manager: NFOManager, path_mapper: PathMapper): + def __init__(self, db: NFOGuardDatabase, nfo_manager, path_mapper: PathMapper): + # nfo_manager parameter kept for backward compatibility but no longer used (Phase 3) self.db = db - self.nfo_manager = nfo_manager - self.async_nfo_manager = AsyncNFOManager(config.manager_brand, config.debug) self.path_mapper = path_mapper self.sonarr = SonarrClient( os.environ.get("SONARR_URL", ""), @@ -147,7 +145,7 @@ class TVProcessor: def process_series(self, series_path: Path, force_scan: bool = False, scan_mode: str = "smart") -> str: """Process a TV series directory""" - imdb_id = self.nfo_manager.parse_imdb_from_path(series_path) + imdb_id = parse_imdb_from_path(series_path) # Phase 3: Using imdb_utils if not imdb_id: _log("ERROR", f"No IMDb ID found in series path: {series_path}") return "error" @@ -193,19 +191,8 @@ class TVProcessor: if (season, episode) in disk_episodes: episode_count += 1 - # Create NFO - if config.manage_nfo: - season_dir = series_path / config.get_season_dir_name(season) - self.nfo_manager.create_episode_nfo( - season_dir, - season, episode, aired, dateadded, source, config.lock_metadata - ) - - # Update file mtimes - if config.fix_dir_mtimes and dateadded: - video_files = disk_episodes[(season, episode)] - for video_file in video_files: - self.nfo_manager.set_file_mtime(video_file, dateadded) + # NFO file operations removed - database is now the single source of truth + # (Phase 1: Remove NFO file write operations) # Save to database try: @@ -259,64 +246,12 @@ class TVProcessor: # Not in database or incomplete - needs NFO check episodes_needing_nfo_check.append((season, episode)) - _log("INFO", f"Database cache hits: {db_cache_hits}/{len(disk_episodes)} episodes. Need NFO check: {len(episodes_needing_nfo_check)}") - - # TIER 2: Check NFO files for NFOGuard dates and cache them in database - nfo_cache_hits = 0 - episodes_needing_lookup = [] - - if episodes_needing_nfo_check: - _log("DEBUG", f"TIER 2 - Checking NFO files for NFOGuard dates for {len(episodes_needing_nfo_check)} episodes") - - for (season, episode) in episodes_needing_nfo_check: - # Look for existing NFO files for this episode - season_dir = series_path / config.get_season_dir_name(season) - episode_files = disk_episodes[(season, episode)] - - nfo_found = False - for episode_file in episode_files: - # Try to find matching NFO file - nfo_path = episode_file.with_suffix('.nfo') - if nfo_path.exists(): - # Extract NFOGuard data from episode NFO - nfo_data = self.nfo_manager.extract_nfoguard_dates_from_nfo(nfo_path) - if nfo_data: - aired = nfo_data.get('aired') - dateadded = nfo_data.get('dateadded') - source = nfo_data.get('source', 'nfo_cache') - - _log("DEBUG", f"S{season:02d}E{episode:02d}: NFO data found - aired={aired}, dateadded={dateadded}, source={source}") - - # Skip incomplete NFO files with "unknown" source or no useful dates - if source == "unknown" and not dateadded and not aired: - _log("INFO", f"S{season:02d}E{episode:02d}: Ignoring incomplete NFO file with source 'unknown' and no dates") - break # Break out of NFO search to mark episode as needing API lookup - - # Apply fallback logic if NFO has aired but no dateadded - if not dateadded and aired: - dateadded = aired - source = f"{source}_aired_fallback" if source != 'nfo_cache' else 'nfo_aired_fallback' - _log("DEBUG", f"S{season:02d}E{episode:02d}: NFO has aired but no dateadded, using aired as fallback: {dateadded}") - - if dateadded: - episode_dates[(season, episode)] = (aired, dateadded, source) - nfo_cache_hits += 1 - nfo_found = True - - # Cache NFO data in database for future lookups - self.db.upsert_episode_date(imdb_id, season, episode, aired, dateadded, source, True) - _log("DEBUG", f"NFO cache hit for S{season:02d}E{episode:02d}: {dateadded} - cached in DB") - break - - if not nfo_found: - # No NFO data found - needs API lookup - episodes_needing_lookup.append((season, episode)) - - _log("INFO", f"NFO cache hits: {nfo_cache_hits}/{len(episodes_needing_nfo_check)} episodes. Need API lookup: {len(episodes_needing_lookup)}") - - # TIER 3: Only call Sonarr API for episodes not in database or NFO files + _log("INFO", f"TIER 1 - Database cache hits: {db_cache_hits}/{len(disk_episodes)} episodes. Need API lookup: {len(episodes_needing_nfo_check)}") + + # TIER 2: Query Sonarr API for episodes not in database (NFO layer removed in Phase 2) + episodes_needing_lookup = episodes_needing_nfo_check # Renamed for clarity if episodes_needing_lookup: - _log("DEBUG", f"TIER 3 - Querying Sonarr for {len(episodes_needing_lookup)} episodes missing from database and NFO files") + _log("DEBUG", f"TIER 2 - Querying Sonarr for {len(episodes_needing_lookup)} episodes missing from database") sonarr_episodes = self._get_sonarr_episodes(imdb_id, episodes_needing_lookup) # Process episodes that needed lookup @@ -385,77 +320,35 @@ class TVProcessor: return episode_dates def _gather_episode_dates_nfo_first(self, series_path: Path, imdb_id: str, disk_episodes: Dict[Tuple[int, int], List[Path]]) -> Dict[Tuple[int, int], Tuple[Optional[str], Optional[str], str]]: - """Gather episode dates for incomplete mode: Check NFO files first for missing dateadded elements""" - _log("INFO", f"🔍 NFO-FIRST MODE: Checking {len(disk_episodes)} episodes for missing dateadded in NFO files") + """Gather episode dates for incomplete mode: Database-first then API (NFO checks removed in Phase 2)""" + _log("INFO", f"🔍 INCOMPLETE MODE: Checking {len(disk_episodes)} episodes for missing data") episode_dates = {} - episodes_needing_db_check = [] episodes_needing_api_lookup = [] - - # STEP 1: Check each NFO file to determine if dateadded is missing - _log("DEBUG", f"STEP 1 - Checking NFO files for missing dateadded elements") - nfo_complete_count = 0 - + + # STEP 1: Check database for all episodes (NFO check removed in Phase 2) + _log("DEBUG", f"STEP 1 - Checking database for {len(disk_episodes)} episodes") + db_hits = 0 + for (season, episode) in disk_episodes: - season_dir = series_path / config.get_season_dir_name(season) - episode_files = disk_episodes[(season, episode)] - - nfo_has_dateadded = False - nfo_data = None - - # Look for NFO file for this episode - for episode_file in episode_files: - nfo_path = episode_file.with_suffix('.nfo') - if nfo_path.exists(): - # Extract NFOGuard data from episode NFO - nfo_data = self.nfo_manager.extract_nfoguard_dates_from_nfo(nfo_path) - if nfo_data and nfo_data.get('dateadded'): - # NFO has dateadded - this episode is complete - aired = nfo_data.get('aired') - dateadded = nfo_data.get('dateadded') - source = nfo_data.get('source', 'nfo_cache') - episode_dates[(season, episode)] = (aired, dateadded, source) - nfo_complete_count += 1 - nfo_has_dateadded = True - _log("DEBUG", f"S{season:02d}E{episode:02d}: NFO has dateadded={dateadded}, marked as complete") - break - - if not nfo_has_dateadded: - # NFO missing or missing dateadded - needs further processing - if nfo_data: - # NFO exists but missing dateadded - _log("DEBUG", f"S{season:02d}E{episode:02d}: NFO exists but missing dateadded - needs DB/API lookup") - else: - # No NFO file found - _log("DEBUG", f"S{season:02d}E{episode:02d}: No NFO file found - needs DB/API lookup") - episodes_needing_db_check.append((season, episode)) - - _log("INFO", f"NFO files with dateadded: {nfo_complete_count}/{len(disk_episodes)} episodes. Need DB check: {len(episodes_needing_db_check)}") - - # STEP 2: For episodes missing dateadded in NFO, check database - if episodes_needing_db_check: - _log("DEBUG", f"STEP 2 - Checking database for {len(episodes_needing_db_check)} episodes missing dateadded") - db_hits = 0 - - for (season, episode) in episodes_needing_db_check: - db_result = self.db.get_episode_date(imdb_id, season, episode) - - if db_result and db_result.get('dateadded'): - # Found in database - use cached data and will add to NFO later - aired = db_result.get('aired') - dateadded = db_result.get('dateadded') - source = db_result.get('source', 'database_cache') - episode_dates[(season, episode)] = (aired, dateadded, source) - db_hits += 1 - _log("DEBUG", f"S{season:02d}E{episode:02d}: Database has dateadded={dateadded} - will add to NFO") - else: - # Not in database or incomplete - needs API lookup - episodes_needing_api_lookup.append((season, episode)) - - _log("INFO", f"Database hits: {db_hits}/{len(episodes_needing_db_check)} episodes. Need API lookup: {len(episodes_needing_api_lookup)}") - - # STEP 3: For episodes still missing dateadded, query Sonarr + db_result = self.db.get_episode_date(imdb_id, season, episode) + + if db_result and db_result.get('dateadded'): + # Found in database - use cached data + aired = db_result.get('aired') + dateadded = db_result.get('dateadded') + source = db_result.get('source', 'database_cache') + episode_dates[(season, episode)] = (aired, dateadded, source) + db_hits += 1 + _log("DEBUG", f"S{season:02d}E{episode:02d}: Database has dateadded={dateadded}") + else: + # Not in database or incomplete - needs API lookup + episodes_needing_api_lookup.append((season, episode)) + + _log("INFO", f"Database hits: {db_hits}/{len(disk_episodes)} episodes. Need API lookup: {len(episodes_needing_api_lookup)}") + + # STEP 2: For episodes missing from database, query Sonarr if episodes_needing_api_lookup: - _log("DEBUG", f"STEP 3 - Querying Sonarr for {len(episodes_needing_api_lookup)} episodes missing from NFO and database") + _log("DEBUG", f"STEP 2 - Querying Sonarr for {len(episodes_needing_api_lookup)} episodes missing from database") sonarr_episodes = self._get_sonarr_episodes(imdb_id, episodes_needing_api_lookup) for (season, episode) in episodes_needing_api_lookup: @@ -503,7 +396,7 @@ class TVProcessor: episode_dates[(season, episode)] = (aired, dateadded, source) - _log("INFO", f"🔍 NFO-FIRST COMPLETE: {len(episode_dates)} episodes processed") + _log("INFO", f"🔍 INCOMPLETE MODE COMPLETE: {len(episode_dates)} episodes processed") for (s, e), (aired, dateadded, source) in episode_dates.items(): _log("INFO", f" S{s:02d}E{e:02d}: aired={aired}, dateadded={dateadded}, source={source}") @@ -648,18 +541,8 @@ class TVProcessor: processed_count = 0 for (season, episode), (aired, dateadded, source) in episode_dates.items(): if (season, episode) in season_episodes: - # Create NFO - if config.manage_nfo: - self.nfo_manager.create_episode_nfo( - season_path, - season, episode, aired, dateadded, source, config.lock_metadata - ) - - # Update file mtimes - if config.fix_dir_mtimes and dateadded: - video_files = season_episodes[(season, episode)] - for video_file in video_files: - self.nfo_manager.set_file_mtime(video_file, dateadded) + # NFO file operations removed - database is now the single source of truth + # (Phase 1: Remove NFO file write operations) # Save to database try: @@ -728,16 +611,8 @@ class TVProcessor: aired, dateadded, source = episode_dates[(season_num, episode_num)] - # Create NFO - if config.manage_nfo: - self.nfo_manager.create_episode_nfo( - season_path, - season_num, episode_num, aired, dateadded, source, config.lock_metadata - ) - - # Update file mtime - if config.fix_dir_mtimes and dateadded: - self.nfo_manager.set_file_mtime(episode_path, dateadded) + # NFO file operations removed - database is now the single source of truth + # (Phase 1: Remove NFO file write operations) # Save to database self.db.upsert_episode_date(imdb_id, season_num, episode_num, aired, dateadded, source, True) @@ -765,7 +640,7 @@ class TVProcessor: Returns: Dictionary with processing results """ - imdb_id = self.nfo_manager.parse_imdb_from_path(series_path) + imdb_id = parse_imdb_from_path(series_path) # Phase 3: Using imdb_utils if not imdb_id: return {"status": "error", "reason": f"No IMDb ID found in series path: {series_path}"} @@ -787,25 +662,8 @@ class TVProcessor: for (season, episode), (aired, dateadded, source) in episode_dates.items(): if (season, episode) in disk_episodes: - season_dir = series_path / config.get_season_dir_name(season) - - # Prepare NFO creation data - if config.manage_nfo: - episode_data_list.append({ - 'season_dir': season_dir, - 'season': season, - 'episode': episode, - 'aired': aired, - 'dateadded': dateadded, - 'source': source, - 'lock_metadata': config.lock_metadata - }) - - # Prepare mtime operations - if config.fix_dir_mtimes and dateadded: - video_files = disk_episodes[(season, episode)] - for video_file in video_files: - mtime_operations.append((video_file, dateadded)) + # NFO file operations removed - database is now the single source of truth + # (Phase 1: Remove NFO file write operations) # Save to database try: @@ -815,27 +673,10 @@ class TVProcessor: _log("ERROR", f"S{season:02d}E{episode:02d}: Database write failed: {e}") # Continue processing other episodes - # Process NFOs and mtimes concurrently + # NFO and mtime operations removed - database is now the single source of truth + # (Phase 1: Remove NFO file write operations) results = {} - if episode_data_list: - _log("INFO", f"Creating {len(episode_data_list)} episode NFOs concurrently") - nfo_results = await self.async_nfo_manager.async_batch_create_episode_nfos( - episode_data_list, - max_concurrent=config.max_concurrent - ) - results['nfo_created'] = sum(nfo_results) - results['nfo_failed'] = len(nfo_results) - sum(nfo_results) - - if mtime_operations: - _log("INFO", f"Setting mtimes for {len(mtime_operations)} files concurrently") - mtime_results = await self.async_nfo_manager.async_batch_set_file_mtimes( - mtime_operations, - max_concurrent=10 - ) - results['mtime_updated'] = sum(mtime_results) - results['mtime_failed'] = len(mtime_results) - sum(mtime_results) - _log("INFO", f"Completed async processing TV series: {series_path.name}") return { @@ -892,7 +733,7 @@ class TVProcessor: def process_webhook_episodes(self, series_path: Path, webhook_episodes: List[Dict[str, Any]]) -> None: """Process only the specific episodes mentioned in a webhook (targeted mode)""" - imdb_id = self.nfo_manager.parse_imdb_from_path(series_path) + imdb_id = parse_imdb_from_path(series_path) # Phase 3: Using imdb_utils if not imdb_id: _log("ERROR", f"No IMDb ID found in series path: {series_path}") return @@ -942,18 +783,8 @@ class TVProcessor: aired, dateadded, source = self._get_webhook_episode_date(imdb_id, season_num, episode_num, series_metadata, webhook_episode) enhanced_metadata = self._get_episode_metadata(series_metadata, season_num, episode_num, season_dir) if series_metadata else self._get_episode_metadata(None, season_num, episode_num, season_dir) - # Create NFO - if config.manage_nfo: - self.nfo_manager.create_episode_nfo( - season_dir, - season_num, episode_num, aired, dateadded, source, config.lock_metadata, - enhanced_metadata - ) - - # Update file mtimes - if config.fix_dir_mtimes and dateadded: - for episode_file in episode_files: - self.nfo_manager.set_file_mtime(episode_file, dateadded) + # NFO file operations removed - database is now the single source of truth + # (Phase 1: Remove NFO file write operations) # Save to database self.db.upsert_episode_date(imdb_id, season_num, episode_num, aired, dateadded, source, True) @@ -1208,17 +1039,13 @@ class TVProcessor: dateadded = episode_data.get('dateadded') # Get IMDb ID - imdb_id = self.nfo_manager.parse_imdb_from_path(series_path) + imdb_id = parse_imdb_from_path(series_path) # Phase 3: Using imdb_utils if not imdb_id: return {"status": "error", "reason": "No IMDb ID found"} - # Create NFO if needed - nfo_success = True - if config.manage_nfo: - season_dir = series_path / config.get_season_dir_name(season) - nfo_success = await self.async_nfo_manager.async_create_episode_nfo( - season_dir, season, episode, aired, dateadded, "webhook", config.lock_metadata - ) + # NFO file operations removed - database is now the single source of truth + # (Phase 1: Remove NFO file write operations) + nfo_success = True # Always True since we're not creating NFOs anymore # Update database self.db.upsert_episode_date(imdb_id, season, episode, aired, dateadded, "webhook", True) diff --git a/static/index.html b/static/index.html index 45f2b3a..a48e886 100644 --- a/static/index.html +++ b/static/index.html @@ -382,6 +382,39 @@ Refresh Stats + +
+

Populate Database

+

Bulk import data from Radarr/Sonarr into NFOGuard database

+
+
+ + +
+ +
+ +
diff --git a/static/js/app.js b/static/js/app.js index d6e8e5c..a9ddb73 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -77,6 +77,7 @@ function initializeEventListeners() { document.getElementById('edit-form').addEventListener('submit', handleEditSubmit); document.getElementById('bulk-update-form').addEventListener('submit', handleBulkUpdate); document.getElementById('manual-scan-form').addEventListener('submit', handleManualScan); + document.getElementById('populate-form').addEventListener('submit', handlePopulateDatabase); } // API calls @@ -1742,3 +1743,186 @@ async function bulkDeleteSelected() { showToast(`❌ Failed to delete ${failCount} episode(s)`, 'error'); } } + +// --------------------------- +// Database Population Functions +// --------------------------- + +async function handlePopulateDatabase(event) { + event.preventDefault(); + + const mediaType = document.getElementById('populate-media-type').value; + + // Validate input + if (!mediaType) { + showToast('❌ Please select media type', 'error'); + return; + } + + // Confirm with user + const confirmMsg = `Are you sure you want to populate the database with ${mediaType}? This will query Radarr/Sonarr and may take several minutes.`; + if (!confirm(confirmMsg)) { + return; + } + + try { + // Show populate status + showPopulateStatus(); + + // Start the population + showToast('🚀 Starting database population...', 'info'); + const response = await fetch(`/admin/populate-database?media_type=${mediaType}`, { + method: 'POST', + credentials: 'include' + }); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + + const result = await response.json(); + + if (result.status === 'started') { + showToast('✅ Population started successfully', 'success'); + // Start polling for status + startPopulatePolling(); + } else { + showToast(`ℹī¸ ${result.message || 'Population completed'}`, 'info'); + hidePopulateStatus(); + } + + } catch (error) { + console.error('Database population failed:', error); + showToast(`❌ Population failed: ${error.message}`, 'error'); + hidePopulateStatus(); + } +} + +function showPopulateStatus() { + const populateStatus = document.getElementById('populate-status'); + const progressBar = document.getElementById('populate-progress-bar'); + const operationText = document.getElementById('populate-current-operation'); + const progressText = document.getElementById('populate-progress-text'); + const resultsDiv = document.getElementById('populate-results'); + + populateStatus.style.display = 'block'; + progressBar.style.width = '0%'; + operationText.textContent = 'Initializing...'; + progressText.textContent = '0%'; + resultsDiv.innerHTML = ''; +} + +function hidePopulateStatus() { + const populateStatus = document.getElementById('populate-status'); + if (populateStatus) { + populateStatus.style.display = 'none'; + } +} + +function startPopulatePolling() { + // Poll every 2 seconds for populate status + window.populatePollingInterval = setInterval(async () => { + try { + const response = await fetch('/api/populate/status'); + if (!response.ok) { + throw new Error('Failed to get populate status'); + } + + const status = await response.json(); + updatePopulateProgress(status); + + // Stop polling if population is complete + if (!status.running && status.completed) { + stopPopulatePolling(); + showToast('✅ Database population completed!', 'success'); + } + + } catch (error) { + console.error('Failed to poll populate status:', error); + // Don't show error toast repeatedly, just stop polling + stopPopulatePolling(); + } + }, 2000); +} + +function stopPopulatePolling() { + if (window.populatePollingInterval) { + clearInterval(window.populatePollingInterval); + window.populatePollingInterval = null; + } +} + +function updatePopulateProgress(status) { + const progressBar = document.getElementById('populate-progress-bar'); + const operationText = document.getElementById('populate-current-operation'); + const progressText = document.getElementById('populate-progress-text'); + const resultsDiv = document.getElementById('populate-results'); + + if (status.error) { + progressBar.style.width = '100%'; + progressBar.style.backgroundColor = '#e74c3c'; + operationText.textContent = 'Error occurred'; + progressText.textContent = 'Failed'; + resultsDiv.innerHTML = `

Error: ${status.error}

`; + return; + } + + if (!status.running && status.completed) { + progressBar.style.width = '100%'; + operationText.textContent = 'Population completed'; + progressText.textContent = '100%'; + + // Display results + let resultsHTML = '

Population Results:

'; + + if (status.movies && status.movies.stats) { + const m = status.movies.stats; + resultsHTML += ` +
+ Movies:
+ Total: ${m.total || 0} | Added: ${m.added || 0} | Skipped: ${m.skipped || 0} | Errors: ${m.errors || 0}
+ Duration: ${m.duration ? m.duration.toFixed(2) : 0}s +
+ `; + } + + if (status.tv && status.tv.stats) { + const t = status.tv.stats; + resultsHTML += ` +
+ TV Shows:
+ Series: ${t.total_series || 0} | Episodes: ${t.total_episodes || 0}
+ Added: ${t.added || 0} | Skipped: ${t.skipped || 0} | Errors: ${t.errors || 0}
+ Duration: ${t.duration ? t.duration.toFixed(2) : 0}s +
+ `; + } + + resultsDiv.innerHTML = resultsHTML; + return; + } + + // Update progress based on status + let totalProgress = 0; + let progressDetails = ''; + + if (status.movies && status.movies.status === 'running') { + totalProgress = 50; + progressDetails = 'Processing movies...'; + } else if (status.movies && status.movies.status === 'completed') { + totalProgress = 50; + progressDetails = 'Movies completed, processing TV...'; + } + + if (status.tv && status.tv.status === 'running') { + totalProgress = 75; + progressDetails = 'Processing TV shows...'; + } else if (status.tv && status.tv.status === 'completed') { + totalProgress = 100; + progressDetails = 'Completed'; + } + + progressBar.style.width = `${totalProgress}%`; + operationText.textContent = progressDetails; + progressText.textContent = `${totalProgress}%`; +} diff --git a/utils/imdb_utils.py b/utils/imdb_utils.py new file mode 100644 index 0000000..092deaa --- /dev/null +++ b/utils/imdb_utils.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +""" +IMDb ID Extraction Utilities +Parses IMDb IDs from directory/file names (no file I/O) +Phase 3: Replaces NFOManager for IMDb ID extraction only +""" +import re +from pathlib import Path +from typing import Optional + + +def parse_imdb_from_path(path: Path) -> Optional[str]: + """ + Extract IMDb ID from directory path or filename using regex patterns. + Does NOT read any files - only parses the path string. + + Supported patterns: + - [imdb-tt1234567] + - [tt1234567] + - {imdb-tt1234567} + - (imdb-tt1234567) + - -tt1234567 (at end) + - _tt1234567 (at end) + + Args: + path: Path object to parse + + Returns: + IMDb ID (e.g., "tt1234567") or None if not found + """ + path_str = str(path).lower() + + # Try [imdb-ttXXXXXXX] format first (most explicit) + match = re.search(r'\[imdb-?(tt\d+)\]', path_str) + if match: + return match.group(1) + + # Try standalone [ttXXXXXXX] format in brackets + match = re.search(r'\[(tt\d+)\]', path_str) + if match: + return match.group(1) + + # Try {imdb-ttXXXXXXX} format with curly braces + match = re.search(r'\{imdb-?(tt\d+)\}', path_str) + if match: + return match.group(1) + + # Try (imdb-ttXXXXXXX) format with parentheses + match = re.search(r'\(imdb-?(tt\d+)\)', path_str) + if match: + return match.group(1) + + # Try ttXXXXXXX at end of filename/dirname (common pattern) + match = re.search(r'[-_\s](tt\d+)$', path_str) + if match: + return match.group(1) + + return None + + +def find_imdb_in_directory(directory: Path) -> Optional[str]: + """ + Find IMDb ID from directory name or filenames within the directory. + Does NOT read file contents - only checks filenames. + + Args: + directory: Directory path to search + + Returns: + IMDb ID or None if not found + """ + # First try directory name itself + imdb_id = parse_imdb_from_path(directory) + if imdb_id: + return imdb_id + + # Try all filenames in the directory + if directory.is_dir(): + for file_path in directory.iterdir(): + if file_path.is_file(): + imdb_id = parse_imdb_from_path(file_path) + if imdb_id: + return imdb_id + + return None diff --git a/webhooks/webhook_batcher.py b/webhooks/webhook_batcher.py index d7b97d5..6caca77 100644 --- a/webhooks/webhook_batcher.py +++ b/webhooks/webhook_batcher.py @@ -10,12 +10,14 @@ from concurrent.futures import ThreadPoolExecutor from config.settings import config from utils.logging import _log +from utils.imdb_utils import find_imdb_in_directory, parse_imdb_from_path # Phase 3: Replaced NFOManager class WebhookBatcher: """Batches webhook events to avoid processing storms""" - + def __init__(self, nfo_manager=None): + # nfo_manager parameter kept for backward compatibility but no longer used (Phase 3) self.pending: Dict[str, Dict] = {} self.timers: Dict[str, threading.Timer] = {} self.processing: Set[str] = set() @@ -24,8 +26,6 @@ class WebhookBatcher: # Will be set by the application when processors are available self.tv_processor = None self.movie_processor = None - # NFO manager for comprehensive IMDb detection - self.nfo_manager = nfo_manager def set_processors(self, tv_processor, movie_processor): """Set the processor instances""" @@ -84,56 +84,40 @@ class WebhookBatcher: # CRITICAL: Validate that the path contains the expected IMDb ID for movies if media_type == 'movie': expected_imdb = key.replace('movie:', '') if key.startswith('movie:') else key - - # Use comprehensive IMDb detection (directory, filenames, NFO files) - if self.nfo_manager: - detected_imdb = self.nfo_manager.find_movie_imdb_id(path_obj) - imdb_match = False - if detected_imdb: - # Compare with and without 'tt' prefix for flexibility - if detected_imdb == expected_imdb or detected_imdb.replace('tt', '') == expected_imdb.replace('tt', ''): - imdb_match = True - - if not imdb_match: - _log("ERROR", f"BATCH VALIDATION FAILED: Expected IMDb {expected_imdb} not found via comprehensive detection in path {path_str}") - _log("ERROR", f"Detected IMDb: {detected_imdb}, Expected: {expected_imdb}") - _log("ERROR", f"This prevents processing wrong movies due to batch corruption") - return - _log("DEBUG", f"Batch validation passed: IMDb {expected_imdb} matches detected {detected_imdb}") - else: - # Fallback to simple string search if nfo_manager not available - if expected_imdb not in path_str.lower(): - _log("ERROR", f"BATCH VALIDATION FAILED: Expected IMDb {expected_imdb} not found in path {path_str} (fallback mode)") - _log("ERROR", f"This prevents processing wrong movies due to batch corruption") - return - _log("DEBUG", f"Batch validation passed: IMDb {expected_imdb} found in path (fallback mode)") + + # Use imdb_utils for IMDb detection (Phase 3: no NFO reading) + detected_imdb = find_imdb_in_directory(path_obj) + imdb_match = False + if detected_imdb: + # Compare with and without 'tt' prefix for flexibility + if detected_imdb == expected_imdb or detected_imdb.replace('tt', '') == expected_imdb.replace('tt', ''): + imdb_match = True + + if not imdb_match: + _log("ERROR", f"BATCH VALIDATION FAILED: Expected IMDb {expected_imdb} but found {detected_imdb} in {path_str}") + _log("ERROR", f"Detected IMDb: {detected_imdb}, Expected: {expected_imdb}") + _log("ERROR", f"This prevents processing wrong movies due to batch corruption") + return + _log("DEBUG", f"Batch validation passed: IMDb {expected_imdb} matches detected {detected_imdb}") # CRITICAL: Validate that the path contains the expected IMDb ID for TV shows if media_type == 'tv': expected_imdb = key.replace('tv:', '') if key.startswith('tv:') else key - - # Use comprehensive IMDb detection (directory, filenames, tvshow.nfo files) - if self.nfo_manager: - detected_imdb = self.nfo_manager.find_series_imdb_id(path_obj) - imdb_match = False - if detected_imdb: - # Compare with and without 'tt' prefix for flexibility - if detected_imdb == expected_imdb or detected_imdb.replace('tt', '') == expected_imdb.replace('tt', ''): - imdb_match = True - - if not imdb_match: - _log("ERROR", f"BATCH VALIDATION FAILED: Expected IMDb {expected_imdb} not found via comprehensive detection in TV path {path_str}") - _log("ERROR", f"Detected TV IMDb: {detected_imdb}, Expected: {expected_imdb}") - _log("ERROR", f"This prevents processing wrong TV series due to batch corruption") - return - _log("DEBUG", f"TV batch validation passed: IMDb {expected_imdb} matches detected {detected_imdb}") - else: - # Fallback to simple string search if nfo_manager not available - if expected_imdb not in path_str.lower(): - _log("ERROR", f"BATCH VALIDATION FAILED: Expected IMDb {expected_imdb} not found in TV path {path_str} (fallback mode)") - _log("ERROR", f"This prevents processing wrong TV series due to batch corruption") - return - _log("DEBUG", f"TV batch validation passed: IMDb {expected_imdb} found in path (fallback mode)") + + # Use imdb_utils for IMDb detection (Phase 3: no NFO reading) + detected_imdb = parse_imdb_from_path(path_obj) + imdb_match = False + if detected_imdb: + # Compare with and without 'tt' prefix for flexibility + if detected_imdb == expected_imdb or detected_imdb.replace('tt', '') == expected_imdb.replace('tt', ''): + imdb_match = True + + if not imdb_match: + _log("ERROR", f"BATCH VALIDATION FAILED: Expected IMDb {expected_imdb} but found {detected_imdb} in TV {path_str}") + _log("ERROR", f"Detected TV IMDb: {detected_imdb}, Expected: {expected_imdb}") + _log("ERROR", f"This prevents processing wrong TV series due to batch corruption") + return + _log("DEBUG", f"TV batch validation passed: IMDb {expected_imdb} matches detected {detected_imdb}") if not self.tv_processor: _log("ERROR", "TV processor not available")