sbcrumb/nfoguard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: update and remove anything to do with NFO files
Local Docker Build (Dev) / build-dev (push) Successful in 35s
Details

Browse Source 
moving to an all DB approach since radarr overwrites .nfo files
This commit is contained in:
SBCrumb
2025-11-02 13:43:28 -05:00
parent 041caf0d0d
commit 97a9f3431a
13 changed files with 837 additions and 2252 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+12 -8
View File
@@ -97,18 +97,22 @@ ALLOW_FILE_DATE_FALLBACK=false
TMDB_COUNTRY=US
# ===========================================
# NFO FILE MANAGEMENT
# NFO FILE MANAGEMENT (DEPRECATED - Phase 1 Migration)
# ===========================================
# Create/update .nfo files
MANAGE_NFO=true
# NFO file operations have been removed in favor of database-only architecture
# These settings are preserved for backward compatibility but no longer have effect
# The PostgreSQL database is now the single source of truth for all metadata
# Update file modification times to match import dates
FIX_DIR_MTIMES=true
# Create/update .nfo files (DEPRECATED - no longer used)
MANAGE_NFO=false
# Add lockdata tags to prevent metadata overwrites
LOCK_METADATA=true
# Update file modification times to match import dates (DEPRECATED - no longer used)
FIX_DIR_MTIMES=false
# Brand name in NFO comments
# Add lockdata tags to prevent metadata overwrites (DEPRECATED - no longer used)
LOCK_METADATA=false
# Brand name in NFO comments (DEPRECATED - no longer used)
MANAGER_BRAND=NFOGuard
# ===========================================
api/routes.py
+117
View File
@@ -2538,6 +2538,115 @@ async def lookup_movie(imdb_id: str, dependencies: dict):
raise HTTPException(status_code=500, detail=f"Movie lookup failed: {str(e)}")
async def populate_database(background_tasks: BackgroundTasks, media_type: str = "both", dependencies: dict = None):
"""
Populate NFOGuard database from Radarr/Sonarr sources
Args:
background_tasks: FastAPI background tasks
media_type: Type of media to populate ("movies", "tv", or "both")
dependencies: Dictionary with db, radarr_client, sonarr_client
Returns:
Status message indicating population has started
"""
from core.database_populator import DatabasePopulator
db = dependencies["db"]
config = dependencies["config"]
# Get Radarr and Sonarr clients
from clients.radarr_client import RadarrClient
from clients.sonarr_client import SonarrClient
radarr_client = RadarrClient(config)
sonarr_client = SonarrClient(config)
if media_type not in ["both", "movies", "tv"]:
raise HTTPException(status_code=400, detail="media_type must be 'both', 'movies', or 'tv'")
# Create global status tracking
populate_status = {
"running": True,
"media_type": media_type,
"start_time": datetime.now().isoformat(),
"movies": {"status": "pending", "stats": None},
"tv": {"status": "pending", "stats": None},
"completed": False,
"error": None
}
# Store status globally so it can be queried
global _populate_status
_populate_status = populate_status
async def run_population():
"""Background task to populate the database"""
try:
populator = DatabasePopulator(db, radarr_client, sonarr_client)
_log("INFO", f"Starting database population: {media_type}")
if media_type == "movies":
populate_status["movies"]["status"] = "running"
movie_stats = populator.populate_movies()
populate_status["movies"]["status"] = "completed"
populate_status["movies"]["stats"] = movie_stats
_log("INFO", f"Movie population completed: {movie_stats}")
elif media_type == "tv":
populate_status["tv"]["status"] = "running"
tv_stats = populator.populate_tv_episodes()
populate_status["tv"]["status"] = "completed"
populate_status["tv"]["stats"] = tv_stats
_log("INFO", f"TV population completed: {tv_stats}")
elif media_type == "both":
populate_status["movies"]["status"] = "running"
movie_stats = populator.populate_movies()
populate_status["movies"]["status"] = "completed"
populate_status["movies"]["stats"] = movie_stats
_log("INFO", f"Movie population completed: {movie_stats}")
populate_status["tv"]["status"] = "running"
tv_stats = populator.populate_tv_episodes()
populate_status["tv"]["status"] = "completed"
populate_status["tv"]["stats"] = tv_stats
_log("INFO", f"TV population completed: {tv_stats}")
populate_status["completed"] = True
populate_status["running"] = False
_log("INFO", "Database population completed successfully")
except Exception as e:
_log("ERROR", f"Database population failed: {e}")
populate_status["error"] = str(e)
populate_status["running"] = False
populate_status["completed"] = True
# Add task to background
background_tasks.add_task(run_population)
_log("INFO", f"Database population started for: {media_type}")
return {
"status": "started",
"media_type": media_type,
"message": f"Database population started for {media_type}"
}
async def get_populate_status():
"""Get the current status of database population"""
global _populate_status
if '_populate_status' not in globals():
return {"running": False, "completed": False}
return _populate_status
# Initialize global populate status
_populate_status = {"running": False, "completed": False}
def register_routes(app, dependencies: dict):
"""
Register all routes with the FastAPI app
@@ -2641,6 +2750,14 @@ def register_routes(app, dependencies: dict):
async def _scan_status():
return await get_scan_status()
@app.post("/admin/populate-database")
async def _populate_database(background_tasks: BackgroundTasks, media_type: str = "both"):
return await populate_database(background_tasks, media_type, dependencies)
@app.get("/api/populate/status")
async def _populate_status():
return await get_populate_status()
@app.post("/tv/scan-season")
async def _scan_tv_season(background_tasks: BackgroundTasks, request: TVSeasonRequest):
return await scan_tv_season(background_tasks, request, dependencies)
core/async_nfo_manager.py
-473
View File
@@ -1,473 +0,0 @@
"""
Async NFO Manager for NFOGuard
High-performance async NFO file operations with concurrent processing
"""
import asyncio
from pathlib import Path
from typing import Optional, List, Dict, Any, Tuple
from datetime import datetime
import xml.etree.ElementTree as ET
from utils.logging import _log
from utils.async_file_utils import (
async_read_nfo_file,
async_write_nfo_file,
async_file_exists,
async_set_file_mtime,
async_batch_nfo_operations,
async_concurrent_episode_processing,
aiofiles
)
from utils.nfo_patterns import (
create_basic_nfo_structure,
extract_imdb_from_nfo_content,
extract_dates_from_nfo,
extract_imdb_id_from_text
)
from utils.validation import validate_date_string
from utils.file_utils import VIDEO_EXTENSIONS
class AsyncNFOManager:
"""Async NFO file manager with concurrent processing capabilities"""
def __init__(self, manager_brand: str = "NFOGuard", debug: bool = False):
self.manager_brand = manager_brand
self.debug = debug
async def _async_get_target_nfo_path(self, season_dir: Path, season: int, episode: int) -> Path:
"""
Get the target NFO path using video filename matching to prevent concatenation
Args:
season_dir: Path to season directory
season: Season number
episode: Episode number
Returns:
Path to target NFO file (video-matching preferred, generic fallback)
"""
try:
# Find video files in the season directory
video_files = []
if await async_file_exists(season_dir):
import re
try:
entries = await aiofiles.os.listdir(season_dir)
for entry in entries:
entry_path = season_dir / entry
if await aiofiles.os.path.isfile(entry_path):
if entry_path.suffix.lower() in VIDEO_EXTENSIONS:
# Parse episode info from filename
match = re.search(r'[Ss](\d{1,2})[Ee](\d{1,2})', entry)
if match:
s, e = int(match.group(1)), int(match.group(2))
if s == season and e == episode:
# Found matching video file - use its name for NFO
target_nfo = season_dir / f"{entry_path.stem}.nfo"
if self.debug:
_log("DEBUG", f"Video-matching NFO path: {target_nfo.name}")
return target_nfo
except Exception as e:
if self.debug:
_log("WARNING", f"Failed to scan season directory {season_dir}: {e}")
# Fallback to generic filename if no matching video found
target_nfo = season_dir / f"S{season:02d}E{episode:02d}.nfo"
if self.debug:
_log("WARNING", f"No video file found for S{season:02d}E{episode:02d}, using generic: {target_nfo.name}")
return target_nfo
except Exception as e:
_log("ERROR", f"Failed to determine NFO path for S{season:02d}E{episode:02d}: {e}")
# Emergency fallback
return season_dir / f"S{season:02d}E{episode:02d}.nfo"
async def async_parse_imdb_from_path(self, path: Path) -> Optional[str]:
"""
Async extract IMDb ID from directory path or filename
Args:
path: Path to examine
Returns:
IMDb ID if found, None otherwise
"""
# Use the sync version since it's just string processing
return extract_imdb_id_from_text(str(path))
async def async_parse_imdb_from_nfo(self, nfo_path: Path) -> Optional[str]:
"""
Async extract IMDb ID from NFO file content
Args:
nfo_path: Path to NFO file
Returns:
IMDb ID if found, None otherwise
"""
root = await async_read_nfo_file(nfo_path)
if root is None:
return None
return extract_imdb_from_nfo_content(root)
async def async_find_movie_imdb_id(self, movie_dir: Path) -> Optional[str]:
"""
Async find IMDb ID from directory name, filenames, or NFO file
Args:
movie_dir: Path to movie directory
Returns:
IMDb ID if found, None otherwise
"""
if self.debug:
_log("DEBUG", f"Async searching for IMDb ID in: {movie_dir.name}")
# First try directory name
imdb_id = await self.async_parse_imdb_from_path(movie_dir)
if imdb_id:
if self.debug:
_log("DEBUG", f"Found IMDb ID in directory name: {imdb_id}")
return imdb_id
# Try all files in the directory concurrently
if await async_file_exists(movie_dir):
try:
from utils.async_file_utils import aiofiles
entries = await aiofiles.os.listdir(movie_dir)
# Create tasks to check all files
async def check_file(filename: str) -> Optional[str]:
file_path = movie_dir / filename
if await aiofiles.os.path.isfile(file_path):
return await self.async_parse_imdb_from_path(file_path)
return None
# Check all files concurrently
results = await asyncio.gather(
*[check_file(filename) for filename in entries],
return_exceptions=True
)
# Find first valid IMDb ID
for result in results:
if isinstance(result, str) and result:
if self.debug:
_log("DEBUG", f"Found IMDb ID in filename: {result}")
return result
except Exception as e:
_log("WARNING", f"Failed to scan directory {movie_dir}: {e}")
# Finally, try NFO file content
nfo_path = movie_dir / "movie.nfo"
imdb_id = await self.async_parse_imdb_from_nfo(nfo_path)
if imdb_id:
if self.debug:
_log("DEBUG", f"Found IMDb ID in NFO file: {imdb_id}")
return imdb_id
if self.debug:
_log("DEBUG", f"No IMDb ID found for: {movie_dir.name}")
return None
async def async_create_movie_nfo(
self,
movie_dir: Path,
imdb_id: str,
dateadded: str,
premiered: Optional[str] = None,
lock_metadata: bool = True
) -> bool:
"""
Async create movie NFO file
Args:
movie_dir: Path to movie directory
imdb_id: IMDb ID
dateadded: Date added
premiered: Optional premiere date
lock_metadata: Whether to lock metadata
Returns:
True if successful, False otherwise
"""
try:
nfo_path = movie_dir / "movie.nfo"
# Prepare dates
dates = {"dateadded": dateadded}
if premiered and validate_date_string(premiered):
dates["premiered"] = premiered
# Create NFO structure
root = create_basic_nfo_structure(
media_type="movie",
title=movie_dir.name,
imdb_id=imdb_id,
dates=dates,
additional_fields={"source": self.manager_brand}
)
# Write NFO file asynchronously
success = await async_write_nfo_file(nfo_path, root, lock_metadata)
if success and self.debug:
_log("DEBUG", f"Created movie NFO: {nfo_path}")
return success
except Exception as e:
_log("ERROR", f"Failed to create movie NFO for {movie_dir}: {e}")
return False
async def async_create_episode_nfo(
self,
season_dir: Path,
season: int,
episode: int,
aired: Optional[str] = None,
dateadded: Optional[str] = None,
source: str = "unknown",
lock_metadata: bool = True
) -> bool:
"""
Async create episode NFO file with proper video filename matching
Args:
season_dir: Path to season directory
season: Season number
episode: Episode number
aired: Optional air date
dateadded: Optional date added
source: Source of the data
lock_metadata: Whether to lock metadata
Returns:
True if successful, False otherwise
"""
try:
# Use proper video filename matching to prevent NFO concatenation
nfo_path = await self._async_get_target_nfo_path(season_dir, season, episode)
# Prepare dates
dates = {}
if aired and validate_date_string(aired):
dates["aired"] = aired
if dateadded and validate_date_string(dateadded):
dates["dateadded"] = dateadded
# Create NFO structure
root = create_basic_nfo_structure(
media_type="episodedetails",
title=f"S{season:02d}E{episode:02d}",
dates=dates,
additional_fields={
"season": str(season),
"episode": str(episode),
"source": source
}
)
# Write NFO file asynchronously
success = await async_write_nfo_file(nfo_path, root, lock_metadata)
if success and self.debug:
_log("DEBUG", f"Created episode NFO: {nfo_path}")
return success
except Exception as e:
_log("ERROR", f"Failed to create episode NFO S{season:02d}E{episode:02d}: {e}")
return False
async def async_batch_create_episode_nfos(
self,
episode_data_list: List[Dict[str, Any]],
max_concurrent: int = 5
) -> List[bool]:
"""
Batch create multiple episode NFOs concurrently
Args:
episode_data_list: List of episode data dictionaries
max_concurrent: Maximum concurrent NFO operations
Returns:
List of success/failure results
"""
async def _create_episode_nfo(episode_data: Dict[str, Any]) -> bool:
return await self.async_create_episode_nfo(
season_dir=episode_data.get('season_dir'),
season=episode_data.get('season'),
episode=episode_data.get('episode'),
aired=episode_data.get('aired'),
dateadded=episode_data.get('dateadded'),
source=episode_data.get('source', 'unknown'),
lock_metadata=episode_data.get('lock_metadata', True)
)
return await async_concurrent_episode_processing(
episode_data_list,
_create_episode_nfo,
max_concurrent
)
async def async_set_file_mtime(self, file_path: Path, date_str: str) -> bool:
"""
Async set file modification time from date string
Args:
file_path: Path to file
date_str: Date string in ISO format
Returns:
True if successful, False otherwise
"""
try:
if not validate_date_string(date_str):
_log("WARNING", f"Invalid date format for mtime: {date_str}")
return False
# Parse date string to timestamp
dt = datetime.fromisoformat(date_str.replace('Z', '+00:00'))
timestamp = dt.timestamp()
# Set file mtime asynchronously
success = await async_set_file_mtime(file_path, timestamp)
if success and self.debug:
_log("DEBUG", f"Set mtime for {file_path}: {date_str}")
return success
except Exception as e:
_log("ERROR", f"Failed to set mtime for {file_path}: {e}")
return False
async def async_batch_set_file_mtimes(
self,
file_mtime_pairs: List[Tuple[Path, str]],
max_concurrent: int = 10
) -> List[bool]:
"""
Batch set file modification times concurrently
Args:
file_mtime_pairs: List of (file_path, date_str) tuples
max_concurrent: Maximum concurrent operations
Returns:
List of success/failure results
"""
async def _set_single_mtime(file_path: Path, date_str: str) -> bool:
return await self.async_set_file_mtime(file_path, date_str)
# Create tasks for all mtime operations
semaphore = asyncio.Semaphore(max_concurrent)
async def _set_mtime_with_semaphore(file_path: Path, date_str: str) -> bool:
async with semaphore:
return await _set_single_mtime(file_path, date_str)
tasks = [
_set_mtime_with_semaphore(file_path, date_str)
for file_path, date_str in file_mtime_pairs
]
results = await asyncio.gather(*tasks, return_exceptions=True)
return [result if isinstance(result, bool) else False for result in results]
async def async_validate_nfo_integrity(
self,
nfo_paths: List[Path],
max_concurrent: int = 10
) -> Dict[str, Any]:
"""
Async validate integrity of multiple NFO files
Args:
nfo_paths: List of NFO file paths to validate
max_concurrent: Maximum concurrent validations
Returns:
Dictionary with validation results and statistics
"""
results = {
'total_files': len(nfo_paths),
'valid_files': 0,
'invalid_files': 0,
'missing_files': 0,
'validation_errors': [],
'file_results': {}
}
async def _validate_single_nfo(nfo_path: Path) -> Dict[str, Any]:
file_result = {
'path': str(nfo_path),
'exists': False,
'valid_xml': False,
'has_imdb_id': False,
'has_dates': False,
'error': None
}
try:
# Check if file exists
if not await async_file_exists(nfo_path):
file_result['error'] = 'File does not exist'
return file_result
file_result['exists'] = True
# Try to parse NFO
root = await async_read_nfo_file(nfo_path)
if root is None:
file_result['error'] = 'Failed to parse XML'
return file_result
file_result['valid_xml'] = True
# Check for IMDb ID
imdb_id = extract_imdb_from_nfo_content(root)
file_result['has_imdb_id'] = bool(imdb_id)
# Check for dates
dates = extract_dates_from_nfo(root)
file_result['has_dates'] = any(dates.values())
except Exception as e:
file_result['error'] = str(e)
return file_result
# Validate all files concurrently
semaphore = asyncio.Semaphore(max_concurrent)
async def _validate_with_semaphore(nfo_path: Path) -> Dict[str, Any]:
async with semaphore:
return await _validate_single_nfo(nfo_path)
tasks = [_validate_with_semaphore(nfo_path) for nfo_path in nfo_paths]
file_results = await asyncio.gather(*tasks, return_exceptions=True)
# Process results
for i, result in enumerate(file_results):
if isinstance(result, dict):
path_str = str(nfo_paths[i])
results['file_results'][path_str] = result
if not result['exists']:
results['missing_files'] += 1
elif result['valid_xml']:
results['valid_files'] += 1
else:
results['invalid_files'] += 1
if result.get('error'):
results['validation_errors'].append(f"{path_str}: {result['error']}")
return results
core/database_populator.py
+267
View File
@@ -0,0 +1,267 @@
#!/usr/bin/env python3
"""
Database Populator for NFOGuard
Bulk populates the NFOGuard database from Radarr/Sonarr
Phase 4: Replace NFO-based initial population with direct DB/API queries
"""
import time
from typing import Dict, List, Optional, Tuple
from datetime import datetime
from core.database import NFOGuardDatabase
from clients.radarr_client import RadarrClient
from clients.sonarr_client import SonarrClient
from utils.logging import _log
class DatabasePopulator:
"""Populates NFOGuard database from Radarr/Sonarr sources"""
def __init__(self, db: NFOGuardDatabase, radarr_client: RadarrClient, sonarr_client: SonarrClient):
self.db = db
self.radarr = radarr_client
self.sonarr = sonarr_client
def populate_movies(self) -> Dict[str, any]:
"""
Populate movies from Radarr database/API
Returns:
Dictionary with statistics: {
'total': int,
'added': int,
'updated': int,
'skipped': int,
'errors': int,
'duration': float
}
"""
_log("INFO", "Starting movie population from Radarr")
start_time = time.time()
stats = {
'total': 0,
'added': 0,
'updated': 0,
'skipped': 0,
'errors': 0,
'duration': 0.0
}
try:
# Get all movies from Radarr
movies = self.radarr.get_all_movies()
if not movies:
_log("WARNING", "No movies found in Radarr")
return stats
stats['total'] = len(movies)
_log("INFO", f"Found {stats['total']} movies in Radarr")
# Process each movie
for movie in movies:
try:
imdb_id = movie.get('imdbId')
if not imdb_id:
_log("DEBUG", f"Skipping movie without IMDb ID: {movie.get('title')}")
stats['skipped'] += 1
continue
# Check if movie already exists in database
existing = self.db.get_movie_dates(imdb_id)
if existing and existing.get('dateadded'):
_log("DEBUG", f"Movie {imdb_id} already in database, skipping")
stats['skipped'] += 1
continue
# Get movie path
path = movie.get('path', '')
# Get release date
released = None
if movie.get('digitalRelease'):
released = movie.get('digitalRelease')
source_type = 'radarr:digital'
elif movie.get('physicalRelease'):
released = movie.get('physicalRelease')
source_type = 'radarr:physical'
elif movie.get('inCinemas'):
released = movie.get('inCinemas')
source_type = 'radarr:theatrical'
else:
source_type = 'radarr:unknown'
# Get import date from Radarr history
import_date = self.radarr.get_movie_import_date(imdb_id)
if import_date:
dateadded = import_date
source = 'radarr:import_history'
elif released:
# Use release date as fallback
dateadded = released
source = f'{source_type}_fallback'
else:
_log("DEBUG", f"No date available for movie {imdb_id}, skipping")
stats['skipped'] += 1
continue
# Insert into database
self.db.upsert_movie_dates(imdb_id, released, dateadded, source, has_video_file=True)
stats['added'] += 1
_log("DEBUG", f"Added movie {imdb_id}: {movie.get('title')} (source: {source})")
except Exception as e:
_log("ERROR", f"Error processing movie {movie.get('title', 'unknown')}: {e}")
stats['errors'] += 1
continue
except Exception as e:
_log("ERROR", f"Error during movie population: {e}")
stats['errors'] += 1
stats['duration'] = time.time() - start_time
_log("INFO", f"Movie population complete: {stats['added']} added, {stats['skipped']} skipped, {stats['errors']} errors in {stats['duration']:.2f}s")
return stats
def populate_tv_episodes(self) -> Dict[str, any]:
"""
Populate TV episodes from Sonarr API
Returns:
Dictionary with statistics: {
'total_series': int,
'total_episodes': int,
'added': int,
'updated': int,
'skipped': int,
'errors': int,
'duration': float
}
"""
_log("INFO", "Starting TV episode population from Sonarr")
start_time = time.time()
stats = {
'total_series': 0,
'total_episodes': 0,
'added': 0,
'updated': 0,
'skipped': 0,
'errors': 0,
'duration': 0.0
}
try:
# Get all series from Sonarr
all_series = self.sonarr.get_all_series()
if not all_series:
_log("WARNING", "No series found in Sonarr")
return stats
stats['total_series'] = len(all_series)
_log("INFO", f"Found {stats['total_series']} series in Sonarr")
# Process each series
for series in all_series:
try:
imdb_id = series.get('imdbId')
if not imdb_id:
_log("DEBUG", f"Skipping series without IMDb ID: {series.get('title')}")
continue
series_id = series.get('id')
series_path = series.get('path', '')
series_title = series.get('title', 'Unknown')
# Update series record
self.db.upsert_series(imdb_id, series_path)
# Get all episodes for this series
episodes = self.sonarr.episodes_for_series(series_id)
if not episodes:
continue
_log("DEBUG", f"Processing {len(episodes)} episodes for {series_title}")
# Process each episode
for episode in episodes:
try:
season_num = episode.get('seasonNumber', 0)
episode_num = episode.get('episodeNumber', 0)
if season_num < 0 or episode_num <= 0:
continue
stats['total_episodes'] += 1
# Check if episode already exists
existing = self.db.get_episode_date(imdb_id, season_num, episode_num)
if existing and existing.get('dateadded'):
stats['skipped'] += 1
continue
# Get air date
aired = episode.get('airDate')
# Get import date from history
dateadded = None
episode_id = episode.get('id')
if episode_id and episode.get('hasFile'):
import_date = self.sonarr.get_episode_import_history(episode_id)
if import_date:
dateadded = import_date
source = 'sonarr:import_history'
# Fallback to air date if no import date
if not dateadded and aired:
dateadded = aired
source = 'sonarr:aired_fallback'
elif not dateadded:
# No date available
stats['skipped'] += 1
continue
# Insert into database
has_file = episode.get('hasFile', False)
self.db.upsert_episode_date(imdb_id, season_num, episode_num, aired, dateadded, source, has_file)
stats['added'] += 1
except Exception as e:
_log("ERROR", f"Error processing episode S{season_num:02d}E{episode_num:02d} of {series_title}: {e}")
stats['errors'] += 1
continue
except Exception as e:
_log("ERROR", f"Error processing series {series.get('title', 'unknown')}: {e}")
stats['errors'] += 1
continue
except Exception as e:
_log("ERROR", f"Error during TV episode population: {e}")
stats['errors'] += 1
stats['duration'] = time.time() - start_time
_log("INFO", f"TV episode population complete: {stats['added']} added, {stats['skipped']} skipped, {stats['errors']} errors in {stats['duration']:.2f}s")
return stats
def populate_all(self) -> Dict[str, any]:
"""
Populate both movies and TV episodes
Returns:
Combined statistics dictionary
"""
_log("INFO", "Starting full database population")
start_time = time.time()
movie_stats = self.populate_movies()
tv_stats = self.populate_tv_episodes()
combined_stats = {
'movies': movie_stats,
'tv': tv_stats,
'total_duration': time.time() - start_time
}
_log("INFO", f"Full database population complete in {combined_stats['total_duration']:.2f}s")
return combined_stats
core/episode_nfo_manager.py
-276
View File
@@ -1,276 +0,0 @@
#!/usr/bin/env python3
"""
Episode NFO Manager - Handles TV episode NFO creation with video filename matching
Core principle: NFO filenames should match video filenames
"""
import xml.etree.ElementTree as ET
from pathlib import Path
from typing import Optional, Dict, Any, List, Tuple
import re
from .logging import _log
class EpisodeNFOManager:
"""Manages episode NFO files with video filename matching"""
def __init__(self, manager_brand: str = "NFOGuard"):
self.manager_brand = manager_brand
def find_video_files_for_season(self, season_dir: Path) -> Dict[Tuple[int, int], List[Path]]:
"""Find all video files in season directory, grouped by (season, episode)"""
if not season_dir.exists():
_log("DEBUG", f"Season directory does not exist: {season_dir}")
return {}
video_extensions = [".mkv", ".mp4", ".avi", ".mov", ".m4v"]
episodes = {}
_log("DEBUG", f"Scanning video files in: {season_dir}")
for video_file in season_dir.iterdir():
_log("DEBUG", f"Checking file: {video_file.name} (is_file: {video_file.is_file()}, suffix: {video_file.suffix.lower()})")
if (video_file.is_file() and
video_file.suffix.lower() in video_extensions):
episode_info = self._parse_episode_from_filename(video_file.name)
_log("DEBUG", f"Episode parsing for '{video_file.name}': {episode_info}")
if episode_info:
season_num, episode_num = episode_info
key = (season_num, episode_num)
if key not in episodes:
episodes[key] = []
episodes[key].append(video_file)
_log("DEBUG", f"Added video file: S{season_num:02d}E{episode_num:02d} → {video_file.name}")
_log("DEBUG", f"Total video files found: {len(episodes)} episodes")
return episodes
def _parse_episode_from_filename(self, filename: str) -> Optional[Tuple[int, int]]:
"""Extract season and episode numbers from filename"""
# Try S##E## format first (most common)
match = re.search(r'[Ss](\d{1,2})[Ee](\d{1,2})', filename)
if match:
return int(match.group(1)), int(match.group(2))
# Try ##x## format
match = re.search(r'(\d{1,2})x(\d{1,2})', filename)
if match:
return int(match.group(1)), int(match.group(2))
return None
def find_nfo_for_episode(self, season_dir: Path, season_num: int, episode_num: int) -> Optional[Path]:
"""Find existing NFO file for episode (prefer video-matching filename)"""
if not season_dir.exists():
return None
# First, look for NFO files that match video filenames
video_files = self.find_video_files_for_season(season_dir)
key = (season_num, episode_num)
if key in video_files:
for video_file in video_files[key]:
potential_nfo = season_dir / f"{video_file.stem}.nfo"
if potential_nfo.exists():
_log("DEBUG", f"Found video-matching NFO: {potential_nfo.name}")
return potential_nfo
# Fallback: look for short name NFO
short_nfo = season_dir / f"S{season_num:02d}E{episode_num:02d}.nfo"
if short_nfo.exists():
_log("DEBUG", f"Found short-name NFO: {short_nfo.name}")
return short_nfo
# Last resort: search all NFO files for matching season/episode data
for nfo_file in season_dir.glob("*.nfo"):
if self._nfo_matches_episode(nfo_file, season_num, episode_num):
_log("DEBUG", f"Found matching NFO by content: {nfo_file.name}")
return nfo_file
return None
def _nfo_matches_episode(self, nfo_path: Path, season_num: int, episode_num: int) -> bool:
"""Check if NFO file contains the specified season/episode"""
try:
tree = ET.parse(nfo_path)
root = tree.getroot()
if root.tag == "episodedetails":
season_elem = root.find("season")
episode_elem = root.find("episode")
if (season_elem is not None and episode_elem is not None and
season_elem.text and episode_elem.text):
try:
file_season = int(season_elem.text)
file_episode = int(episode_elem.text)
return file_season == season_num and file_episode == episode_num
except ValueError:
pass
except (ET.ParseError, Exception):
pass
return False
def get_target_nfo_path(self, season_dir: Path, season_num: int, episode_num: int) -> Path:
"""Get the target NFO path (prefer video filename, fallback to short name)"""
video_files = self.find_video_files_for_season(season_dir)
key = (season_num, episode_num)
if key in video_files and video_files[key]:
# Use the first video file found (handle multiple files gracefully)
video_file = video_files[key][0]
target_nfo = season_dir / f"{video_file.stem}.nfo"
_log("DEBUG", f"Target NFO will match video: {target_nfo.name}")
return target_nfo
else:
# Fallback to short name if no video file found
target_nfo = season_dir / f"S{season_num:02d}E{episode_num:02d}.nfo"
_log("WARNING", f"No video file found for S{season_num:02d}E{episode_num:02d}, using short name: {target_nfo.name}")
return target_nfo
def migrate_nfo_to_video_filename(self, season_dir: Path, season_num: int, episode_num: int) -> bool:
"""If short-name NFO exists, rename it to match video filename"""
existing_nfo = self.find_nfo_for_episode(season_dir, season_num, episode_num)
target_nfo = self.get_target_nfo_path(season_dir, season_num, episode_num)
# If we already have the right filename, nothing to do
if existing_nfo and existing_nfo == target_nfo:
return True
# If we have an NFO but it doesn't match target, rename it
if existing_nfo and existing_nfo != target_nfo:
try:
_log("INFO", f"Migrating NFO filename: {existing_nfo.name} -> {target_nfo.name}")
existing_nfo.rename(target_nfo)
return True
except Exception as e:
_log("ERROR", f"Failed to rename NFO: {e}")
return False
return False
def create_episode_nfo(self, season_dir: Path, season_num: int, episode_num: int,
aired: Optional[str], dateadded: Optional[str], source: str,
title: Optional[str] = None, plot: Optional[str] = None) -> bool:
"""Create or update episode NFO with video filename matching"""
# Get the target NFO path (matching video filename)
nfo_path = self.get_target_nfo_path(season_dir, season_num, episode_num)
# Migrate existing NFO if needed
self.migrate_nfo_to_video_filename(season_dir, season_num, episode_num)
try:
# Load existing NFO if it exists
episode_elem = None
if nfo_path.exists():
try:
tree = ET.parse(nfo_path)
episode_elem = tree.getroot()
if episode_elem.tag != "episodedetails":
raise ValueError("Root element is not <episodedetails>")
# Remove NFOGuard-managed fields (we'll re-add them)
for tag in ["season", "episode", "aired", "premiered", "dateadded", "lockdata"]:
existing = episode_elem.find(tag)
if existing is not None:
episode_elem.remove(existing)
_log("DEBUG", f"Loaded existing NFO content for {nfo_path.name}")
except (ET.ParseError, ValueError) as e:
_log("WARNING", f"Corrupted NFO file {nfo_path.name}: {e}. Creating new one.")
episode_elem = None
# Create new structure if needed
if episode_elem is None:
episode_elem = ET.Element("episodedetails")
# Add title if provided and not already present
if title and not episode_elem.find("title"):
title_elem = ET.SubElement(episode_elem, "title")
title_elem.text = title
# Add plot if provided and not already present
if plot and not episode_elem.find("plot"):
plot_elem = ET.SubElement(episode_elem, "plot")
plot_elem.text = plot
# Add NFOGuard fields at the end
season_elem = ET.SubElement(episode_elem, "season")
season_elem.text = str(season_num)
episode_num_elem = ET.SubElement(episode_elem, "episode")
episode_num_elem.text = str(episode_num)
if aired:
aired_elem = ET.SubElement(episode_elem, "aired")
aired_elem.text = aired[:10] if len(aired) >= 10 else aired
# Also add premiered for compatibility
premiered_elem = ET.SubElement(episode_elem, "premiered")
premiered_elem.text = aired[:10] if len(aired) >= 10 else aired
if dateadded:
dateadded_elem = ET.SubElement(episode_elem, "dateadded")
dateadded_elem.text = dateadded
# Add lockdata
lockdata_elem = ET.SubElement(episode_elem, "lockdata")
lockdata_elem.text = "true"
# Add comment with source at the bottom
comment = ET.Comment(f" Created by {self.manager_brand} - Source: {source} ")
episode_elem.append(comment)
# Write the NFO file
tree = ET.ElementTree(episode_elem)
ET.indent(tree, space=" ", level=0)
tree.write(nfo_path, encoding='utf-8', xml_declaration=True)
_log("INFO", f"✅ Created/updated episode NFO: {nfo_path.name}")
_log("INFO", f" S{season_num:02d}E{episode_num:02d}, Aired: {aired}, DateAdded: {dateadded}, Source: {source}")
return True
except Exception as e:
_log("ERROR", f"Failed to create episode NFO {nfo_path}: {e}")
return False
def extract_nfoguard_data(self, nfo_path: Path) -> Optional[Dict[str, str]]:
"""Extract NFOGuard-managed data from existing NFO"""
if not nfo_path.exists():
return None
try:
tree = ET.parse(nfo_path)
root = tree.getroot()
if root.tag != "episodedetails":
return None
# Look for NFOGuard fields
dateadded_elem = root.find("dateadded")
aired_elem = root.find("aired")
lockdata_elem = root.find("lockdata")
# Only consider it NFOGuard-managed if it has dateadded and lockdata
if (dateadded_elem is not None and dateadded_elem.text and
lockdata_elem is not None and lockdata_elem.text == "true"):
result = {
"dateadded": dateadded_elem.text.strip(),
"source": "existing_nfo"
}
if aired_elem is not None and aired_elem.text:
result["aired"] = aired_elem.text.strip()
_log("DEBUG", f"Found NFOGuard data in {nfo_path.name}: {result}")
return result
except (ET.ParseError, Exception) as e:
_log("WARNING", f"Error parsing NFO {nfo_path.name}: {e}")
return None
core/nfo_manager.py
-890
View File
@@ -1,890 +0,0 @@
#!/usr/bin/env python3
"""
NFO Manager for creating and managing metadata files
Handles NFO creation for movies, TV shows, seasons, and episodes
"""
import os
import xml.etree.ElementTree as ET
from pathlib import Path
from datetime import datetime
from typing import Optional, Dict, Any, Tuple
import re
class NFOManager:
"""Manages NFO file creation and updates"""
def __init__(self, manager_brand: str = "NFOGuard", debug: bool = False):
self.manager_brand = manager_brand
self.debug = debug
def parse_imdb_from_path(self, path: Path) -> Optional[str]:
"""Extract IMDb ID from directory path or filename"""
# Look for various IMDb patterns in both directory and file names
path_str = str(path).lower()
# Try [imdb-ttXXXXXXX] format first (most explicit)
match = re.search(r'\[imdb-?(tt\d+)\]', path_str)
if match:
return match.group(1)
# Try standalone [ttXXXXXXX] format in brackets
match = re.search(r'\[(tt\d+)\]', path_str)
if match:
return match.group(1)
# Try {imdb-ttXXXXXXX} format with curly braces
match = re.search(r'\{imdb-?(tt\d+)\}', path_str)
if match:
return match.group(1)
# Try (imdb-ttXXXXXXX) format with parentheses
match = re.search(r'\(imdb-?(tt\d+)\)', path_str)
if match:
return match.group(1)
# Try ttXXXXXXX at end of filename/dirname (common pattern)
match = re.search(r'[-_\s](tt\d+)$', path_str)
if match:
return match.group(1)
return None
def parse_imdb_from_path_with_nfo_fallback(self, path: Path, sonarr_client=None, shutdown_event=None) -> Optional[str]:
"""
Enhanced IMDb detection that fallback to Sonarr ID lookup from NFO files
1. First try to parse IMDb ID from directory path/name
2. If not found, scan NFO files in directory for Sonarr series ID
3. Use Sonarr API to lookup series and extract IMDb ID
"""
# Primary: Try directory name first
imdb_id = self.parse_imdb_from_path(path)
if imdb_id:
return imdb_id
# Check for shutdown signal before expensive operations
if shutdown_event and shutdown_event.is_set():
return None
# Fallback: Check NFO files for Sonarr series ID
if not sonarr_client or not sonarr_client.enabled:
return None
try:
# Look for episode NFO files in the directory (and subdirectories)
nfo_files = []
if path.is_dir():
# Check current directory and season subdirectories
nfo_files.extend(list(path.glob("*.nfo")))
for subdir in path.iterdir():
if subdir.is_dir() and subdir.name.lower().startswith('season'):
nfo_files.extend(list(subdir.glob("*.nfo")))
# Extract Sonarr series ID from any NFO file
for nfo_file in nfo_files:
# Check for shutdown signal during NFO processing
if shutdown_event and shutdown_event.is_set():
return None
try:
tree = ET.parse(nfo_file)
root = tree.getroot()
# Look for Sonarr series ID
for uniqueid in root.findall('.//uniqueid[@type="sonarr"]'):
sonarr_id = uniqueid.text
if sonarr_id and sonarr_id.isdigit():
# Check for shutdown signal before API call
if shutdown_event and shutdown_event.is_set():
return None
# Look up series in Sonarr to get IMDb ID
series_data = sonarr_client.get_series_by_id(int(sonarr_id))
if series_data:
imdb_id = series_data.get('imdbId')
if imdb_id:
return imdb_id.replace('tt', '') if imdb_id.startswith('tt') else imdb_id
except Exception as e:
continue # Skip this NFO file and try the next one
except Exception as e:
pass # Fallback failed, return None
return None
def parse_imdb_from_nfo(self, nfo_path: Path) -> Optional[str]:
"""Extract IMDb ID from NFO file content"""
if not nfo_path.exists():
return None
try:
root = self._parse_nfo_with_tolerance(nfo_path)
# Check for <uniqueid type="imdb">ttXXXXXX</uniqueid>
imdb_uniqueid = root.find('.//uniqueid[@type="imdb"]')
if imdb_uniqueid is not None and imdb_uniqueid.text:
imdb_id = imdb_uniqueid.text.strip()
if imdb_id.startswith('tt'):
return imdb_id
# Check for legacy <imdbid>ttXXXXXX</imdbid>
imdbid_elem = root.find('.//imdbid')
if imdbid_elem is not None and imdbid_elem.text:
imdb_id = imdbid_elem.text.strip()
if imdb_id.startswith('tt'):
return imdb_id
# Check for legacy <imdb>ttXXXXXX</imdb>
imdb_elem = root.find('.//imdb')
if imdb_elem is not None and imdb_elem.text:
imdb_id = imdb_elem.text.strip()
if imdb_id.startswith('tt'):
return imdb_id
# Last resort: Check for TMDB ID as fallback identifier
# This handles movies that only have TMDB IDs in NFO files
tmdb_uniqueid = root.find('.//uniqueid[@type="tmdb"]')
if tmdb_uniqueid is not None and tmdb_uniqueid.text:
tmdb_id = tmdb_uniqueid.text.strip()
if tmdb_id.isdigit():
print(f"⚠️ Found TMDB ID {tmdb_id} but no IMDb ID - using TMDB ID as fallback")
# Return TMDB ID with prefix to distinguish from IMDb IDs
return f"tmdb-{tmdb_id}"
except (ET.ParseError, Exception):
# Skip corrupted or non-XML files
pass
return None
def find_movie_imdb_id(self, movie_dir: Path) -> Optional[str]:
"""Find IMDb ID from directory name, filenames, or NFO file"""
# First try directory name
imdb_id = self.parse_imdb_from_path(movie_dir)
if imdb_id:
return imdb_id
# Try all files in the directory for IMDb ID patterns
for file_path in movie_dir.iterdir():
if file_path.is_file():
imdb_id = self.parse_imdb_from_path(file_path)
if imdb_id:
return imdb_id
# Finally, try NFO file content (including TMDB fallback)
nfo_path = movie_dir / "movie.nfo"
imdb_id = self.parse_imdb_from_nfo(nfo_path)
if imdb_id:
return imdb_id
return None
def find_series_imdb_id(self, series_dir: Path) -> Optional[str]:
"""Find IMDb ID from TV series directory name, filenames, or tvshow.nfo file"""
# First try directory name
imdb_id = self.parse_imdb_from_path(series_dir)
if imdb_id:
return imdb_id
# Try all files in the directory for IMDb ID patterns
for file_path in series_dir.iterdir():
if file_path.is_file():
imdb_id = self.parse_imdb_from_path(file_path)
if imdb_id:
return imdb_id
# Finally, try tvshow.nfo file content
nfo_path = series_dir / "tvshow.nfo"
imdb_id = self.parse_imdb_from_nfo(nfo_path)
if imdb_id:
return imdb_id
return None
def extract_nfoguard_dates_from_nfo(self, nfo_path: Path) -> Optional[Dict[str, str]]:
"""Extract NFOGuard-managed dates from existing NFO file"""
if not nfo_path.exists():
return None
try:
tree = ET.parse(nfo_path)
root = tree.getroot()
# Look for NFOGuard fields
dateadded_elem = root.find('.//dateadded')
premiered_elem = root.find('.//premiered')
aired_elem = root.find('.//aired') # For TV episodes
lockdata_elem = root.find('.//lockdata')
# Consider it NFOGuard-managed ONLY if it has BOTH lockdata=true AND dateadded
# This prevents incomplete NFO files from being marked as "complete"
if (lockdata_elem is not None and lockdata_elem.text == "true" and
dateadded_elem is not None and dateadded_elem.text):
# Extract original source from NFOGuard comment, default to nfo_file_existing
source = "nfo_file_existing"
# Parse XML content to find NFOGuard comment with source
nfo_content = nfo_path.read_text(encoding='utf-8')
import re
source_match = re.search(r'<!--\s*NFOGuard\s*-\s*Source:\s*([^-]+?)\s*-->', nfo_content)
if source_match:
source = source_match.group(1).strip()
print(f"🔍 Extracted original source from NFO comment: {source}")
result = {
"source": source
}
if dateadded_elem is not None and dateadded_elem.text:
result["dateadded"] = dateadded_elem.text.strip()
if premiered_elem is not None and premiered_elem.text:
result["released"] = premiered_elem.text.strip()
if aired_elem is not None and aired_elem.text:
result["aired"] = aired_elem.text.strip()
print(f"✅ Found NFOGuard data in NFO: dateadded={result.get('dateadded', 'None')}, source={source}, released={result.get('released', 'None')}, aired={result.get('aired', 'None')}")
return result
except ET.ParseError as e:
# Handle malformed XML files gracefully (common with external NFO files)
print(f"🔍 NFO XML parse error (will try text fallback): {nfo_path.name}")
# Try text-based fallback for extracting NFOGuard data
return self._extract_nfoguard_data_text_fallback(nfo_path)
except Exception as e:
print(f"⚠️ Unexpected error reading NFO file {nfo_path.name}: {e}")
pass
return None
def _extract_nfoguard_data_text_fallback(self, nfo_path: Path) -> Optional[Dict[str, str]]:
"""Text-based fallback for extracting NFOGuard data from malformed XML files"""
try:
content = nfo_path.read_text(encoding='utf-8', errors='ignore')
# Look for NFOGuard elements using regex
import re
# Check for lockdata=true first (NFOGuard marker)
if not re.search(r'<lockdata>true</lockdata>', content, re.IGNORECASE):
return None
# Extract dateadded
dateadded_match = re.search(r'<dateadded>([^<]+)</dateadded>', content, re.IGNORECASE)
if not dateadded_match:
return None
dateadded = dateadded_match.group(1).strip()
if not dateadded:
return None
result = {
"dateadded": dateadded,
"source": "nfo_file_existing" # Default source
}
# Extract optional fields
premiered_match = re.search(r'<premiered>([^<]+)</premiered>', content, re.IGNORECASE)
if premiered_match:
result["released"] = premiered_match.group(1).strip()
aired_match = re.search(r'<aired>([^<]+)</aired>', content, re.IGNORECASE)
if aired_match:
result["aired"] = aired_match.group(1).strip()
# Extract source from NFOGuard comment
source_match = re.search(r'<!--\s*NFOGuard\s*-\s*Source:\s*([^-]+?)\s*-->', content)
if source_match:
result["source"] = source_match.group(1).strip()
print(f"✅ Extracted NFOGuard data via text fallback: dateadded={result.get('dateadded', 'None')}, source={result['source']}")
return result
except Exception as e:
print(f"⚠️ Text fallback extraction failed for {nfo_path.name}: {e}")
return None
def extract_nfoguard_dates_from_episode_nfo(self, season_path: Path, season_num: int, episode_num: int) -> Optional[Dict[str, str]]:
"""Extract NFOGuard-managed dates from existing episode NFO file"""
nfo_filename = f"s{season_num:02d}e{episode_num:02d}.nfo"
nfo_path = season_path / nfo_filename
if not nfo_path.exists():
return None
try:
tree = ET.parse(nfo_path)
root = tree.getroot()
# Look for NFOGuard fields in episode NFO
dateadded_elem = root.find('.//dateadded')
aired_elem = root.find('.//aired')
lockdata_elem = root.find('.//lockdata')
# Consider it NFOGuard-managed ONLY if it has BOTH lockdata=true AND dateadded
# This prevents incomplete episode NFO files from being marked as "complete"
if (lockdata_elem is not None and lockdata_elem.text == "true" and
dateadded_elem is not None and dateadded_elem.text):
# Extract original source from NFOGuard comment, default to episode_nfo_existing
source = "episode_nfo_existing"
# Parse XML content to find NFOGuard comment with source
nfo_content = nfo_path.read_text(encoding='utf-8')
import re
source_match = re.search(r'<!--\s*NFOGuard\s*-\s*Source:\s*([^-]+?)\s*-->', nfo_content)
if source_match:
source = source_match.group(1).strip()
print(f"🔍 Extracted original source from episode NFO comment: {source}")
result = {
"source": source
}
if dateadded_elem is not None and dateadded_elem.text:
result["dateadded"] = dateadded_elem.text.strip()
if aired_elem is not None and aired_elem.text:
result["aired"] = aired_elem.text.strip()
print(f"✅ Found NFOGuard data in episode NFO S{season_num:02d}E{episode_num:02d}: dateadded={result.get('dateadded', 'None')}, source={source}, aired={result.get('aired', 'None')}")
return result
except (ET.ParseError, Exception) as e:
print(f"⚠️ Error parsing episode NFO for NFOGuard data: {e}")
pass
return None
def _parse_nfo_with_tolerance(self, nfo_path: Path):
"""Parse NFO file with tolerance for URLs appended after XML"""
try:
# First try normal parsing
tree = ET.parse(nfo_path)
return tree.getroot()
except ET.ParseError as e:
# If parsing fails, try to extract just the XML part
try:
with open(nfo_path, 'r', encoding='utf-8') as f:
content = f.read()
# Find the last </movie> tag and truncate after it
last_movie_end = content.rfind('</movie>')
if last_movie_end != -1:
xml_content = content[:last_movie_end + 8] # +8 for </movie>
# Try parsing the truncated content
root = ET.fromstring(xml_content)
print(f"✅ Successfully parsed NFO after removing trailing content: {nfo_path.name}")
return root
else:
# Re-raise original error if we can't find </movie>
raise e
except Exception:
# Re-raise original error if our fix attempt fails
raise e
def create_movie_nfo(self, movie_dir: Path, imdb_id: str, dateadded: str,
released: Optional[str] = None, source: str = "unknown",
lock_metadata: bool = True) -> None:
"""Create or update movie.nfo file preserving existing content"""
nfo_path = movie_dir / "movie.nfo"
# Debug output only if DEBUG=true in environment
import os
if os.environ.get("DEBUG", "false").lower() == "true":
print(f"🔍 create_movie_nfo called: imdb_id={imdb_id}, dateadded={dateadded}, released={released}, source={source}")
print(f"🔍 NFO path: {nfo_path}")
print(f"🔍 NFO exists: {nfo_path.exists()}")
try:
# Try to load existing NFO file
if nfo_path.exists():
try:
# Try to parse the XML, handling URLs appended after </movie>
movie = self._parse_nfo_with_tolerance(nfo_path)
# Ensure root element is <movie>
if movie.tag != "movie":
raise ValueError("Root element is not <movie>")
# Only remove elements that are clearly NFOGuard-managed
# Look for elements that have NFOGuard characteristics
elements_to_remove = []
# Remove lockdata=true (this is definitely NFOGuard)
for lockdata in movie.findall("lockdata"):
if lockdata.text == "true":
elements_to_remove.append(lockdata)
# Remove IMDb uniqueids that we manage
for uniqueid in movie.findall("uniqueid[@type='imdb']"):
elements_to_remove.append(uniqueid)
# For dateadded/premiered/year, only remove if they appear to be NFOGuard-managed
# (i.e., if lockdata=true exists, these are likely ours)
has_nfoguard_lockdata = any(ld.text == "true" for ld in movie.findall("lockdata"))
if has_nfoguard_lockdata:
# This NFO was managed by NFOGuard, safe to remove our fields
for tag in ["dateadded", "premiered", "year"]:
existing = movie.find(tag)
if existing is not None:
# Store the value before removing (for premiered/year)
if tag == "premiered" and not released:
print(f"🔍 Preserving existing premiered date: {existing.text}")
released = existing.text
elements_to_remove.append(existing)
else:
# No NFOGuard lockdata found, be more conservative
# Only remove dateadded if it looks like NFOGuard format (ISO timestamp)
dateadded_elem = movie.find("dateadded")
if dateadded_elem is not None and dateadded_elem.text:
# NFOGuard uses ISO format like "2025-10-12 16:26:02"
if re.match(r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}', dateadded_elem.text.strip()):
elements_to_remove.append(dateadded_elem)
# Remove all identified elements
for elem in elements_to_remove:
movie.remove(elem)
except (ET.ParseError, ValueError) as e:
print(f"⚠️ Corrupted NFO detected: {nfo_path} - {str(e)[:100]}...")
print(f" Creating new clean NFO file to replace corrupted one")
movie = ET.Element("movie")
else:
# Create new NFO structure
movie = ET.Element("movie")
# Create all NFOGuard elements first, then append them in correct order
# This ensures they appear as a group at the bottom of the file
nfoguard_elements = []
# Add IMDb uniqueid
uniqueid = ET.Element("uniqueid", type="imdb", default="true")
uniqueid.text = imdb_id
nfoguard_elements.append(uniqueid)
# Add premiered date if we have it
if released:
premiered_elem = ET.Element("premiered")
premiered_elem.text = released[:10] if len(released) >= 10 else released
nfoguard_elements.append(premiered_elem)
# Extract year from premiered date for consistency
try:
year_value = released[:4] if len(released) >= 4 else None
if year_value and year_value.isdigit():
year_elem = ET.Element("year")
year_elem.text = year_value
nfoguard_elements.append(year_elem)
except:
pass # Skip year if we can't extract it
# Add dateadded - THIS IS CRITICAL FOR EMBY PLUGIN
if os.environ.get("DEBUG", "false").lower() == "true":
print(f"🔍 About to add dateadded: {dateadded} (type: {type(dateadded)})")
if dateadded:
dateadded_elem = ET.Element("dateadded")
dateadded_elem.text = dateadded
nfoguard_elements.append(dateadded_elem)
print(f"✅ Adding dateadded to NFO: {dateadded}")
else:
print(f"❌ dateadded is empty/None, not adding to NFO")
# Add lockdata at the very end
if lock_metadata:
lockdata = ET.Element("lockdata")
lockdata.text = "true"
nfoguard_elements.append(lockdata)
# Add NFOGuard comment as the very last element (appears at bottom)
nfoguard_comment = ET.Comment(f" NFOGuard - Source: {source} ")
nfoguard_elements.append(nfoguard_comment)
# Now append all NFOGuard elements to the movie in one batch
# This ensures they appear as a contiguous block at the bottom
for elem in nfoguard_elements:
movie.append(elem)
print(f"✅ Added {len(nfoguard_elements)} NFOGuard elements to bottom of NFO")
# Write file with proper formatting
tree = ET.ElementTree(movie)
ET.indent(tree, space=" ", level=0)
# Write directly to file (comment is already embedded in XML at bottom)
with open(nfo_path, 'w', encoding='utf-8') as f:
f.write('<?xml version="1.0" encoding="utf-8"?>\n')
tree.write(f, encoding='unicode', xml_declaration=False)
print(f"✅ Successfully created/updated movie NFO: {nfo_path}")
print(f" IMDb ID: {imdb_id}, Date Added: {dateadded}, Source: {source}")
except Exception as e:
print(f"❌ Error creating/updating movie NFO {nfo_path}: {e}")
def create_tvshow_nfo(self, series_dir: Path, imdb_id: str, tvdb_id: Optional[str] = None) -> None:
"""Create or update tvshow.nfo file preserving existing content"""
nfo_path = series_dir / "tvshow.nfo"
try:
# Try to load existing NFO file
if nfo_path.exists():
try:
tree = ET.parse(nfo_path)
tvshow = tree.getroot()
# Ensure root element is <tvshow>
if tvshow.tag != "tvshow":
raise ValueError("Root element is not <tvshow>")
# Remove existing NFOGuard-managed elements to avoid duplicates
# These will be re-added at the bottom
for tag in ["lockdata"]:
existing = tvshow.find(tag)
if existing is not None:
tvshow.remove(existing)
# Remove ALL existing uniqueid with type="imdb" regardless of attributes
for uniqueid in tvshow.findall("uniqueid[@type='imdb']"):
tvshow.remove(uniqueid)
except (ET.ParseError, ValueError) as e:
print(f"⚠️ Corrupted TV show NFO detected: {nfo_path} - {str(e)[:100]}...")
print(f" Creating new clean tvshow.nfo file to replace corrupted one")
tvshow = ET.Element("tvshow")
else:
# Create new NFO structure
tvshow = ET.Element("tvshow")
# Add NFOGuard fields at the bottom
# Add IMDb uniqueid at the end
imdb_uniqueid = ET.SubElement(tvshow, "uniqueid", type="imdb", default="true")
imdb_uniqueid.text = imdb_id
# Add TVDB ID if available (preserve existing or add new)
if tvdb_id and not tvshow.find("uniqueid[@type='tvdb']"):
tvdb_uniqueid = ET.SubElement(tvshow, "uniqueid", type="tvdb")
tvdb_uniqueid.text = tvdb_id
# Add lockdata at the very end
lockdata = ET.SubElement(tvshow, "lockdata")
lockdata.text = "true"
# Add NFOGuard comment at the bottom
comment = ET.Comment(f" Created by {self.manager_brand} ")
tvshow.append(comment)
# Write file with proper formatting
tree = ET.ElementTree(tvshow)
ET.indent(tree, space=" ", level=0)
tree.write(nfo_path, encoding='utf-8', xml_declaration=True)
print(f"✅ Successfully created/updated TV show NFO: {nfo_path}")
print(f" IMDb ID: {imdb_id}" + (f", TVDB ID: {tvdb_id}" if tvdb_id else ""))
except Exception as e:
print(f"❌ Error creating/updating tvshow NFO {nfo_path}: {e}")
def create_season_nfo(self, season_dir: Path, season_number: int) -> None:
"""Create or update season.nfo file preserving existing content"""
nfo_path = season_dir / "season.nfo"
try:
season_dir.mkdir(exist_ok=True)
# Try to load existing NFO file
if nfo_path.exists():
try:
tree = ET.parse(nfo_path)
season = tree.getroot()
# Ensure root element is <season>
if season.tag != "season":
raise ValueError("Root element is not <season>")
# Remove existing NFOGuard-managed elements
for tag in ["seasonnumber", "lockdata"]:
existing = season.find(tag)
if existing is not None:
season.remove(existing)
except (ET.ParseError, ValueError) as e:
print(f"⚠️ Corrupted season NFO detected: {nfo_path} - {str(e)[:100]}...")
print(f" Creating new clean season.nfo file to replace corrupted one")
season = ET.Element("season")
else:
# Create new NFO structure
season = ET.Element("season")
# Add NFOGuard fields at the bottom
seasonnumber = ET.SubElement(season, "seasonnumber")
seasonnumber.text = str(season_number)
# Add lockdata at the end
lockdata = ET.SubElement(season, "lockdata")
lockdata.text = "true"
# Add NFOGuard comment at the bottom
comment = ET.Comment(f" Created by {self.manager_brand} ")
season.append(comment)
# Write file with proper formatting
tree = ET.ElementTree(season)
ET.indent(tree, space=" ", level=0)
tree.write(nfo_path, encoding='utf-8', xml_declaration=True)
print(f"✅ Successfully created/updated season NFO: {nfo_path}")
print(f" Season: {season_number}")
except Exception as e:
print(f"❌ Error creating/updating season NFO {nfo_path}: {e}")
def find_existing_episode_nfo(self, season_dir: Path, season_num: int, episode_num: int) -> Optional[Path]:
"""Find any existing episode NFO file that matches season/episode"""
if not season_dir.exists():
return None
# Look for NFO files in the season directory
for nfo_file in season_dir.glob("*.nfo"):
# Check if this NFO contains the right season/episode
try:
tree = ET.parse(nfo_file)
root = tree.getroot()
if root.tag == "episodedetails":
# Check for season/episode elements
season_elem = root.find("season")
episode_elem = root.find("episode")
if (season_elem is not None and episode_elem is not None and
season_elem.text and episode_elem.text):
try:
file_season = int(season_elem.text)
file_episode = int(episode_elem.text)
if file_season == season_num and file_episode == episode_num:
print(f"🔍 Found existing episode NFO: {nfo_file.name}")
return nfo_file
except ValueError:
continue
except (ET.ParseError, Exception):
# Skip corrupted or non-XML files
continue
return None
def _get_target_episode_nfo_name(self, season_dir: Path, season_num: int, episode_num: int) -> str:
"""Get target NFO filename - prefer existing NFO, then matching video file, fallback to short name"""
if not season_dir.exists():
return f"S{season_num:02d}E{episode_num:02d}.nfo"
# First check if an NFO already exists for this episode
existing_nfo = self.find_existing_episode_nfo(season_dir, season_num, episode_num)
if existing_nfo:
print(f"📂 Existing NFO found, will preserve filename: {existing_nfo.name}")
return existing_nfo.name
# Look for video files with matching season/episode
video_extensions = [".mkv", ".mp4", ".avi", ".mov", ".m4v", ".wmv", ".flv", ".webm"]
for video_file in season_dir.iterdir():
if (video_file.is_file() and
video_file.suffix.lower() in video_extensions):
# Parse episode info from video filename
episode_info = self._parse_episode_from_filename(video_file.name)
if episode_info and episode_info == (season_num, episode_num):
# Found matching video file - use its name for NFO
target_nfo_name = f"{video_file.stem}.nfo"
print(f"🎯 Target NFO will match video: {target_nfo_name}")
return target_nfo_name
# Fallback to short name if no matching video found
short_name = f"S{season_num:02d}E{episode_num:02d}.nfo"
print(f"⚠️ No matching video file found, using short name: {short_name}")
return short_name
def _parse_episode_from_filename(self, filename: str) -> Optional[Tuple[int, int]]:
"""Parse season and episode numbers from filename"""
# Try S##E## format first
match = re.search(r'[Ss](\d{1,2})[Ee](\d{1,2})', filename)
if match:
return int(match.group(1)), int(match.group(2))
# Try ##x## format
match = re.search(r'(\d{1,2})x(\d{1,2})', filename)
if match:
return int(match.group(1)), int(match.group(2))
return None
def create_episode_nfo(self, season_dir: Path, season_num: int, episode_num: int,
aired: Optional[str], dateadded: Optional[str], source: str,
lock_metadata: bool = True, enhanced_metadata: Optional[Dict[str, Any]] = None) -> None:
"""Create or update episode NFO file preserving existing content"""
# Get target NFO filename (prefer long name matching video file)
target_nfo_name = self._get_target_episode_nfo_name(season_dir, season_num, episode_num)
nfo_path = season_dir / target_nfo_name
try:
# Check for existing NFO file at target location
source_nfo_path = nfo_path if nfo_path.exists() else None
if source_nfo_path:
try:
tree = ET.parse(source_nfo_path)
episode = tree.getroot()
# Ensure root element is <episodedetails>
if episode.tag != "episodedetails":
raise ValueError("Root element is not <episodedetails>")
print(f"📝 Updating existing NFO: {nfo_path.name}")
# Preserve existing content fields and store NFOGuard-managed fields for re-adding
preserved_values = {}
# Extract and preserve NFOGuard-managed fields (we'll re-add these at the bottom)
nfoguard_fields = ["aired", "dateadded", "lockdata", "season", "episode"]
for tag in nfoguard_fields:
existing = episode.find(tag)
if existing is not None:
# Store the value before removing
if tag == "aired" and not aired:
aired = existing.text # Preserve existing aired date
preserved_values[tag] = existing.text
episode.remove(existing)
# Important: DO NOT remove content fields like title, plot, runtime, premiered, etc.
# These should be preserved from the long-named NFO files
# Debug: Show what fields are preserved after removing NFOGuard fields (if DEBUG enabled)
if self.debug:
preserved_fields = [elem.tag for elem in episode]
if preserved_fields:
print(f" 🔍 Content preserved after cleanup: {', '.join(preserved_fields)}")
else:
print(f" ℹ️ NFO contains only NFOGuard metadata (no additional content fields)")
except (ET.ParseError, ValueError) as e:
print(f"⚠️ Corrupted episode NFO detected: {nfo_path} - {str(e)[:100]}...")
print(f" Creating new clean episode NFO file to replace corrupted one")
episode = ET.Element("episodedetails")
else:
# Create new NFO structure
episode = ET.Element("episodedetails")
# Add enhanced metadata only if not already present (preserve existing from long-named NFO)
if enhanced_metadata:
if enhanced_metadata.get("title") and not episode.find("title"):
title_elem = ET.SubElement(episode, "title")
title_elem.text = enhanced_metadata["title"]
if enhanced_metadata.get("overview") and not episode.find("plot"):
plot_elem = ET.SubElement(episode, "plot")
plot_elem.text = enhanced_metadata["overview"]
if enhanced_metadata.get("runtime") and not episode.find("runtime"):
runtime_elem = ET.SubElement(episode, "runtime")
runtime_elem.text = str(enhanced_metadata["runtime"])
# Add NFOGuard fields at the bottom
# Basic episode info at the end
season_elem = ET.SubElement(episode, "season")
season_elem.text = str(season_num)
episode_elem = ET.SubElement(episode, "episode")
episode_elem.text = str(episode_num)
# Dates at the end
if aired:
aired_elem = ET.SubElement(episode, "aired")
# Convert datetime objects to strings
aired_str = str(aired)
aired_elem.text = aired_str[:10] if len(aired_str) >= 10 else aired_str
# Debug logging for dateadded
print(f"🔍 DEBUG: dateadded value: {repr(dateadded)} (type: {type(dateadded)})")
if dateadded:
dateadded_elem = ET.SubElement(episode, "dateadded")
# Convert datetime objects to strings
dateadded_elem.text = str(dateadded)
print(f"✅ Added dateadded to episode NFO: {dateadded}")
else:
print(f"❌ dateadded is empty/None, not adding to episode NFO")
# Add lockdata at the very end
if lock_metadata:
lockdata = ET.SubElement(episode, "lockdata")
lockdata.text = "true"
# Add NFOGuard comment at the bottom
comment = ET.Comment(f" Created by {self.manager_brand} - Source: {source} ")
episode.append(comment)
# Write file with proper formatting
tree = ET.ElementTree(episode)
ET.indent(tree, space=" ", level=0)
tree.write(nfo_path, encoding='utf-8', xml_declaration=True)
print(f"✅ Successfully created/updated episode NFO: {nfo_path}")
print(f" S{season_num:02d}E{episode_num:02d}, Aired: {aired}, Date Added: {dateadded}")
# NFO file created/updated successfully
pass
except Exception as e:
print(f"❌ Error creating/updating episode NFO {nfo_path}: {e}")
def set_file_mtime(self, file_path: Path, iso_timestamp) -> None:
"""Set file modification time to match import date"""
try:
# Convert datetime objects to strings first
if hasattr(iso_timestamp, 'isoformat'):
iso_timestamp = iso_timestamp.isoformat()
elif not isinstance(iso_timestamp, str):
iso_timestamp = str(iso_timestamp)
# Parse ISO timestamp
if iso_timestamp.endswith('Z'):
dt = datetime.fromisoformat(iso_timestamp.replace('Z', '+00:00'))
elif '+' in iso_timestamp or 'T' in iso_timestamp:
dt = datetime.fromisoformat(iso_timestamp)
elif ' ' in iso_timestamp:
# Handle space-separated datetime format (e.g., "2025-10-16 20:31:22")
dt = datetime.fromisoformat(iso_timestamp.replace(' ', 'T'))
else:
# Assume it's a simple date (e.g., "2025-10-16")
dt = datetime.fromisoformat(iso_timestamp + 'T00:00:00')
# Convert to timestamp
timestamp = dt.timestamp()
# Set both access and modification times
os.utime(file_path, (timestamp, timestamp))
print(f"✅ Updated file timestamp: {file_path.name} -> {iso_timestamp}")
except Exception as e:
print(f"❌ Error setting mtime for {file_path}: {e}")
def update_movie_files_mtime(self, movie_dir: Path, iso_timestamp: str) -> None:
"""Update modification times for all video files in movie directory"""
video_exts = (".mkv", ".mp4", ".avi", ".mov", ".m4v")
updated_files = []
for file_path in movie_dir.iterdir():
if file_path.is_file() and file_path.suffix.lower() in video_exts:
self.set_file_mtime(file_path, iso_timestamp)
updated_files.append(file_path.name)
if updated_files:
print(f"✅ Updated {len(updated_files)} video file timestamps in {movie_dir.name}")
else:
print(f"⚠️ No video files found to update in {movie_dir.name}")
main.py
+11 -11
View File
@@ -22,7 +22,7 @@ from utils.logging import _log
# Import core components
from core.database import NFOGuardDatabase
from core.nfo_manager import NFOManager
# from core.nfo_manager import NFOManager # Phase 3: Removed - no longer needed
from core.path_mapper import PathMapper
# Import clients
@@ -93,20 +93,20 @@ def initialize_components():
# Initialize core components
db = NFOGuardDatabase(config=config)
nfo_manager = NFOManager(config.manager_brand, config.debug)
# nfo_manager = NFOManager(config.manager_brand, config.debug) # Phase 3: Removed
path_mapper = PathMapper(config)
# Initialize processors
tv_processor = TVProcessor(db, nfo_manager, path_mapper)
movie_processor = MovieProcessor(db, nfo_manager, path_mapper)
# Initialize webhook batcher with nfo_manager for comprehensive IMDb detection
batcher = WebhookBatcher(nfo_manager)
# Initialize processors (nfo_manager=None for backward compatibility)
tv_processor = TVProcessor(db, None, path_mapper)
movie_processor = MovieProcessor(db, None, path_mapper)
# Initialize webhook batcher (no longer needs nfo_manager - Phase 3)
batcher = WebhookBatcher(nfo_manager=None)
batcher.set_processors(tv_processor, movie_processor)
return {
"db": db,
"nfo_manager": nfo_manager,
# "nfo_manager": nfo_manager, # Phase 3: Removed
"path_mapper": path_mapper,
"tv_processor": tv_processor,
"movie_processor": movie_processor,
processors/movie_processor.py
+42 -319
View File
@@ -11,12 +11,12 @@ from datetime import datetime, timezone
from zoneinfo import ZoneInfo
from core.database import NFOGuardDatabase
from core.nfo_manager import NFOManager
from core.path_mapper import PathMapper
from clients.radarr_client import RadarrClient
from clients.external_clients import ExternalClientManager
from config.settings import config
from utils.logging import _log
from utils.imdb_utils import find_imdb_in_directory # Phase 3: Replaced NFOManager
from utils.file_utils import find_media_path_by_imdb_and_title
@@ -68,9 +68,9 @@ def convert_utc_to_local(utc_iso_string: str) -> str:
class MovieProcessor:
"""Handles movie processing"""
def __init__(self, db: NFOGuardDatabase, nfo_manager: NFOManager, path_mapper: PathMapper):
def __init__(self, db: NFOGuardDatabase, nfo_manager, path_mapper: PathMapper):
# nfo_manager parameter kept for backward compatibility but no longer used (Phase 3)
self.db = db
self.nfo_manager = nfo_manager
self.path_mapper = path_mapper
self.radarr = RadarrClient(
os.environ.get("RADARR_URL", ""),
@@ -153,7 +153,7 @@ class MovieProcessor:
def process_movie(self, movie_path: Path, webhook_mode: bool = False, force_scan: bool = False, scan_mode: str = "smart", shutdown_event=None) -> str:
"""Process a movie directory"""
imdb_id = self.nfo_manager.find_movie_imdb_id(movie_path)
imdb_id = find_imdb_in_directory(movie_path) # Phase 3: Using imdb_utils instead of NFOManager
if not imdb_id:
_log("ERROR", f"No IMDb ID found in movie directory, filenames, or NFO file: {movie_path}")
return "error"
@@ -225,106 +225,16 @@ class MovieProcessor:
if released and hasattr(released, 'isoformat'):
released = released.isoformat()
# Create NFO with existing data and update files
if config.manage_nfo:
self.nfo_manager.create_movie_nfo(
movie_path, imdb_id, dateadded, released, source, config.lock_metadata
)
if config.fix_dir_mtimes and dateadded:
self.nfo_manager.update_movie_files_mtime(movie_path, dateadded)
# NFO file operations removed - database is now the single source of truth
# (Phase 1: Remove NFO file write operations)
_log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [database-cached]")
return "processed"
else:
_log("INFO", f"🔍 TIER 1 SKIP - {imdb_id}: Database incomplete, proceeding to Tier 2")
# TIER 2: Check if NFO file has NFOGuard data and cache it in database
nfo_path = movie_path / "movie.nfo"
_log("INFO", f"🔍 TIER 2 - Checking NFO file: {nfo_path}")
_log("INFO", f"🔍 TIER 2 - NFO exists: {nfo_path.exists()}")
nfo_data = self.nfo_manager.extract_nfoguard_dates_from_nfo(nfo_path)
_log("INFO", f"🔍 TIER 2 - NFOGuard data extracted: {nfo_data}")
if nfo_data:
_log("INFO", f"🚀 TIER 2 - Found NFOGuard data in NFO file: {nfo_data['dateadded']} (source: {nfo_data['source']})")
dateadded = nfo_data["dateadded"]
source = nfo_data["source"]
released = nfo_data.get("released")
# Cache NFO data in database for future lookups
# Fixed parameter order: imdb_id, released, dateadded, source
self.db.upsert_movie_dates(imdb_id, released, dateadded, source, True)
_log("INFO", f"✅ Cached NFO data in database for {imdb_id}")
# Update file mtimes if enabled (NFO is already correct)
if config.fix_dir_mtimes and dateadded:
self.nfo_manager.update_movie_files_mtime(movie_path, dateadded)
_log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [nfo-cached]")
return "processed"
# TIER 2.5: Check for any existing valid date data in NFO (even without lockdata marker)
# Only use NFO dates if prioritize_nfo is enabled, otherwise check external APIs first
existing_nfo_data = self._extract_any_valid_dates_from_nfo(nfo_path)
if existing_nfo_data and config.manual_scan_prioritize_nfo:
_log("INFO", f"🔍 TIER 2.5 - Found existing date data in NFO (no lockdata): {existing_nfo_data['dateadded']} (source: {existing_nfo_data['source']})")
_log("INFO", f"⚡ MANUAL_SCAN_PRIORITIZE_NFO=True - Using NFO date for speed")
dateadded = existing_nfo_data["dateadded"]
source = existing_nfo_data["source"]
released = existing_nfo_data.get("released")
# Cache existing data in database and add proper NFOGuard formatting
# Fixed parameter order: imdb_id, released, dateadded, source
self.db.upsert_movie_dates(imdb_id, released, dateadded, source, True)
_log("INFO", f"✅ Cached existing NFO data in database for {imdb_id}")
# Update NFO file to add NFOGuard formatting (lockdata, comment)
if config.manage_nfo:
self.nfo_manager.create_movie_nfo(
movie_path, imdb_id, dateadded, released, source, config.lock_metadata
)
_log("INFO", f"✅ Added NFOGuard formatting to existing NFO for {imdb_id}")
# Update file mtimes if enabled
if config.fix_dir_mtimes and dateadded:
self.nfo_manager.update_movie_files_mtime(movie_path, dateadded)
_log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [existing-nfo-enhanced]")
return "processed"
elif existing_nfo_data:
_log("INFO", f"🔍 TIER 2.5 - Found existing date data in NFO (no lockdata): {existing_nfo_data['dateadded']} (source: {existing_nfo_data['source']})")
_log("INFO", f"🎯 MANUAL_SCAN_PRIORITIZE_NFO=False - Will verify against external APIs first")
# Store NFO data as fallback but continue to TIER 3 to check external APIs
nfo_fallback_data = existing_nfo_data
# TIER 1.5: Special handling for TMDB-only movies - extract dates from existing NFO
if is_tmdb_fallback:
tmdb_nfo_data = self._extract_dates_from_tmdb_nfo(nfo_path)
if tmdb_nfo_data:
_log("INFO", f"🎬 Using TMDB data from existing NFO file: {tmdb_nfo_data['dateadded']} (source: {tmdb_nfo_data['source']})")
dateadded = tmdb_nfo_data["dateadded"]
source = tmdb_nfo_data["source"]
released = tmdb_nfo_data.get("released")
# Create NFO with NFOGuard fields added
if config.manage_nfo:
self.nfo_manager.create_movie_nfo(
movie_path, imdb_id, dateadded, released, source, config.lock_metadata
)
# Update file mtimes if enabled
if config.fix_dir_mtimes and dateadded:
self.nfo_manager.update_movie_files_mtime(movie_path, dateadded)
# Save to database
self.db.upsert_movie_dates(imdb_id, released, dateadded, source, True)
_log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [tmdb-nfo]")
return "processed"
# TIER 3: No cached data found - proceed with API lookups and verification
# TIER 2: Query external APIs directly (NFO layer removed in Phase 2)
_log("INFO", f"🔍 TIER 2 - No database cache, querying external APIs")
# Check for shutdown signal before expensive API operations
if shutdown_event and shutdown_event.is_set():
@@ -362,18 +272,8 @@ class MovieProcessor:
final_source = f"{source}_as_dateadded" if source else "release_date_fallback"
_log("INFO", f"Using release date as dateadded: {final_dateadded} (source: {final_source})")
# Create NFO regardless of date availability (preserves existing metadata)
if config.debug:
print(f"🔍 TIER3 - config.manage_nfo: {config.manage_nfo}")
if config.manage_nfo:
if config.debug:
print(f"🔍 TIER3 - Calling create_movie_nfo with final_dateadded: {final_dateadded}")
self.nfo_manager.create_movie_nfo(
movie_path, imdb_id, final_dateadded, released, final_source, config.lock_metadata
)
else:
if config.debug:
print(f"❌ TIER3 - manage_nfo is disabled, skipping NFO creation")
# NFO file operations removed - database is now the single source of truth
# (Phase 1: Remove NFO file write operations)
# Skip remaining processing if no valid date found and file dates disabled
if final_dateadded is None:
@@ -387,9 +287,8 @@ class MovieProcessor:
_log("DEBUG", f"Movie {movie_path.name} proceeding to save: dateadded={dateadded}, source={source}")
# Update file mtimes (only if we have a valid date)
if config.fix_dir_mtimes and dateadded and dateadded != "MANUAL_REVIEW_NEEDED":
self.nfo_manager.update_movie_files_mtime(movie_path, dateadded)
# File mtime operations removed - database is now the single source of truth
# (Phase 1: Remove NFO file write operations)
_log("DEBUG", f"Movie processing reached file mtime section: fix_dir_mtimes={config.fix_dir_mtimes}, dateadded={dateadded}")
@@ -407,76 +306,34 @@ class MovieProcessor:
return "processed"
def _process_movie_nfo_first(self, movie_path: Path, imdb_id: str, shutdown_event=None) -> str:
"""Process movie for incomplete mode: Check NFO files first for missing dateadded elements"""
_log("INFO", f"🔍 NFO-FIRST MODE: Checking movie for missing dateadded in NFO file: {movie_path.name}")
"""Process movie for incomplete mode: Database-first then API (NFO checks removed in Phase 2)"""
_log("INFO", f"🔍 INCOMPLETE MODE: Checking movie for missing data: {movie_path.name}")
# Check for shutdown signal
if shutdown_event and shutdown_event.is_set():
_log("INFO", f"⚠️ SHUTDOWN SIGNAL RECEIVED - Stopping movie NFO-first processing: {movie_path.name}")
_log("INFO", f"⚠️ SHUTDOWN SIGNAL RECEIVED - Stopping movie processing: {movie_path.name}")
return "shutdown"
nfo_path = movie_path / "movie.nfo"
# STEP 1: Check if NFO file exists and has dateadded
_log("DEBUG", f"STEP 1 - Checking NFO file for missing dateadded: {nfo_path}")
_log("INFO", f"🔍 NFO exists: {nfo_path.exists()}")
if nfo_path.exists():
nfo_data = self.nfo_manager.extract_nfoguard_dates_from_nfo(nfo_path)
_log("INFO", f"🔍 NFOGuard data extracted: {nfo_data}")
if nfo_data and nfo_data.get('dateadded'):
# NFO has dateadded - this movie is complete
_log("INFO", f"✅ NFO has dateadded={nfo_data['dateadded']}, movie marked as complete")
dateadded = nfo_data["dateadded"]
source = nfo_data["source"]
released = nfo_data.get("released")
# Cache NFO data in database for future lookups
self.db.upsert_movie_dates(imdb_id, released, dateadded, source, True)
# Update file mtimes if needed
if config.fix_dir_mtimes and dateadded:
self.nfo_manager.update_movie_files_mtime(movie_path, dateadded)
_log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [nfo-complete]")
return "processed"
else:
# NFO exists but missing dateadded
_log("INFO", f"🔍 NFO exists but missing dateadded - needs DB/API lookup")
else:
# No NFO file found
_log("INFO", f"🔍 No NFO file found - needs DB/API lookup")
# STEP 2: For movies missing dateadded in NFO, check database
_log("DEBUG", f"STEP 2 - Checking database for missing dateadded")
# STEP 1: Check database for existing data (Phase 2: NFO check removed)
_log("DEBUG", f"STEP 1 - Checking database for existing data")
existing = self.db.get_movie_dates(imdb_id)
if existing and existing.get("dateadded") and existing.get("source") != "no_valid_date_source":
# Found in database - use cached data and will add to NFO
_log("INFO", f"✅ Database has dateadded={existing['dateadded']} - will add to NFO")
# Found in database - data is complete
_log("INFO", f"✅ Database has dateadded={existing['dateadded']}")
dateadded, source, released = existing["dateadded"], existing["source"], existing.get("released")
# Convert datetime objects to strings for NFO manager
# Convert datetime objects to strings
if hasattr(dateadded, 'isoformat'):
dateadded = dateadded.isoformat()
if released and hasattr(released, 'isoformat'):
released = released.isoformat()
# Create NFO with database data
if config.manage_nfo:
self.nfo_manager.create_movie_nfo(
movie_path, imdb_id, dateadded, released, source, config.lock_metadata
)
if config.fix_dir_mtimes and dateadded:
self.nfo_manager.update_movie_files_mtime(movie_path, dateadded)
_log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [database-to-nfo]")
_log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [database-cached]")
return "processed"
# STEP 3: For movies still missing dateadded, query APIs
_log("DEBUG", f"STEP 3 - Querying APIs for missing dateadded")
# STEP 2: Database incomplete or missing, query APIs
_log("DEBUG", f"STEP 2 - Querying APIs for missing data")
# Check for shutdown signal before API calls
if shutdown_event and shutdown_event.is_set():
@@ -487,19 +344,10 @@ class MovieProcessor:
is_tmdb_fallback = imdb_id.startswith("tmdb-")
if is_tmdb_fallback:
# TMDB fallback processing
# TMDB fallback processing - use file modification time
_log("INFO", f"🔍 TMDB fallback processing for {imdb_id}")
# Check for existing TMDB NFO date data
tmdb_data = self._extract_dates_from_tmdb_nfo(nfo_path)
if tmdb_data and tmdb_data.get('dateadded'):
dateadded = tmdb_data['dateadded']
released = tmdb_data.get('released')
source = "tmdb_nfo"
else:
# Use file modification time as fallback for TMDB
dateadded, source, released = self._get_file_mtime_date(movie_path)
_log("INFO", f"Using file mtime as fallback for TMDB movie: {dateadded}")
dateadded, source, released = self._get_file_mtime_date(movie_path)
_log("INFO", f"Using file mtime for TMDB movie: {dateadded}")
else:
# Standard IMDb processing
# Try to get digital release date from external APIs
@@ -511,116 +359,22 @@ class MovieProcessor:
released = digital_date # For movies, digital release is often the key date
_log("INFO", f"Got digital release date from APIs: {dateadded} (source: {source})")
else:
# Check Radarr NFO for premiered date
radarr_premiered = self._get_radarr_nfo_premiered_date(movie_path)
if radarr_premiered:
dateadded = radarr_premiered
source = "radarr_nfo"
released = radarr_premiered
_log("INFO", f"Got premiered date from Radarr NFO: {dateadded}")
else:
# Last resort: file modification time
dateadded, source, released = self._get_file_mtime_date(movie_path)
_log("INFO", f"Using file mtime as last resort: {dateadded}")
# Last resort: file modification time
dateadded, source, released = self._get_file_mtime_date(movie_path)
_log("INFO", f"Using file mtime as fallback: {dateadded}")
# Save to database and create NFO
# Save to database only (NFO operations removed in Phase 1)
if dateadded:
self.db.upsert_movie_dates(imdb_id, released, dateadded, source, True)
if config.manage_nfo:
self.nfo_manager.create_movie_nfo(
movie_path, imdb_id, dateadded, released, source, config.lock_metadata
)
if config.fix_dir_mtimes and dateadded:
self.nfo_manager.update_movie_files_mtime(movie_path, dateadded)
_log("INFO", f"🔍 NFO-FIRST COMPLETE: {movie_path.name} (source: {source})")
_log("INFO", f"🔍 INCOMPLETE MODE COMPLETE: {movie_path.name} (source: {source})")
return "processed"
else:
_log("WARNING", f"Could not determine dateadded for movie: {movie_path.name}")
return "error"
def _extract_dates_from_tmdb_nfo(self, nfo_path: Path) -> Optional[Dict[str, str]]:
"""Extract date information from TMDB-based NFO file"""
if not nfo_path.exists():
return None
try:
root = self.nfo_manager._parse_nfo_with_tolerance(nfo_path)
# Look for premiered date (from TMDB)
premiered_elem = root.find('.//premiered')
if premiered_elem is not None and premiered_elem.text:
premiered_date = premiered_elem.text.strip()
print(f"✅ Found TMDB premiered date: {premiered_date}")
return {
"dateadded": premiered_date,
"source": "tmdb:premiered_from_nfo",
"released": premiered_date
}
except (ET.ParseError, Exception) as e:
print(f"⚠️ Error parsing TMDB NFO for dates: {e}")
return None
def _extract_any_valid_dates_from_nfo(self, nfo_path: Path) -> Optional[Dict[str, str]]:
"""Extract any valid date information from NFO file, even without NFOGuard markers"""
if not nfo_path.exists():
return None
try:
import xml.etree.ElementTree as ET
tree = ET.parse(nfo_path)
root = tree.getroot()
# Look for dateadded element (indicates previously processed by NFOGuard or similar)
dateadded_elem = root.find('.//dateadded')
premiered_elem = root.find('.//premiered')
if dateadded_elem is not None and dateadded_elem.text:
dateadded = dateadded_elem.text.strip()
# Try to determine source from NFOGuard comment if present
source = "existing_nfo_data"
try:
nfo_content = nfo_path.read_text(encoding='utf-8')
import re
# Look for NFOGuard comment pattern
source_match = re.search(r'<!--.*?NFOGuard.*?Source:\s*([^-\s]+).*?-->', nfo_content, re.DOTALL | re.IGNORECASE)
if source_match:
source = source_match.group(1).strip()
_log("DEBUG", f"Found source in NFOGuard comment: {source}")
else:
# Try to infer source from dateadded format/content
if "tmdb" in nfo_content.lower() or (premiered_elem and premiered_elem.text):
source = "tmdb:digital"
elif "radarr" in nfo_content.lower():
source = "radarr:db.history.import"
else:
source = "existing_nfo_data"
_log("DEBUG", f"Inferred source from NFO content: {source}")
except Exception as e:
_log("DEBUG", f"Could not determine source from NFO content: {e}")
result = {
"dateadded": dateadded,
"source": source
}
if premiered_elem is not None and premiered_elem.text:
result["released"] = premiered_elem.text.strip()
_log("INFO", f"✅ Found existing date data in NFO: dateadded={dateadded}, source={source}")
return result
except (ET.ParseError, Exception) as e:
_log("DEBUG", f"Error parsing NFO for existing date data: {e}")
return None
# NFO helper methods removed in Phase 2 - database is the single source of truth
def _decide_movie_dates(self, imdb_id: str, movie_path: Path, should_query: bool, existing: Optional[Dict]) -> Tuple[str, str, Optional[str]]:
"""Decide movie dates based on configuration and available data"""
_log("DEBUG", f"_decide_movie_dates for {imdb_id}: should_query={should_query}, existing={existing}")
@@ -687,14 +441,7 @@ class MovieProcessor:
# When using digital release date, store it as both dateadded and released
return digital_date, digital_source, digital_date
else:
_log("WARNING", f"⚠️ Movie {imdb_id}: No import date OR digital release date found - trying additional fallbacks")
# Try Radarr's own NFO premiered date as fallback
radarr_premiered = self._get_radarr_nfo_premiered_date(movie_path)
if radarr_premiered:
_log("INFO", f"✅ Movie {imdb_id}: Using Radarr NFO premiered date {radarr_premiered}")
# When using Radarr NFO premiered date, store it as both dateadded and released
return radarr_premiered, "radarr:nfo.premiered", radarr_premiered
_log("WARNING", f"⚠️ Movie {imdb_id}: No import date OR digital release date found")
else: # digital_then_import
# Try digital release first
@@ -753,32 +500,8 @@ class MovieProcessor:
_log("ERROR", f"External clients error for {imdb_id}: {e}")
return None, f"release:error:{str(e)}"
def _get_radarr_nfo_premiered_date(self, movie_path: Path) -> Optional[str]:
"""Extract premiered date from Radarr's existing movie.nfo file"""
try:
nfo_path = movie_path / "movie.nfo"
if not nfo_path.exists():
_log("DEBUG", f"No existing NFO file found at {nfo_path}")
return None
nfo_content = nfo_path.read_text(encoding='utf-8')
# Look for <premiered>YYYY-MM-DD</premiered>
match = re.search(r'<premiered>(\d{4}-\d{2}-\d{2})</premiered>', nfo_content)
if match:
premiered_date = match.group(1)
# Convert to ISO format with timezone
iso_date = f"{premiered_date}T00:00:00+00:00"
_log("INFO", f"✅ Found Radarr NFO premiered date: {premiered_date}")
return iso_date
else:
_log("DEBUG", f"No <premiered> tag found in existing NFO")
return None
except Exception as e:
_log("ERROR", f"Error reading Radarr NFO file: {e}")
return None
# _get_radarr_nfo_premiered_date() removed in Phase 2 - no longer reading NFO files
def _log_failed_movie(self, movie_path: Path, imdb_id: str, reason: str, available_countries: List[str] = None):
"""Log movies that failed to get valid dates to a debug file"""
try:
processors/tv_processor.py
+53 -226
View File
@@ -11,13 +11,12 @@ from typing import Optional, Dict, List, Set, Tuple, Any
from datetime import datetime
from core.database import NFOGuardDatabase
from core.nfo_manager import NFOManager
from core.async_nfo_manager import AsyncNFOManager
from core.path_mapper import PathMapper
from clients.sonarr_client import SonarrClient
from clients.external_clients import ExternalClientManager
from config.settings import config
from utils.logging import _log
from utils.imdb_utils import parse_imdb_from_path # Phase 3: Replaced NFOManager
from utils.file_utils import (
find_media_path_by_imdb_and_title,
find_episodes_on_disk,
@@ -32,10 +31,9 @@ from utils.async_file_utils import (
class TVProcessor:
"""Handles TV series processing"""
def __init__(self, db: NFOGuardDatabase, nfo_manager: NFOManager, path_mapper: PathMapper):
def __init__(self, db: NFOGuardDatabase, nfo_manager, path_mapper: PathMapper):
# nfo_manager parameter kept for backward compatibility but no longer used (Phase 3)
self.db = db
self.nfo_manager = nfo_manager
self.async_nfo_manager = AsyncNFOManager(config.manager_brand, config.debug)
self.path_mapper = path_mapper
self.sonarr = SonarrClient(
os.environ.get("SONARR_URL", ""),
@@ -147,7 +145,7 @@ class TVProcessor:
def process_series(self, series_path: Path, force_scan: bool = False, scan_mode: str = "smart") -> str:
"""Process a TV series directory"""
imdb_id = self.nfo_manager.parse_imdb_from_path(series_path)
imdb_id = parse_imdb_from_path(series_path) # Phase 3: Using imdb_utils
if not imdb_id:
_log("ERROR", f"No IMDb ID found in series path: {series_path}")
return "error"
@@ -193,19 +191,8 @@ class TVProcessor:
if (season, episode) in disk_episodes:
episode_count += 1
# Create NFO
if config.manage_nfo:
season_dir = series_path / config.get_season_dir_name(season)
self.nfo_manager.create_episode_nfo(
season_dir,
season, episode, aired, dateadded, source, config.lock_metadata
)
# Update file mtimes
if config.fix_dir_mtimes and dateadded:
video_files = disk_episodes[(season, episode)]
for video_file in video_files:
self.nfo_manager.set_file_mtime(video_file, dateadded)
# NFO file operations removed - database is now the single source of truth
# (Phase 1: Remove NFO file write operations)
# Save to database
try:
@@ -259,64 +246,12 @@ class TVProcessor:
# Not in database or incomplete - needs NFO check
episodes_needing_nfo_check.append((season, episode))
_log("INFO", f"Database cache hits: {db_cache_hits}/{len(disk_episodes)} episodes. Need NFO check: {len(episodes_needing_nfo_check)}")
# TIER 2: Check NFO files for NFOGuard dates and cache them in database
nfo_cache_hits = 0
episodes_needing_lookup = []
if episodes_needing_nfo_check:
_log("DEBUG", f"TIER 2 - Checking NFO files for NFOGuard dates for {len(episodes_needing_nfo_check)} episodes")
for (season, episode) in episodes_needing_nfo_check:
# Look for existing NFO files for this episode
season_dir = series_path / config.get_season_dir_name(season)
episode_files = disk_episodes[(season, episode)]
nfo_found = False
for episode_file in episode_files:
# Try to find matching NFO file
nfo_path = episode_file.with_suffix('.nfo')
if nfo_path.exists():
# Extract NFOGuard data from episode NFO
nfo_data = self.nfo_manager.extract_nfoguard_dates_from_nfo(nfo_path)
if nfo_data:
aired = nfo_data.get('aired')
dateadded = nfo_data.get('dateadded')
source = nfo_data.get('source', 'nfo_cache')
_log("DEBUG", f"S{season:02d}E{episode:02d}: NFO data found - aired={aired}, dateadded={dateadded}, source={source}")
# Skip incomplete NFO files with "unknown" source or no useful dates
if source == "unknown" and not dateadded and not aired:
_log("INFO", f"S{season:02d}E{episode:02d}: Ignoring incomplete NFO file with source 'unknown' and no dates")
break # Break out of NFO search to mark episode as needing API lookup
# Apply fallback logic if NFO has aired but no dateadded
if not dateadded and aired:
dateadded = aired
source = f"{source}_aired_fallback" if source != 'nfo_cache' else 'nfo_aired_fallback'
_log("DEBUG", f"S{season:02d}E{episode:02d}: NFO has aired but no dateadded, using aired as fallback: {dateadded}")
if dateadded:
episode_dates[(season, episode)] = (aired, dateadded, source)
nfo_cache_hits += 1
nfo_found = True
# Cache NFO data in database for future lookups
self.db.upsert_episode_date(imdb_id, season, episode, aired, dateadded, source, True)
_log("DEBUG", f"NFO cache hit for S{season:02d}E{episode:02d}: {dateadded} - cached in DB")
break
if not nfo_found:
# No NFO data found - needs API lookup
episodes_needing_lookup.append((season, episode))
_log("INFO", f"NFO cache hits: {nfo_cache_hits}/{len(episodes_needing_nfo_check)} episodes. Need API lookup: {len(episodes_needing_lookup)}")
# TIER 3: Only call Sonarr API for episodes not in database or NFO files
_log("INFO", f"TIER 1 - Database cache hits: {db_cache_hits}/{len(disk_episodes)} episodes. Need API lookup: {len(episodes_needing_nfo_check)}")
# TIER 2: Query Sonarr API for episodes not in database (NFO layer removed in Phase 2)
episodes_needing_lookup = episodes_needing_nfo_check # Renamed for clarity
if episodes_needing_lookup:
_log("DEBUG", f"TIER 3 - Querying Sonarr for {len(episodes_needing_lookup)} episodes missing from database and NFO files")
_log("DEBUG", f"TIER 2 - Querying Sonarr for {len(episodes_needing_lookup)} episodes missing from database")
sonarr_episodes = self._get_sonarr_episodes(imdb_id, episodes_needing_lookup)
# Process episodes that needed lookup
@@ -385,77 +320,35 @@ class TVProcessor:
return episode_dates
def _gather_episode_dates_nfo_first(self, series_path: Path, imdb_id: str, disk_episodes: Dict[Tuple[int, int], List[Path]]) -> Dict[Tuple[int, int], Tuple[Optional[str], Optional[str], str]]:
"""Gather episode dates for incomplete mode: Check NFO files first for missing dateadded elements"""
_log("INFO", f"🔍 NFO-FIRST MODE: Checking {len(disk_episodes)} episodes for missing dateadded in NFO files")
"""Gather episode dates for incomplete mode: Database-first then API (NFO checks removed in Phase 2)"""
_log("INFO", f"🔍 INCOMPLETE MODE: Checking {len(disk_episodes)} episodes for missing data")
episode_dates = {}
episodes_needing_db_check = []
episodes_needing_api_lookup = []
# STEP 1: Check each NFO file to determine if dateadded is missing
_log("DEBUG", f"STEP 1 - Checking NFO files for missing dateadded elements")
nfo_complete_count = 0
# STEP 1: Check database for all episodes (NFO check removed in Phase 2)
_log("DEBUG", f"STEP 1 - Checking database for {len(disk_episodes)} episodes")
db_hits = 0
for (season, episode) in disk_episodes:
season_dir = series_path / config.get_season_dir_name(season)
episode_files = disk_episodes[(season, episode)]
nfo_has_dateadded = False
nfo_data = None
# Look for NFO file for this episode
for episode_file in episode_files:
nfo_path = episode_file.with_suffix('.nfo')
if nfo_path.exists():
# Extract NFOGuard data from episode NFO
nfo_data = self.nfo_manager.extract_nfoguard_dates_from_nfo(nfo_path)
if nfo_data and nfo_data.get('dateadded'):
# NFO has dateadded - this episode is complete
aired = nfo_data.get('aired')
dateadded = nfo_data.get('dateadded')
source = nfo_data.get('source', 'nfo_cache')
episode_dates[(season, episode)] = (aired, dateadded, source)
nfo_complete_count += 1
nfo_has_dateadded = True
_log("DEBUG", f"S{season:02d}E{episode:02d}: NFO has dateadded={dateadded}, marked as complete")
break
if not nfo_has_dateadded:
# NFO missing or missing dateadded - needs further processing
if nfo_data:
# NFO exists but missing dateadded
_log("DEBUG", f"S{season:02d}E{episode:02d}: NFO exists but missing dateadded - needs DB/API lookup")
else:
# No NFO file found
_log("DEBUG", f"S{season:02d}E{episode:02d}: No NFO file found - needs DB/API lookup")
episodes_needing_db_check.append((season, episode))
_log("INFO", f"NFO files with dateadded: {nfo_complete_count}/{len(disk_episodes)} episodes. Need DB check: {len(episodes_needing_db_check)}")
# STEP 2: For episodes missing dateadded in NFO, check database
if episodes_needing_db_check:
_log("DEBUG", f"STEP 2 - Checking database for {len(episodes_needing_db_check)} episodes missing dateadded")
db_hits = 0
for (season, episode) in episodes_needing_db_check:
db_result = self.db.get_episode_date(imdb_id, season, episode)
if db_result and db_result.get('dateadded'):
# Found in database - use cached data and will add to NFO later
aired = db_result.get('aired')
dateadded = db_result.get('dateadded')
source = db_result.get('source', 'database_cache')
episode_dates[(season, episode)] = (aired, dateadded, source)
db_hits += 1
_log("DEBUG", f"S{season:02d}E{episode:02d}: Database has dateadded={dateadded} - will add to NFO")
else:
# Not in database or incomplete - needs API lookup
episodes_needing_api_lookup.append((season, episode))
_log("INFO", f"Database hits: {db_hits}/{len(episodes_needing_db_check)} episodes. Need API lookup: {len(episodes_needing_api_lookup)}")
# STEP 3: For episodes still missing dateadded, query Sonarr
db_result = self.db.get_episode_date(imdb_id, season, episode)
if db_result and db_result.get('dateadded'):
# Found in database - use cached data
aired = db_result.get('aired')
dateadded = db_result.get('dateadded')
source = db_result.get('source', 'database_cache')
episode_dates[(season, episode)] = (aired, dateadded, source)
db_hits += 1
_log("DEBUG", f"S{season:02d}E{episode:02d}: Database has dateadded={dateadded}")
else:
# Not in database or incomplete - needs API lookup
episodes_needing_api_lookup.append((season, episode))
_log("INFO", f"Database hits: {db_hits}/{len(disk_episodes)} episodes. Need API lookup: {len(episodes_needing_api_lookup)}")
# STEP 2: For episodes missing from database, query Sonarr
if episodes_needing_api_lookup:
_log("DEBUG", f"STEP 3 - Querying Sonarr for {len(episodes_needing_api_lookup)} episodes missing from NFO and database")
_log("DEBUG", f"STEP 2 - Querying Sonarr for {len(episodes_needing_api_lookup)} episodes missing from database")
sonarr_episodes = self._get_sonarr_episodes(imdb_id, episodes_needing_api_lookup)
for (season, episode) in episodes_needing_api_lookup:
@@ -503,7 +396,7 @@ class TVProcessor:
episode_dates[(season, episode)] = (aired, dateadded, source)
_log("INFO", f"🔍 NFO-FIRST COMPLETE: {len(episode_dates)} episodes processed")
_log("INFO", f"🔍 INCOMPLETE MODE COMPLETE: {len(episode_dates)} episodes processed")
for (s, e), (aired, dateadded, source) in episode_dates.items():
_log("INFO", f" S{s:02d}E{e:02d}: aired={aired}, dateadded={dateadded}, source={source}")
@@ -648,18 +541,8 @@ class TVProcessor:
processed_count = 0
for (season, episode), (aired, dateadded, source) in episode_dates.items():
if (season, episode) in season_episodes:
# Create NFO
if config.manage_nfo:
self.nfo_manager.create_episode_nfo(
season_path,
season, episode, aired, dateadded, source, config.lock_metadata
)
# Update file mtimes
if config.fix_dir_mtimes and dateadded:
video_files = season_episodes[(season, episode)]
for video_file in video_files:
self.nfo_manager.set_file_mtime(video_file, dateadded)
# NFO file operations removed - database is now the single source of truth
# (Phase 1: Remove NFO file write operations)
# Save to database
try:
@@ -728,16 +611,8 @@ class TVProcessor:
aired, dateadded, source = episode_dates[(season_num, episode_num)]
# Create NFO
if config.manage_nfo:
self.nfo_manager.create_episode_nfo(
season_path,
season_num, episode_num, aired, dateadded, source, config.lock_metadata
)
# Update file mtime
if config.fix_dir_mtimes and dateadded:
self.nfo_manager.set_file_mtime(episode_path, dateadded)
# NFO file operations removed - database is now the single source of truth
# (Phase 1: Remove NFO file write operations)
# Save to database
self.db.upsert_episode_date(imdb_id, season_num, episode_num, aired, dateadded, source, True)
@@ -765,7 +640,7 @@ class TVProcessor:
Returns:
Dictionary with processing results
"""
imdb_id = self.nfo_manager.parse_imdb_from_path(series_path)
imdb_id = parse_imdb_from_path(series_path) # Phase 3: Using imdb_utils
if not imdb_id:
return {"status": "error", "reason": f"No IMDb ID found in series path: {series_path}"}
@@ -787,25 +662,8 @@ class TVProcessor:
for (season, episode), (aired, dateadded, source) in episode_dates.items():
if (season, episode) in disk_episodes:
season_dir = series_path / config.get_season_dir_name(season)
# Prepare NFO creation data
if config.manage_nfo:
episode_data_list.append({
'season_dir': season_dir,
'season': season,
'episode': episode,
'aired': aired,
'dateadded': dateadded,
'source': source,
'lock_metadata': config.lock_metadata
})
# Prepare mtime operations
if config.fix_dir_mtimes and dateadded:
video_files = disk_episodes[(season, episode)]
for video_file in video_files:
mtime_operations.append((video_file, dateadded))
# NFO file operations removed - database is now the single source of truth
# (Phase 1: Remove NFO file write operations)
# Save to database
try:
@@ -815,27 +673,10 @@ class TVProcessor:
_log("ERROR", f"S{season:02d}E{episode:02d}: Database write failed: {e}")
# Continue processing other episodes
# Process NFOs and mtimes concurrently
# NFO and mtime operations removed - database is now the single source of truth
# (Phase 1: Remove NFO file write operations)
results = {}
if episode_data_list:
_log("INFO", f"Creating {len(episode_data_list)} episode NFOs concurrently")
nfo_results = await self.async_nfo_manager.async_batch_create_episode_nfos(
episode_data_list,
max_concurrent=config.max_concurrent
)
results['nfo_created'] = sum(nfo_results)
results['nfo_failed'] = len(nfo_results) - sum(nfo_results)
if mtime_operations:
_log("INFO", f"Setting mtimes for {len(mtime_operations)} files concurrently")
mtime_results = await self.async_nfo_manager.async_batch_set_file_mtimes(
mtime_operations,
max_concurrent=10
)
results['mtime_updated'] = sum(mtime_results)
results['mtime_failed'] = len(mtime_results) - sum(mtime_results)
_log("INFO", f"Completed async processing TV series: {series_path.name}")
return {
@@ -892,7 +733,7 @@ class TVProcessor:
def process_webhook_episodes(self, series_path: Path, webhook_episodes: List[Dict[str, Any]]) -> None:
"""Process only the specific episodes mentioned in a webhook (targeted mode)"""
imdb_id = self.nfo_manager.parse_imdb_from_path(series_path)
imdb_id = parse_imdb_from_path(series_path) # Phase 3: Using imdb_utils
if not imdb_id:
_log("ERROR", f"No IMDb ID found in series path: {series_path}")
return
@@ -942,18 +783,8 @@ class TVProcessor:
aired, dateadded, source = self._get_webhook_episode_date(imdb_id, season_num, episode_num, series_metadata, webhook_episode)
enhanced_metadata = self._get_episode_metadata(series_metadata, season_num, episode_num, season_dir) if series_metadata else self._get_episode_metadata(None, season_num, episode_num, season_dir)
# Create NFO
if config.manage_nfo:
self.nfo_manager.create_episode_nfo(
season_dir,
season_num, episode_num, aired, dateadded, source, config.lock_metadata,
enhanced_metadata
)
# Update file mtimes
if config.fix_dir_mtimes and dateadded:
for episode_file in episode_files:
self.nfo_manager.set_file_mtime(episode_file, dateadded)
# NFO file operations removed - database is now the single source of truth
# (Phase 1: Remove NFO file write operations)
# Save to database
self.db.upsert_episode_date(imdb_id, season_num, episode_num, aired, dateadded, source, True)
@@ -1208,17 +1039,13 @@ class TVProcessor:
dateadded = episode_data.get('dateadded')
# Get IMDb ID
imdb_id = self.nfo_manager.parse_imdb_from_path(series_path)
imdb_id = parse_imdb_from_path(series_path) # Phase 3: Using imdb_utils
if not imdb_id:
return {"status": "error", "reason": "No IMDb ID found"}
# Create NFO if needed
nfo_success = True
if config.manage_nfo:
season_dir = series_path / config.get_season_dir_name(season)
nfo_success = await self.async_nfo_manager.async_create_episode_nfo(
season_dir, season, episode, aired, dateadded, "webhook", config.lock_metadata
)
# NFO file operations removed - database is now the single source of truth
# (Phase 1: Remove NFO file write operations)
nfo_success = True # Always True since we're not creating NFOs anymore
# Update database
self.db.upsert_episode_date(imdb_id, season, episode, aired, dateadded, "webhook", True)
static/index.html
+33
View File
@@ -382,6 +382,39 @@
<i class="fas fa-sync"></i> Refresh Stats
</button>
</div>
<div class="tool-card">
<h3><i class="fas fa-upload"></i> Populate Database</h3>
<p>Bulk import data from Radarr/Sonarr into NFOGuard database</p>
<form id="populate-form">
<div class="form-group">
<label>Media Type:</label>
<select id="populate-media-type" required>
<option value="both">Movies & TV Shows</option>
<option value="movies">Movies Only</option>
<option value="tv">TV Shows Only</option>
</select>
</div>
<button type="submit" class="btn btn-primary">
<i class="fas fa-play"></i> Start Population
</button>
</form>
<div id="populate-status" class="scan-status" style="display: none;">
<div class="scan-progress">
<div class="progress-bar">
<div class="progress-fill" id="populate-progress-bar"></div>
</div>
<div class="scan-info">
<span id="populate-current-operation">Running...</span>
<span id="populate-progress-text">In Progress</span>
</div>
</div>
<div id="populate-results" class="stats-display" style="margin-top: 10px;"></div>
<button class="btn btn-secondary btn-sm" onclick="stopPopulatePolling()">
<i class="fas fa-times"></i> Hide Status
</button>
</div>
</div>
</div>
</div>
</main>
static/js/app.js
+184
View File
@@ -77,6 +77,7 @@ function initializeEventListeners() {
document.getElementById('edit-form').addEventListener('submit', handleEditSubmit);
document.getElementById('bulk-update-form').addEventListener('submit', handleBulkUpdate);
document.getElementById('manual-scan-form').addEventListener('submit', handleManualScan);
document.getElementById('populate-form').addEventListener('submit', handlePopulateDatabase);
}
// API calls
@@ -1742,3 +1743,186 @@ async function bulkDeleteSelected() {
showToast(`❌ Failed to delete ${failCount} episode(s)`, 'error');
}
}
// ---------------------------
// Database Population Functions
// ---------------------------
async function handlePopulateDatabase(event) {
event.preventDefault();
const mediaType = document.getElementById('populate-media-type').value;
// Validate input
if (!mediaType) {
showToast('❌ Please select media type', 'error');
return;
}
// Confirm with user
const confirmMsg = `Are you sure you want to populate the database with ${mediaType}? This will query Radarr/Sonarr and may take several minutes.`;
if (!confirm(confirmMsg)) {
return;
}
try {
// Show populate status
showPopulateStatus();
// Start the population
showToast('🚀 Starting database population...', 'info');
const response = await fetch(`/admin/populate-database?media_type=${mediaType}`, {
method: 'POST',
credentials: 'include'
});
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
}
const result = await response.json();
if (result.status === 'started') {
showToast('✅ Population started successfully', 'success');
// Start polling for status
startPopulatePolling();
} else {
showToast(`ℹ️ ${result.message || 'Population completed'}`, 'info');
hidePopulateStatus();
}
} catch (error) {
console.error('Database population failed:', error);
showToast(`❌ Population failed: ${error.message}`, 'error');
hidePopulateStatus();
}
}
function showPopulateStatus() {
const populateStatus = document.getElementById('populate-status');
const progressBar = document.getElementById('populate-progress-bar');
const operationText = document.getElementById('populate-current-operation');
const progressText = document.getElementById('populate-progress-text');
const resultsDiv = document.getElementById('populate-results');
populateStatus.style.display = 'block';
progressBar.style.width = '0%';
operationText.textContent = 'Initializing...';
progressText.textContent = '0%';
resultsDiv.innerHTML = '';
}
function hidePopulateStatus() {
const populateStatus = document.getElementById('populate-status');
if (populateStatus) {
populateStatus.style.display = 'none';
}
}
function startPopulatePolling() {
// Poll every 2 seconds for populate status
window.populatePollingInterval = setInterval(async () => {
try {
const response = await fetch('/api/populate/status');
if (!response.ok) {
throw new Error('Failed to get populate status');
}
const status = await response.json();
updatePopulateProgress(status);
// Stop polling if population is complete
if (!status.running && status.completed) {
stopPopulatePolling();
showToast('✅ Database population completed!', 'success');
}
} catch (error) {
console.error('Failed to poll populate status:', error);
// Don't show error toast repeatedly, just stop polling
stopPopulatePolling();
}
}, 2000);
}
function stopPopulatePolling() {
if (window.populatePollingInterval) {
clearInterval(window.populatePollingInterval);
window.populatePollingInterval = null;
}
}
function updatePopulateProgress(status) {
const progressBar = document.getElementById('populate-progress-bar');
const operationText = document.getElementById('populate-current-operation');
const progressText = document.getElementById('populate-progress-text');
const resultsDiv = document.getElementById('populate-results');
if (status.error) {
progressBar.style.width = '100%';
progressBar.style.backgroundColor = '#e74c3c';
operationText.textContent = 'Error occurred';
progressText.textContent = 'Failed';
resultsDiv.innerHTML = `<p style="color: #e74c3c;">Error: ${status.error}</p>`;
return;
}
if (!status.running && status.completed) {
progressBar.style.width = '100%';
operationText.textContent = 'Population completed';
progressText.textContent = '100%';
// Display results
let resultsHTML = '<h4>Population Results:</h4>';
if (status.movies && status.movies.stats) {
const m = status.movies.stats;
resultsHTML += `
<div style="margin: 10px 0;">
<strong>Movies:</strong><br>
Total: ${m.total || 0} | Added: ${m.added || 0} | Skipped: ${m.skipped || 0} | Errors: ${m.errors || 0}<br>
Duration: ${m.duration ? m.duration.toFixed(2) : 0}s
</div>
`;
}
if (status.tv && status.tv.stats) {
const t = status.tv.stats;
resultsHTML += `
<div style="margin: 10px 0;">
<strong>TV Shows:</strong><br>
Series: ${t.total_series || 0} | Episodes: ${t.total_episodes || 0}<br>
Added: ${t.added || 0} | Skipped: ${t.skipped || 0} | Errors: ${t.errors || 0}<br>
Duration: ${t.duration ? t.duration.toFixed(2) : 0}s
</div>
`;
}
resultsDiv.innerHTML = resultsHTML;
return;
}
// Update progress based on status
let totalProgress = 0;
let progressDetails = '';
if (status.movies && status.movies.status === 'running') {
totalProgress = 50;
progressDetails = 'Processing movies...';
} else if (status.movies && status.movies.status === 'completed') {
totalProgress = 50;
progressDetails = 'Movies completed, processing TV...';
}
if (status.tv && status.tv.status === 'running') {
totalProgress = 75;
progressDetails = 'Processing TV shows...';
} else if (status.tv && status.tv.status === 'completed') {
totalProgress = 100;
progressDetails = 'Completed';
}
progressBar.style.width = `${totalProgress}%`;
operationText.textContent = progressDetails;
progressText.textContent = `${totalProgress}%`;
}
utils/imdb_utils.py
+85
View File
@@ -0,0 +1,85 @@
#!/usr/bin/env python3
"""
IMDb ID Extraction Utilities
Parses IMDb IDs from directory/file names (no file I/O)
Phase 3: Replaces NFOManager for IMDb ID extraction only
"""
import re
from pathlib import Path
from typing import Optional
def parse_imdb_from_path(path: Path) -> Optional[str]:
"""
Extract IMDb ID from directory path or filename using regex patterns.
Does NOT read any files - only parses the path string.
Supported patterns:
- [imdb-tt1234567]
- [tt1234567]
- {imdb-tt1234567}
- (imdb-tt1234567)
- -tt1234567 (at end)
- _tt1234567 (at end)
Args:
path: Path object to parse
Returns:
IMDb ID (e.g., "tt1234567") or None if not found
"""
path_str = str(path).lower()
# Try [imdb-ttXXXXXXX] format first (most explicit)
match = re.search(r'\[imdb-?(tt\d+)\]', path_str)
if match:
return match.group(1)
# Try standalone [ttXXXXXXX] format in brackets
match = re.search(r'\[(tt\d+)\]', path_str)
if match:
return match.group(1)
# Try {imdb-ttXXXXXXX} format with curly braces
match = re.search(r'\{imdb-?(tt\d+)\}', path_str)
if match:
return match.group(1)
# Try (imdb-ttXXXXXXX) format with parentheses
match = re.search(r'\(imdb-?(tt\d+)\)', path_str)
if match:
return match.group(1)
# Try ttXXXXXXX at end of filename/dirname (common pattern)
match = re.search(r'[-_\s](tt\d+)$', path_str)
if match:
return match.group(1)
return None
def find_imdb_in_directory(directory: Path) -> Optional[str]:
"""
Find IMDb ID from directory name or filenames within the directory.
Does NOT read file contents - only checks filenames.
Args:
directory: Directory path to search
Returns:
IMDb ID or None if not found
"""
# First try directory name itself
imdb_id = parse_imdb_from_path(directory)
if imdb_id:
return imdb_id
# Try all filenames in the directory
if directory.is_dir():
for file_path in directory.iterdir():
if file_path.is_file():
imdb_id = parse_imdb_from_path(file_path)
if imdb_id:
return imdb_id
return None
webhooks/webhook_batcher.py
+33 -49
View File
@@ -10,12 +10,14 @@ from concurrent.futures import ThreadPoolExecutor
from config.settings import config
from utils.logging import _log
from utils.imdb_utils import find_imdb_in_directory, parse_imdb_from_path # Phase 3: Replaced NFOManager
class WebhookBatcher:
"""Batches webhook events to avoid processing storms"""
def __init__(self, nfo_manager=None):
# nfo_manager parameter kept for backward compatibility but no longer used (Phase 3)
self.pending: Dict[str, Dict] = {}
self.timers: Dict[str, threading.Timer] = {}
self.processing: Set[str] = set()
@@ -24,8 +26,6 @@ class WebhookBatcher:
# Will be set by the application when processors are available
self.tv_processor = None
self.movie_processor = None
# NFO manager for comprehensive IMDb detection
self.nfo_manager = nfo_manager
def set_processors(self, tv_processor, movie_processor):
"""Set the processor instances"""
@@ -84,56 +84,40 @@ class WebhookBatcher:
# CRITICAL: Validate that the path contains the expected IMDb ID for movies
if media_type == 'movie':
expected_imdb = key.replace('movie:', '') if key.startswith('movie:') else key
# Use comprehensive IMDb detection (directory, filenames, NFO files)
if self.nfo_manager:
detected_imdb = self.nfo_manager.find_movie_imdb_id(path_obj)
imdb_match = False
if detected_imdb:
# Compare with and without 'tt' prefix for flexibility
if detected_imdb == expected_imdb or detected_imdb.replace('tt', '') == expected_imdb.replace('tt', ''):
imdb_match = True
if not imdb_match:
_log("ERROR", f"BATCH VALIDATION FAILED: Expected IMDb {expected_imdb} not found via comprehensive detection in path {path_str}")
_log("ERROR", f"Detected IMDb: {detected_imdb}, Expected: {expected_imdb}")
_log("ERROR", f"This prevents processing wrong movies due to batch corruption")
return
_log("DEBUG", f"Batch validation passed: IMDb {expected_imdb} matches detected {detected_imdb}")
else:
# Fallback to simple string search if nfo_manager not available
if expected_imdb not in path_str.lower():
_log("ERROR", f"BATCH VALIDATION FAILED: Expected IMDb {expected_imdb} not found in path {path_str} (fallback mode)")
_log("ERROR", f"This prevents processing wrong movies due to batch corruption")
return
_log("DEBUG", f"Batch validation passed: IMDb {expected_imdb} found in path (fallback mode)")
# Use imdb_utils for IMDb detection (Phase 3: no NFO reading)
detected_imdb = find_imdb_in_directory(path_obj)
imdb_match = False
if detected_imdb:
# Compare with and without 'tt' prefix for flexibility
if detected_imdb == expected_imdb or detected_imdb.replace('tt', '') == expected_imdb.replace('tt', ''):
imdb_match = True
if not imdb_match:
_log("ERROR", f"BATCH VALIDATION FAILED: Expected IMDb {expected_imdb} but found {detected_imdb} in {path_str}")
_log("ERROR", f"Detected IMDb: {detected_imdb}, Expected: {expected_imdb}")
_log("ERROR", f"This prevents processing wrong movies due to batch corruption")
return
_log("DEBUG", f"Batch validation passed: IMDb {expected_imdb} matches detected {detected_imdb}")
# CRITICAL: Validate that the path contains the expected IMDb ID for TV shows
if media_type == 'tv':
expected_imdb = key.replace('tv:', '') if key.startswith('tv:') else key
# Use comprehensive IMDb detection (directory, filenames, tvshow.nfo files)
if self.nfo_manager:
detected_imdb = self.nfo_manager.find_series_imdb_id(path_obj)
imdb_match = False
if detected_imdb:
# Compare with and without 'tt' prefix for flexibility
if detected_imdb == expected_imdb or detected_imdb.replace('tt', '') == expected_imdb.replace('tt', ''):
imdb_match = True
if not imdb_match:
_log("ERROR", f"BATCH VALIDATION FAILED: Expected IMDb {expected_imdb} not found via comprehensive detection in TV path {path_str}")
_log("ERROR", f"Detected TV IMDb: {detected_imdb}, Expected: {expected_imdb}")
_log("ERROR", f"This prevents processing wrong TV series due to batch corruption")
return
_log("DEBUG", f"TV batch validation passed: IMDb {expected_imdb} matches detected {detected_imdb}")
else:
# Fallback to simple string search if nfo_manager not available
if expected_imdb not in path_str.lower():
_log("ERROR", f"BATCH VALIDATION FAILED: Expected IMDb {expected_imdb} not found in TV path {path_str} (fallback mode)")
_log("ERROR", f"This prevents processing wrong TV series due to batch corruption")
return
_log("DEBUG", f"TV batch validation passed: IMDb {expected_imdb} found in path (fallback mode)")
# Use imdb_utils for IMDb detection (Phase 3: no NFO reading)
detected_imdb = parse_imdb_from_path(path_obj)
imdb_match = False
if detected_imdb:
# Compare with and without 'tt' prefix for flexibility
if detected_imdb == expected_imdb or detected_imdb.replace('tt', '') == expected_imdb.replace('tt', ''):
imdb_match = True
if not imdb_match:
_log("ERROR", f"BATCH VALIDATION FAILED: Expected IMDb {expected_imdb} but found {detected_imdb} in TV {path_str}")
_log("ERROR", f"Detected TV IMDb: {detected_imdb}, Expected: {expected_imdb}")
_log("ERROR", f"This prevents processing wrong TV series due to batch corruption")
return
_log("DEBUG", f"TV batch validation passed: IMDb {expected_imdb} matches detected {detected_imdb}")
if not self.tv_processor:
_log("ERROR", "TV processor not available")