sbcrumb
82eb29fd63
Local Docker Build (Dev) / build-dev (pull_request) Successful in 20s
- Add utils/async_file_utils.py with aiofiles-based operations - Create core/async_nfo_manager.py for concurrent NFO processing - Add async methods to TVProcessor for concurrent episode handling - Implement async_find_episodes_on_disk() for faster scanning - Add async_batch_create_episode_nfos() for concurrent NFO creation - Support async_process_multiple_series() for parallel processing - Optimize I/O performance for large media libraries - Enable controlled concurrency with semaphores and rate limiting Performance improvements: - Concurrent file operations reduce I/O wait times - Batch processing minimizes individual file access overhead - Configurable concurrency limits prevent system overload - Async directory scanning for faster episode discovery
513 lines
20 KiB
Python
513 lines
20 KiB
Python
"""
|
|
TV Series Processor for NFOGuard
|
|
Handles TV series processing and episode management with async I/O support
|
|
"""
|
|
import os
|
|
import re
|
|
import asyncio
|
|
from pathlib import Path
|
|
from typing import Optional, Dict, List, Set, Tuple, Any
|
|
from datetime import datetime
|
|
|
|
from core.database import NFOGuardDatabase
|
|
from core.nfo_manager import NFOManager
|
|
from core.async_nfo_manager import AsyncNFOManager
|
|
from core.path_mapper import PathMapper
|
|
from clients.sonarr_client import SonarrClient
|
|
from clients.external_clients import ExternalClientManager
|
|
from config.settings import config
|
|
from utils.logging import _log
|
|
from utils.file_utils import (
|
|
find_media_path_by_imdb_and_title,
|
|
find_episodes_on_disk,
|
|
extract_title_from_directory_name
|
|
)
|
|
from utils.async_file_utils import (
|
|
async_find_episodes_on_disk,
|
|
async_concurrent_episode_processing
|
|
)
|
|
|
|
|
|
class TVProcessor:
|
|
"""Handles TV series processing"""
|
|
|
|
def __init__(self, db: NFOGuardDatabase, nfo_manager: NFOManager, path_mapper: PathMapper):
|
|
self.db = db
|
|
self.nfo_manager = nfo_manager
|
|
self.async_nfo_manager = AsyncNFOManager(config.manager_brand, config.debug)
|
|
self.path_mapper = path_mapper
|
|
self.sonarr = SonarrClient(
|
|
os.environ.get("SONARR_URL", ""),
|
|
os.environ.get("SONARR_API_KEY", "")
|
|
)
|
|
self.external_clients = ExternalClientManager()
|
|
|
|
def find_series_path(self, series_title: str, imdb_id: str, sonarr_path: str = None) -> Optional[Path]:
|
|
"""Find series directory path using unified file utilities"""
|
|
return find_media_path_by_imdb_and_title(
|
|
title=series_title,
|
|
imdb_id=imdb_id,
|
|
search_paths=config.tv_paths,
|
|
webhook_path=sonarr_path,
|
|
path_mapper=self.path_mapper
|
|
)
|
|
|
|
def process_series(self, series_path: Path) -> None:
|
|
"""Process a TV series directory"""
|
|
imdb_id = self.nfo_manager.parse_imdb_from_path(series_path)
|
|
if not imdb_id:
|
|
_log("ERROR", f"No IMDb ID found in series path: {series_path}")
|
|
return
|
|
|
|
_log("INFO", f"Processing TV series: {series_path.name}")
|
|
|
|
# Update database
|
|
self.db.upsert_series(imdb_id, str(series_path))
|
|
|
|
# Find video files
|
|
disk_episodes = find_episodes_on_disk(series_path)
|
|
_log("INFO", f"Found {len(disk_episodes)} episodes on disk")
|
|
|
|
# Get episode dates
|
|
episode_dates = self._gather_episode_dates(series_path, imdb_id, disk_episodes)
|
|
|
|
# Process episodes
|
|
for (season, episode), (aired, dateadded, source) in episode_dates.items():
|
|
if (season, episode) in disk_episodes:
|
|
# Create NFO
|
|
if config.manage_nfo:
|
|
season_dir = series_path / config.tv_season_dir_format.format(season=season)
|
|
self.nfo_manager.create_episode_nfo(
|
|
season_dir,
|
|
season, episode, aired, dateadded, source, config.lock_metadata
|
|
)
|
|
|
|
# Update file mtimes
|
|
if config.fix_dir_mtimes and dateadded:
|
|
video_files = disk_episodes[(season, episode)]
|
|
for video_file in video_files:
|
|
self.nfo_manager.set_file_mtime(video_file, dateadded)
|
|
|
|
# Save to database
|
|
self.db.upsert_episode_date(imdb_id, season, episode, aired, dateadded, source, True)
|
|
|
|
# Skip season.nfo and tvshow.nfo creation - focus only on episode NFOs
|
|
pass
|
|
|
|
_log("INFO", f"Completed processing TV series: {series_path.name}")
|
|
|
|
def _extract_series_title_from_path(self, series_path: Path) -> Optional[str]:
|
|
"""Extract series title from directory path using unified file utilities"""
|
|
return extract_title_from_directory_name(series_path.name)
|
|
|
|
|
|
def _gather_episode_dates(self, series_path: Path, imdb_id: str, disk_episodes: Dict[Tuple[int, int], List[Path]]) -> Dict[Tuple[int, int], Tuple[Optional[str], Optional[str], str]]:
|
|
"""Gather episode air dates and date added information"""
|
|
episode_dates = {}
|
|
|
|
# Get data from Sonarr first if available
|
|
sonarr_episodes = self._get_sonarr_episodes(imdb_id)
|
|
|
|
# Get data from external sources for missing information
|
|
for (season, episode) in disk_episodes:
|
|
aired = None
|
|
dateadded = None
|
|
source = "unknown"
|
|
|
|
# Try Sonarr first
|
|
if (season, episode) in sonarr_episodes:
|
|
sonarr_data = sonarr_episodes[(season, episode)]
|
|
aired = sonarr_data.get('airDate')
|
|
dateadded = sonarr_data.get('dateAdded')
|
|
if dateadded:
|
|
source = "sonarr"
|
|
|
|
# Fallback to external sources if needed
|
|
if not aired:
|
|
external_aired = self.external_clients.get_episode_air_date(imdb_id, season, episode)
|
|
if external_aired:
|
|
aired = external_aired
|
|
if not dateadded:
|
|
source = "external"
|
|
|
|
# Use air date as fallback for dateadded if available
|
|
if not dateadded and aired:
|
|
dateadded = aired
|
|
source = f"{source}_fallback" if source != "unknown" else "aired_fallback"
|
|
|
|
episode_dates[(season, episode)] = (aired, dateadded, source)
|
|
|
|
return episode_dates
|
|
|
|
def _get_sonarr_episodes(self, imdb_id: str) -> Dict[Tuple[int, int], Dict[str, Any]]:
|
|
"""Get episode information from Sonarr"""
|
|
try:
|
|
series_data = self.sonarr.get_series_by_imdb(imdb_id)
|
|
if not series_data:
|
|
return {}
|
|
|
|
series_id = series_data.get('id')
|
|
if not series_id:
|
|
return {}
|
|
|
|
episodes = self.sonarr.get_episodes(series_id)
|
|
|
|
episode_map = {}
|
|
for episode in episodes:
|
|
season = episode.get('seasonNumber', 0)
|
|
episode_num = episode.get('episodeNumber', 0)
|
|
|
|
if season > 0 and episode_num > 0:
|
|
episode_map[(season, episode_num)] = {
|
|
'airDate': episode.get('airDate'),
|
|
'dateAdded': episode.get('episodeFile', {}).get('dateAdded') if episode.get('hasFile') else None
|
|
}
|
|
|
|
return episode_map
|
|
|
|
except Exception as e:
|
|
_log("ERROR", f"Failed to get Sonarr episodes for {imdb_id}: {e}")
|
|
return {}
|
|
|
|
def process_season(self, series_path: str, season_name: str) -> Dict[str, Any]:
|
|
"""Process a specific season"""
|
|
series_path_obj = Path(series_path)
|
|
if not series_path_obj.exists():
|
|
raise FileNotFoundError(f"Series path not found: {series_path}")
|
|
|
|
season_path = series_path_obj / season_name
|
|
if not season_path.exists():
|
|
raise FileNotFoundError(f"Season directory not found: {season_path}")
|
|
|
|
# Extract season number from directory name
|
|
season_match = re.search(r'(\d+)', season_name)
|
|
if not season_match:
|
|
raise ValueError(f"Could not extract season number from: {season_name}")
|
|
|
|
season_num = int(season_match.group(1))
|
|
|
|
# Get series IMDb ID
|
|
imdb_id = self.nfo_manager.parse_imdb_from_path(series_path_obj)
|
|
if not imdb_id:
|
|
raise ValueError(f"No IMDb ID found in series path: {series_path}")
|
|
|
|
_log("INFO", f"Processing season {season_num} of series: {series_path_obj.name}")
|
|
|
|
# Find episodes in this season
|
|
disk_episodes = find_episodes_on_disk(series_path_obj)
|
|
season_episodes = {k: v for k, v in disk_episodes.items() if k[0] == season_num}
|
|
|
|
if not season_episodes:
|
|
return {"status": "no_episodes", "season": season_num, "episodes_found": 0}
|
|
|
|
# Get episode dates
|
|
episode_dates = self._gather_episode_dates(series_path_obj, imdb_id, season_episodes)
|
|
|
|
# Process episodes
|
|
processed_count = 0
|
|
for (season, episode), (aired, dateadded, source) in episode_dates.items():
|
|
if (season, episode) in season_episodes:
|
|
# Create NFO
|
|
if config.manage_nfo:
|
|
self.nfo_manager.create_episode_nfo(
|
|
season_path,
|
|
season, episode, aired, dateadded, source, config.lock_metadata
|
|
)
|
|
|
|
# Update file mtimes
|
|
if config.fix_dir_mtimes and dateadded:
|
|
video_files = season_episodes[(season, episode)]
|
|
for video_file in video_files:
|
|
self.nfo_manager.set_file_mtime(video_file, dateadded)
|
|
|
|
# Save to database
|
|
self.db.upsert_episode_date(imdb_id, season, episode, aired, dateadded, source, True)
|
|
processed_count += 1
|
|
|
|
_log("INFO", f"Processed {processed_count} episodes in season {season_num}")
|
|
|
|
return {
|
|
"status": "success",
|
|
"season": season_num,
|
|
"episodes_found": len(season_episodes),
|
|
"episodes_processed": processed_count
|
|
}
|
|
|
|
def process_episode(self, series_path: str, season_name: str, episode_name: str) -> Dict[str, Any]:
|
|
"""Process a specific episode"""
|
|
series_path_obj = Path(series_path)
|
|
if not series_path_obj.exists():
|
|
raise FileNotFoundError(f"Series path not found: {series_path}")
|
|
|
|
season_path = series_path_obj / season_name
|
|
if not season_path.exists():
|
|
raise FileNotFoundError(f"Season directory not found: {season_path}")
|
|
|
|
episode_path = season_path / episode_name
|
|
if not episode_path.exists():
|
|
raise FileNotFoundError(f"Episode file not found: {episode_path}")
|
|
|
|
# Extract season and episode numbers
|
|
season_match = re.search(r'(\d+)', season_name)
|
|
episode_match = re.search(r'[sS](\d+)[eE](\d+)|(\d+)x(\d+)', episode_name)
|
|
|
|
if not season_match:
|
|
raise ValueError(f"Could not extract season number from: {season_name}")
|
|
|
|
if not episode_match:
|
|
raise ValueError(f"Could not extract episode number from: {episode_name}")
|
|
|
|
season_num = int(season_match.group(1))
|
|
|
|
if episode_match.group(1) and episode_match.group(2): # SxxExx format
|
|
episode_num = int(episode_match.group(2))
|
|
elif episode_match.group(3) and episode_match.group(4): # NxNN format
|
|
episode_num = int(episode_match.group(4))
|
|
else:
|
|
raise ValueError(f"Could not parse episode number from: {episode_name}")
|
|
|
|
# Get series IMDb ID
|
|
imdb_id = self.nfo_manager.parse_imdb_from_path(series_path_obj)
|
|
if not imdb_id:
|
|
raise ValueError(f"No IMDb ID found in series path: {series_path}")
|
|
|
|
_log("INFO", f"Processing episode S{season_num:02d}E{episode_num:02d} of series: {series_path_obj.name}")
|
|
|
|
# Get episode data
|
|
disk_episodes = {(season_num, episode_num): [episode_path]}
|
|
episode_dates = self._gather_episode_dates(series_path_obj, imdb_id, disk_episodes)
|
|
|
|
if (season_num, episode_num) not in episode_dates:
|
|
return {"status": "no_data", "season": season_num, "episode": episode_num}
|
|
|
|
aired, dateadded, source = episode_dates[(season_num, episode_num)]
|
|
|
|
# Create NFO
|
|
if config.manage_nfo:
|
|
self.nfo_manager.create_episode_nfo(
|
|
season_path,
|
|
season_num, episode_num, aired, dateadded, source, config.lock_metadata
|
|
)
|
|
|
|
# Update file mtime
|
|
if config.fix_dir_mtimes and dateadded:
|
|
self.nfo_manager.set_file_mtime(episode_path, dateadded)
|
|
|
|
# Save to database
|
|
self.db.upsert_episode_date(imdb_id, season_num, episode_num, aired, dateadded, source, True)
|
|
|
|
_log("INFO", f"Processed episode S{season_num:02d}E{episode_num:02d}")
|
|
|
|
return {
|
|
"status": "success",
|
|
"season": season_num,
|
|
"episode": episode_num,
|
|
"aired": aired,
|
|
"dateadded": dateadded,
|
|
"source": source
|
|
}
|
|
|
|
# ===== ASYNC METHODS =====
|
|
|
|
async def async_process_series(self, series_path: Path) -> Dict[str, Any]:
|
|
"""
|
|
Async process a TV series directory with concurrent episode processing
|
|
|
|
Args:
|
|
series_path: Path to series directory
|
|
|
|
Returns:
|
|
Dictionary with processing results
|
|
"""
|
|
imdb_id = self.nfo_manager.parse_imdb_from_path(series_path)
|
|
if not imdb_id:
|
|
return {"status": "error", "reason": f"No IMDb ID found in series path: {series_path}"}
|
|
|
|
_log("INFO", f"Async processing TV series: {series_path.name}")
|
|
|
|
# Update database
|
|
self.db.upsert_series(imdb_id, str(series_path))
|
|
|
|
# Find video files asynchronously
|
|
disk_episodes = await async_find_episodes_on_disk(series_path)
|
|
_log("INFO", f"Found {len(disk_episodes)} episodes on disk")
|
|
|
|
# Get episode dates (sync for now, could be made async later)
|
|
episode_dates = self._gather_episode_dates(series_path, imdb_id, disk_episodes)
|
|
|
|
# Prepare episode data for concurrent processing
|
|
episode_data_list = []
|
|
mtime_operations = []
|
|
|
|
for (season, episode), (aired, dateadded, source) in episode_dates.items():
|
|
if (season, episode) in disk_episodes:
|
|
season_dir = series_path / config.tv_season_dir_format.format(season=season)
|
|
|
|
# Prepare NFO creation data
|
|
if config.manage_nfo:
|
|
episode_data_list.append({
|
|
'season_dir': season_dir,
|
|
'season': season,
|
|
'episode': episode,
|
|
'aired': aired,
|
|
'dateadded': dateadded,
|
|
'source': source,
|
|
'lock_metadata': config.lock_metadata
|
|
})
|
|
|
|
# Prepare mtime operations
|
|
if config.fix_dir_mtimes and dateadded:
|
|
video_files = disk_episodes[(season, episode)]
|
|
for video_file in video_files:
|
|
mtime_operations.append((video_file, dateadded))
|
|
|
|
# Save to database
|
|
self.db.upsert_episode_date(imdb_id, season, episode, aired, dateadded, source, True)
|
|
|
|
# Process NFOs and mtimes concurrently
|
|
results = {}
|
|
|
|
if episode_data_list:
|
|
_log("INFO", f"Creating {len(episode_data_list)} episode NFOs concurrently")
|
|
nfo_results = await self.async_nfo_manager.async_batch_create_episode_nfos(
|
|
episode_data_list,
|
|
max_concurrent=config.max_concurrent
|
|
)
|
|
results['nfo_created'] = sum(nfo_results)
|
|
results['nfo_failed'] = len(nfo_results) - sum(nfo_results)
|
|
|
|
if mtime_operations:
|
|
_log("INFO", f"Setting mtimes for {len(mtime_operations)} files concurrently")
|
|
mtime_results = await self.async_nfo_manager.async_batch_set_file_mtimes(
|
|
mtime_operations,
|
|
max_concurrent=10
|
|
)
|
|
results['mtime_updated'] = sum(mtime_results)
|
|
results['mtime_failed'] = len(mtime_results) - sum(mtime_results)
|
|
|
|
_log("INFO", f"Completed async processing TV series: {series_path.name}")
|
|
|
|
return {
|
|
"status": "success",
|
|
"imdb_id": imdb_id,
|
|
"episodes_found": len(disk_episodes),
|
|
"episodes_processed": len(episode_data_list),
|
|
"results": results
|
|
}
|
|
|
|
async def async_process_multiple_series(
|
|
self,
|
|
series_paths: List[Path],
|
|
max_concurrent: int = 2
|
|
) -> List[Dict[str, Any]]:
|
|
"""
|
|
Process multiple TV series concurrently
|
|
|
|
Args:
|
|
series_paths: List of series directory paths
|
|
max_concurrent: Maximum concurrent series processing
|
|
|
|
Returns:
|
|
List of processing results for each series
|
|
"""
|
|
semaphore = asyncio.Semaphore(max_concurrent)
|
|
|
|
async def _process_series_with_semaphore(series_path: Path) -> Dict[str, Any]:
|
|
async with semaphore:
|
|
try:
|
|
return await self.async_process_series(series_path)
|
|
except Exception as e:
|
|
_log("ERROR", f"Failed to process series {series_path}: {e}")
|
|
return {"status": "error", "path": str(series_path), "reason": str(e)}
|
|
|
|
_log("INFO", f"Processing {len(series_paths)} series with max {max_concurrent} concurrent")
|
|
|
|
tasks = [_process_series_with_semaphore(path) for path in series_paths]
|
|
results = await asyncio.gather(*tasks, return_exceptions=True)
|
|
|
|
# Filter out exceptions and convert to proper results
|
|
processed_results = []
|
|
for i, result in enumerate(results):
|
|
if isinstance(result, Exception):
|
|
processed_results.append({
|
|
"status": "error",
|
|
"path": str(series_paths[i]),
|
|
"reason": str(result)
|
|
})
|
|
else:
|
|
processed_results.append(result)
|
|
|
|
return processed_results
|
|
|
|
async def async_batch_episode_processing(
|
|
self,
|
|
episodes_data: List[Dict[str, Any]],
|
|
max_concurrent: int = 5
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
Process episodes from webhook data concurrently
|
|
|
|
Args:
|
|
episodes_data: List of episode data from webhooks
|
|
max_concurrent: Maximum concurrent episode processing
|
|
|
|
Returns:
|
|
Processing results summary
|
|
"""
|
|
async def _process_single_episode(episode_data: Dict[str, Any]) -> Dict[str, Any]:
|
|
try:
|
|
# Extract episode information
|
|
series_path = Path(episode_data.get('series_path'))
|
|
season = episode_data.get('season')
|
|
episode = episode_data.get('episode')
|
|
aired = episode_data.get('aired')
|
|
dateadded = episode_data.get('dateadded')
|
|
|
|
# Get IMDb ID
|
|
imdb_id = self.nfo_manager.parse_imdb_from_path(series_path)
|
|
if not imdb_id:
|
|
return {"status": "error", "reason": "No IMDb ID found"}
|
|
|
|
# Create NFO if needed
|
|
nfo_success = True
|
|
if config.manage_nfo:
|
|
season_dir = series_path / config.tv_season_dir_format.format(season=season)
|
|
nfo_success = await self.async_nfo_manager.async_create_episode_nfo(
|
|
season_dir, season, episode, aired, dateadded, "webhook", config.lock_metadata
|
|
)
|
|
|
|
# Update database
|
|
self.db.upsert_episode_date(imdb_id, season, episode, aired, dateadded, "webhook", True)
|
|
|
|
return {
|
|
"status": "success",
|
|
"season": season,
|
|
"episode": episode,
|
|
"nfo_created": nfo_success
|
|
}
|
|
|
|
except Exception as e:
|
|
return {
|
|
"status": "error",
|
|
"episode_data": episode_data,
|
|
"reason": str(e)
|
|
}
|
|
|
|
_log("INFO", f"Processing {len(episodes_data)} episodes concurrently")
|
|
|
|
results = await async_concurrent_episode_processing(
|
|
episodes_data,
|
|
_process_single_episode,
|
|
max_concurrent
|
|
)
|
|
|
|
# Summarize results
|
|
successful = sum(1 for r in results if r and r.get("status") == "success")
|
|
failed = len(results) - successful
|
|
|
|
return {
|
|
"total_episodes": len(episodes_data),
|
|
"successful": successful,
|
|
"failed": failed,
|
|
"results": results
|
|
} |