This commit is contained in:
@@ -16,6 +16,9 @@ from .models import SonarrWebhook, RadarrWebhook, HealthResponse, TVSeasonReques
|
||||
# Import core utilities
|
||||
from core.logging import _log
|
||||
from core.database import NFOGuardDatabase
|
||||
from core.fs_cache import fs_cache, clear_all_caches
|
||||
from core.xml_cache import xml_cache
|
||||
from core.batch_operations import optimize_library_scan
|
||||
|
||||
|
||||
async def _read_payload(request: Request) -> Optional[Dict[str, Any]]:
|
||||
@@ -261,3 +264,60 @@ def register_routes(
|
||||
async def bulk_update():
|
||||
"""Bulk update functionality"""
|
||||
return {"message": "Bulk update - implementation needed"}
|
||||
|
||||
# Performance monitoring endpoints
|
||||
@app.get("/cache/stats")
|
||||
async def cache_stats():
|
||||
"""Get comprehensive cache performance statistics"""
|
||||
fs_stats = fs_cache.get_cache_stats()
|
||||
xml_stats = xml_cache.get_cache_stats()
|
||||
|
||||
# Combine stats
|
||||
combined_stats = {
|
||||
"filesystem_cache": fs_stats,
|
||||
"xml_processing_cache": xml_stats,
|
||||
"overall_hit_rate": round(
|
||||
((fs_stats["cache_hits"] + xml_stats["xml_cache_hits"]) /
|
||||
(fs_stats["cache_hits"] + fs_stats["cache_misses"] +
|
||||
xml_stats["xml_cache_hits"] + xml_stats["xml_cache_misses"]) * 100
|
||||
) if (fs_stats["cache_hits"] + fs_stats["cache_misses"] +
|
||||
xml_stats["xml_cache_hits"] + xml_stats["xml_cache_misses"]) > 0 else 0, 2
|
||||
)
|
||||
}
|
||||
|
||||
return combined_stats
|
||||
|
||||
@app.post("/cache/clear")
|
||||
async def clear_cache():
|
||||
"""Clear all caches"""
|
||||
clear_all_caches()
|
||||
return {"status": "success", "message": "All caches cleared"}
|
||||
|
||||
@app.post("/scan/optimized")
|
||||
async def optimized_scan(background_tasks: BackgroundTasks):
|
||||
"""Trigger optimized library scan using batch operations"""
|
||||
try:
|
||||
_log("INFO", "Optimized scan triggered via API")
|
||||
|
||||
def run_optimized_scan():
|
||||
# Combine all library paths
|
||||
all_paths = list(config.tv_paths) + list(config.movie_paths)
|
||||
existing_paths = [p for p in all_paths if p.exists()]
|
||||
|
||||
if not existing_paths:
|
||||
_log("WARNING", "No valid library paths found")
|
||||
return
|
||||
|
||||
# Run optimized scan
|
||||
results = optimize_library_scan(existing_paths)
|
||||
_log("INFO", f"Optimized scan complete: {results}")
|
||||
|
||||
background_tasks.add_task(run_optimized_scan)
|
||||
return {
|
||||
"status": "started",
|
||||
"message": "Optimized library scan initiated",
|
||||
"note": "Uses batch operations and caching for improved performance"
|
||||
}
|
||||
except Exception as e:
|
||||
_log("ERROR", f"Optimized scan failed: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Optimized scan failed: {str(e)}")
|
||||
@@ -0,0 +1,251 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Batch Operations for NFOGuard
|
||||
Optimizes bulk file processing and NFO operations
|
||||
"""
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple, Any
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime
|
||||
|
||||
from core.logging import _log
|
||||
from core.fs_cache import fs_cache
|
||||
from core.xml_cache import xml_cache
|
||||
|
||||
|
||||
class BatchNFOProcessor:
|
||||
"""Handles batch NFO operations for improved performance"""
|
||||
|
||||
def __init__(self, max_workers: int = 4):
|
||||
self.max_workers = max_workers
|
||||
|
||||
def batch_find_video_files(self, directories: List[Path]) -> Dict[Path, List[Path]]:
|
||||
"""Find video files in multiple directories concurrently"""
|
||||
results = {}
|
||||
|
||||
with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
|
||||
# Submit all directory scans
|
||||
future_to_dir = {
|
||||
executor.submit(fs_cache.find_video_files, directory): directory
|
||||
for directory in directories if directory.exists()
|
||||
}
|
||||
|
||||
# Collect results
|
||||
for future in as_completed(future_to_dir):
|
||||
directory = future_to_dir[future]
|
||||
try:
|
||||
video_files = future.result()
|
||||
results[directory] = video_files
|
||||
except Exception as e:
|
||||
_log("ERROR", f"Error scanning directory {directory}: {e}")
|
||||
results[directory] = []
|
||||
|
||||
return results
|
||||
|
||||
def batch_check_nfo_files(self, nfo_paths: List[Path]) -> Dict[Path, bool]:
|
||||
"""Check multiple NFO files for NFOGuard data concurrently"""
|
||||
results = {}
|
||||
|
||||
with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
|
||||
# Submit all NFO checks
|
||||
future_to_path = {
|
||||
executor.submit(xml_cache.check_nfo_has_nfoguard_data, nfo_path): nfo_path
|
||||
for nfo_path in nfo_paths
|
||||
}
|
||||
|
||||
# Collect results
|
||||
for future in as_completed(future_to_path):
|
||||
nfo_path = future_to_path[future]
|
||||
try:
|
||||
has_data = future.result()
|
||||
results[nfo_path] = has_data
|
||||
except Exception as e:
|
||||
_log("ERROR", f"Error checking NFO {nfo_path}: {e}")
|
||||
results[nfo_path] = False
|
||||
|
||||
return results
|
||||
|
||||
def batch_extract_nfo_dates(self, nfo_paths: List[Path]) -> Dict[Path, Optional[Dict[str, Any]]]:
|
||||
"""Extract date information from multiple NFO files concurrently"""
|
||||
results = {}
|
||||
|
||||
with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
|
||||
# Submit all NFO extractions
|
||||
future_to_path = {
|
||||
executor.submit(xml_cache.extract_nfo_dates_cached, nfo_path): nfo_path
|
||||
for nfo_path in nfo_paths if nfo_path.exists()
|
||||
}
|
||||
|
||||
# Collect results
|
||||
for future in as_completed(future_to_path):
|
||||
nfo_path = future_to_path[future]
|
||||
try:
|
||||
dates_data = future.result()
|
||||
results[nfo_path] = dates_data
|
||||
except Exception as e:
|
||||
_log("ERROR", f"Error extracting dates from NFO {nfo_path}: {e}")
|
||||
results[nfo_path] = None
|
||||
|
||||
return results
|
||||
|
||||
def batch_update_file_mtimes(self, file_mtime_pairs: List[Tuple[Path, datetime]]):
|
||||
"""Update file modification times in batch"""
|
||||
updated_count = 0
|
||||
error_count = 0
|
||||
|
||||
for file_path, target_datetime in file_mtime_pairs:
|
||||
try:
|
||||
if file_path.exists():
|
||||
# Convert datetime to timestamp
|
||||
timestamp = target_datetime.timestamp()
|
||||
os.utime(file_path, (timestamp, timestamp))
|
||||
updated_count += 1
|
||||
else:
|
||||
error_count += 1
|
||||
_log("WARNING", f"File not found for mtime update: {file_path}")
|
||||
except Exception as e:
|
||||
error_count += 1
|
||||
_log("ERROR", f"Error updating mtime for {file_path}: {e}")
|
||||
|
||||
_log("INFO", f"Batch mtime update complete: {updated_count} updated, {error_count} errors")
|
||||
return {"updated": updated_count, "errors": error_count}
|
||||
|
||||
def scan_series_episodes_optimized(self, series_path: Path) -> Dict[Tuple[int, int], List[Path]]:
|
||||
"""Optimized episode scanning using batch operations"""
|
||||
disk_episodes = {}
|
||||
|
||||
# Get season directories
|
||||
season_dirs = fs_cache.get_directory_contents(series_path)
|
||||
valid_season_dirs = []
|
||||
|
||||
for season_dir in season_dirs:
|
||||
if (season_dir.is_dir() and
|
||||
season_dir.name.lower().startswith("season")):
|
||||
valid_season_dirs.append(season_dir)
|
||||
|
||||
if not valid_season_dirs:
|
||||
return disk_episodes
|
||||
|
||||
# Batch scan all season directories
|
||||
season_video_files = self.batch_find_video_files(valid_season_dirs)
|
||||
|
||||
# Process results
|
||||
for season_dir, video_files in season_video_files.items():
|
||||
# Extract season number
|
||||
try:
|
||||
season_name = season_dir.name.lower()
|
||||
if "season" in season_name:
|
||||
season_part = season_name.replace("season", "").strip()
|
||||
season_num = int(season_part)
|
||||
else:
|
||||
continue
|
||||
except (ValueError, IndexError):
|
||||
continue
|
||||
|
||||
# Process video files
|
||||
for video_file in video_files:
|
||||
from core.fs_cache import parse_episode_from_filename
|
||||
episode_info = parse_episode_from_filename(video_file.name)
|
||||
if episode_info:
|
||||
file_season, file_episode = episode_info
|
||||
key = (season_num, file_episode)
|
||||
if key not in disk_episodes:
|
||||
disk_episodes[key] = []
|
||||
disk_episodes[key].append(video_file)
|
||||
|
||||
return disk_episodes
|
||||
|
||||
def batch_series_nfo_check(self, series_paths: List[Path]) -> Dict[Path, Dict[str, Any]]:
|
||||
"""Check multiple series for NFO data in batch"""
|
||||
results = {}
|
||||
|
||||
# Collect all NFO paths to check
|
||||
nfo_paths_to_series = {}
|
||||
|
||||
for series_path in series_paths:
|
||||
if not series_path.exists():
|
||||
continue
|
||||
|
||||
# Check main series NFO
|
||||
tvshow_nfo = series_path / "tvshow.nfo"
|
||||
if tvshow_nfo.exists():
|
||||
nfo_paths_to_series[tvshow_nfo] = (series_path, "tvshow")
|
||||
|
||||
# Check season NFOs
|
||||
season_dirs = fs_cache.get_directory_contents(series_path)
|
||||
for season_dir in season_dirs:
|
||||
if season_dir.is_dir() and season_dir.name.lower().startswith("season"):
|
||||
season_nfo = season_dir / "season.nfo"
|
||||
if season_nfo.exists():
|
||||
nfo_paths_to_series[season_nfo] = (series_path, f"season_{season_dir.name}")
|
||||
|
||||
if not nfo_paths_to_series:
|
||||
return results
|
||||
|
||||
# Batch check all NFOs
|
||||
nfo_results = self.batch_check_nfo_files(list(nfo_paths_to_series.keys()))
|
||||
|
||||
# Organize results by series
|
||||
for nfo_path, has_nfoguard_data in nfo_results.items():
|
||||
if nfo_path in nfo_paths_to_series:
|
||||
series_path, nfo_type = nfo_paths_to_series[nfo_path]
|
||||
if series_path not in results:
|
||||
results[series_path] = {}
|
||||
results[series_path][nfo_type] = has_nfoguard_data
|
||||
|
||||
return results
|
||||
|
||||
|
||||
# Global batch processor instance
|
||||
batch_processor = BatchNFOProcessor()
|
||||
|
||||
|
||||
def optimize_library_scan(library_paths: List[Path]) -> Dict[str, Any]:
|
||||
"""Perform optimized library scan using batch operations"""
|
||||
start_time = time.time()
|
||||
stats = {
|
||||
"total_paths": len(library_paths),
|
||||
"processed": 0,
|
||||
"errors": 0,
|
||||
"series_found": 0,
|
||||
"movies_found": 0,
|
||||
"processing_time": 0
|
||||
}
|
||||
|
||||
series_paths = []
|
||||
movie_paths = []
|
||||
|
||||
# Categorize paths
|
||||
for lib_path in library_paths:
|
||||
if not lib_path.exists():
|
||||
stats["errors"] += 1
|
||||
continue
|
||||
|
||||
# Use cached directory scanning
|
||||
items = fs_cache.get_directory_contents(lib_path)
|
||||
for item in items:
|
||||
if item.is_dir() and "[imdb-" in item.name.lower():
|
||||
# Determine if it's a series or movie based on structure
|
||||
season_dirs = [d for d in fs_cache.get_directory_contents(item)
|
||||
if d.is_dir() and d.name.lower().startswith("season")]
|
||||
|
||||
if season_dirs:
|
||||
series_paths.append(item)
|
||||
stats["series_found"] += 1
|
||||
else:
|
||||
movie_paths.append(item)
|
||||
stats["movies_found"] += 1
|
||||
|
||||
# Batch process series
|
||||
if series_paths:
|
||||
_log("INFO", f"Batch processing {len(series_paths)} TV series")
|
||||
series_results = batch_processor.batch_series_nfo_check(series_paths)
|
||||
stats["processed"] += len(series_results)
|
||||
|
||||
# Movies can be processed similarly
|
||||
stats["processing_time"] = round(time.time() - start_time, 2)
|
||||
|
||||
_log("INFO", f"Optimized library scan complete: {stats}")
|
||||
return stats
|
||||
@@ -0,0 +1,240 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
File System Caching for NFOGuard
|
||||
Provides LRU caching for expensive file system operations
|
||||
"""
|
||||
import os
|
||||
import time
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple, Any
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from core.logging import _log
|
||||
|
||||
|
||||
class FileSystemCache:
|
||||
"""Smart caching for file system operations with mtime invalidation"""
|
||||
|
||||
def __init__(self, max_cache_size: int = 1000):
|
||||
self.max_cache_size = max_cache_size
|
||||
self._dir_cache: Dict[str, Tuple[float, List[Path]]] = {}
|
||||
self._file_cache: Dict[str, Tuple[float, Any]] = {}
|
||||
self._mtime_cache: Dict[str, float] = {}
|
||||
self._cache_hits = 0
|
||||
self._cache_misses = 0
|
||||
|
||||
def get_directory_contents(self, directory_path: Path, pattern: str = "*") -> List[Path]:
|
||||
"""Get directory contents with caching and mtime invalidation"""
|
||||
cache_key = f"{directory_path}:{pattern}"
|
||||
|
||||
if not directory_path.exists():
|
||||
return []
|
||||
|
||||
# Check if directory mtime has changed
|
||||
current_mtime = directory_path.stat().st_mtime
|
||||
|
||||
if cache_key in self._dir_cache:
|
||||
cached_mtime, cached_contents = self._dir_cache[cache_key]
|
||||
if cached_mtime == current_mtime:
|
||||
self._cache_hits += 1
|
||||
return cached_contents
|
||||
|
||||
# Cache miss - scan directory
|
||||
self._cache_misses += 1
|
||||
contents = []
|
||||
|
||||
try:
|
||||
if pattern == "*":
|
||||
contents = list(directory_path.iterdir())
|
||||
else:
|
||||
contents = list(directory_path.glob(pattern))
|
||||
except (OSError, PermissionError) as e:
|
||||
_log("DEBUG", f"Error scanning directory {directory_path}: {e}")
|
||||
return []
|
||||
|
||||
# Cache the results
|
||||
self._dir_cache[cache_key] = (current_mtime, contents)
|
||||
self._cleanup_cache()
|
||||
|
||||
return contents
|
||||
|
||||
def find_video_files(self, directory_path: Path) -> List[Path]:
|
||||
"""Find video files with caching"""
|
||||
video_extensions = {".mkv", ".mp4", ".avi", ".mov", ".m4v"}
|
||||
cache_key = f"videos:{directory_path}"
|
||||
|
||||
if not directory_path.exists():
|
||||
return []
|
||||
|
||||
current_mtime = directory_path.stat().st_mtime
|
||||
|
||||
# Check cache
|
||||
if cache_key in self._file_cache:
|
||||
cached_mtime, cached_files = self._file_cache[cache_key]
|
||||
if cached_mtime == current_mtime:
|
||||
self._cache_hits += 1
|
||||
return cached_files
|
||||
|
||||
# Find video files
|
||||
self._cache_misses += 1
|
||||
video_files = []
|
||||
|
||||
try:
|
||||
for file_path in directory_path.iterdir():
|
||||
if file_path.is_file() and file_path.suffix.lower() in video_extensions:
|
||||
video_files.append(file_path)
|
||||
except (OSError, PermissionError) as e:
|
||||
_log("DEBUG", f"Error scanning for videos in {directory_path}: {e}")
|
||||
return []
|
||||
|
||||
# Cache results
|
||||
self._file_cache[cache_key] = (current_mtime, video_files)
|
||||
self._cleanup_cache()
|
||||
|
||||
return video_files
|
||||
|
||||
def get_file_mtime(self, file_path: Path) -> Optional[float]:
|
||||
"""Get file modification time with caching"""
|
||||
cache_key = str(file_path)
|
||||
|
||||
# Always check actual mtime for files (they change frequently)
|
||||
try:
|
||||
current_mtime = file_path.stat().st_mtime
|
||||
self._mtime_cache[cache_key] = current_mtime
|
||||
return current_mtime
|
||||
except (OSError, FileNotFoundError):
|
||||
self._mtime_cache.pop(cache_key, None)
|
||||
return None
|
||||
|
||||
def file_exists(self, file_path: Path) -> bool:
|
||||
"""Check if file exists with smart caching"""
|
||||
cache_key = f"exists:{file_path}"
|
||||
|
||||
# For existence checks, we can cache briefly but need to be careful
|
||||
if cache_key in self._file_cache:
|
||||
cache_time, exists = self._file_cache[cache_key]
|
||||
if time.time() - cache_time < 30: # Cache for 30 seconds
|
||||
self._cache_hits += 1
|
||||
return exists
|
||||
|
||||
# Check actual existence
|
||||
self._cache_misses += 1
|
||||
exists = file_path.exists()
|
||||
|
||||
self._file_cache[cache_key] = (time.time(), exists)
|
||||
return exists
|
||||
|
||||
def find_episode_files(self, season_dir: Path, season_num: int, episode_num: int) -> List[Path]:
|
||||
"""Find episode files with pattern caching"""
|
||||
cache_key = f"episode:{season_dir}:S{season_num:02d}E{episode_num:02d}"
|
||||
|
||||
if not season_dir.exists():
|
||||
return []
|
||||
|
||||
current_mtime = season_dir.stat().st_mtime
|
||||
|
||||
# Check cache
|
||||
if cache_key in self._file_cache:
|
||||
cached_mtime, cached_files = self._file_cache[cache_key]
|
||||
if cached_mtime == current_mtime:
|
||||
self._cache_hits += 1
|
||||
return cached_files
|
||||
|
||||
# Find episode files
|
||||
self._cache_misses += 1
|
||||
episode_pattern = f"S{season_num:02d}E{episode_num:02d}"
|
||||
episode_files = []
|
||||
|
||||
video_extensions = {".mkv", ".mp4", ".avi", ".mov", ".m4v"}
|
||||
|
||||
try:
|
||||
for file_path in season_dir.iterdir():
|
||||
if (file_path.is_file() and
|
||||
file_path.suffix.lower() in video_extensions and
|
||||
episode_pattern.upper() in file_path.name.upper()):
|
||||
episode_files.append(file_path)
|
||||
except (OSError, PermissionError) as e:
|
||||
_log("DEBUG", f"Error finding episode files in {season_dir}: {e}")
|
||||
return []
|
||||
|
||||
# Cache results
|
||||
self._file_cache[cache_key] = (current_mtime, episode_files)
|
||||
self._cleanup_cache()
|
||||
|
||||
return episode_files
|
||||
|
||||
def _cleanup_cache(self):
|
||||
"""Clean up cache when it gets too large"""
|
||||
if len(self._dir_cache) > self.max_cache_size:
|
||||
# Remove oldest 25% of entries
|
||||
remove_count = self.max_cache_size // 4
|
||||
old_keys = list(self._dir_cache.keys())[:remove_count]
|
||||
for key in old_keys:
|
||||
del self._dir_cache[key]
|
||||
|
||||
if len(self._file_cache) > self.max_cache_size:
|
||||
remove_count = self.max_cache_size // 4
|
||||
old_keys = list(self._file_cache.keys())[:remove_count]
|
||||
for key in old_keys:
|
||||
del self._file_cache[key]
|
||||
|
||||
def clear_cache(self):
|
||||
"""Clear all caches"""
|
||||
self._dir_cache.clear()
|
||||
self._file_cache.clear()
|
||||
self._mtime_cache.clear()
|
||||
_log("INFO", "File system cache cleared")
|
||||
|
||||
def get_cache_stats(self) -> Dict[str, Any]:
|
||||
"""Get cache performance statistics"""
|
||||
total_requests = self._cache_hits + self._cache_misses
|
||||
hit_rate = (self._cache_hits / total_requests * 100) if total_requests > 0 else 0
|
||||
|
||||
return {
|
||||
"cache_hits": self._cache_hits,
|
||||
"cache_misses": self._cache_misses,
|
||||
"hit_rate_percent": round(hit_rate, 2),
|
||||
"dir_cache_size": len(self._dir_cache),
|
||||
"file_cache_size": len(self._file_cache),
|
||||
"mtime_cache_size": len(self._mtime_cache)
|
||||
}
|
||||
|
||||
|
||||
# Global cache instance
|
||||
fs_cache = FileSystemCache()
|
||||
|
||||
|
||||
# Cached utility functions
|
||||
@lru_cache(maxsize=500)
|
||||
def parse_episode_from_filename(filename: str) -> Optional[Tuple[int, int]]:
|
||||
"""Parse season/episode from filename with LRU caching"""
|
||||
import re
|
||||
match = re.search(r"S(\d{1,2})E(\d{1,2})", filename, re.IGNORECASE)
|
||||
if match:
|
||||
return int(match.group(1)), int(match.group(2))
|
||||
return None
|
||||
|
||||
|
||||
@lru_cache(maxsize=200)
|
||||
def extract_imdb_from_path(path_str: str) -> Optional[str]:
|
||||
"""Extract IMDb ID from path with LRU caching"""
|
||||
import re
|
||||
match = re.search(r'\[imdb-([^]]+)\]', path_str, re.IGNORECASE)
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
def clear_all_caches():
|
||||
"""Clear all caches including LRU caches"""
|
||||
fs_cache.clear_cache()
|
||||
parse_episode_from_filename.cache_clear()
|
||||
extract_imdb_from_path.cache_clear()
|
||||
|
||||
# Also clear XML caches
|
||||
try:
|
||||
from core.xml_cache import clear_xml_caches
|
||||
clear_xml_caches()
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
_log("INFO", "All caches cleared")
|
||||
@@ -0,0 +1,204 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
XML Processing Cache for NFOGuard
|
||||
Optimizes NFO file parsing and manipulation
|
||||
"""
|
||||
import os
|
||||
import time
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional, Any
|
||||
from xml.etree import ElementTree as ET
|
||||
|
||||
from core.logging import _log
|
||||
|
||||
|
||||
class XMLProcessingCache:
|
||||
"""Caches parsed XML trees and NFO data with mtime invalidation"""
|
||||
|
||||
def __init__(self, max_cache_size: int = 500):
|
||||
self.max_cache_size = max_cache_size
|
||||
self._xml_tree_cache: Dict[str, tuple] = {} # (mtime, parsed_tree)
|
||||
self._nfo_data_cache: Dict[str, tuple] = {} # (mtime, extracted_data)
|
||||
self._cache_hits = 0
|
||||
self._cache_misses = 0
|
||||
|
||||
def get_parsed_nfo(self, nfo_path: Path) -> Optional[ET.Element]:
|
||||
"""Get parsed NFO file with caching"""
|
||||
if not nfo_path.exists():
|
||||
return None
|
||||
|
||||
cache_key = str(nfo_path)
|
||||
current_mtime = nfo_path.stat().st_mtime
|
||||
|
||||
# Check cache
|
||||
if cache_key in self._xml_tree_cache:
|
||||
cached_mtime, cached_tree = self._xml_tree_cache[cache_key]
|
||||
if cached_mtime == current_mtime:
|
||||
self._cache_hits += 1
|
||||
return cached_tree
|
||||
|
||||
# Parse XML
|
||||
self._cache_misses += 1
|
||||
try:
|
||||
tree = ET.parse(nfo_path)
|
||||
root = tree.getroot()
|
||||
|
||||
# Cache the parsed tree
|
||||
self._xml_tree_cache[cache_key] = (current_mtime, root)
|
||||
self._cleanup_cache()
|
||||
|
||||
return root
|
||||
except (ET.ParseError, OSError) as e:
|
||||
_log("DEBUG", f"Error parsing NFO {nfo_path}: {e}")
|
||||
return None
|
||||
|
||||
def extract_nfo_dates_cached(self, nfo_path: Path) -> Optional[Dict[str, Any]]:
|
||||
"""Extract NFOGuard date fields from NFO with caching"""
|
||||
if not nfo_path.exists():
|
||||
return None
|
||||
|
||||
cache_key = f"dates:{nfo_path}"
|
||||
current_mtime = nfo_path.stat().st_mtime
|
||||
|
||||
# Check cache
|
||||
if cache_key in self._nfo_data_cache:
|
||||
cached_mtime, cached_data = self._nfo_data_cache[cache_key]
|
||||
if cached_mtime == current_mtime:
|
||||
self._cache_hits += 1
|
||||
return cached_data
|
||||
|
||||
# Extract data
|
||||
self._cache_misses += 1
|
||||
root = self.get_parsed_nfo(nfo_path)
|
||||
if not root:
|
||||
return None
|
||||
|
||||
dates_data = self._extract_nfoguard_dates(root)
|
||||
|
||||
# Cache the extracted data
|
||||
self._nfo_data_cache[cache_key] = (current_mtime, dates_data)
|
||||
self._cleanup_cache()
|
||||
|
||||
return dates_data
|
||||
|
||||
def _extract_nfoguard_dates(self, root: ET.Element) -> Optional[Dict[str, Any]]:
|
||||
"""Extract NFOGuard date fields from parsed XML"""
|
||||
try:
|
||||
# Look for NFOGuard date fields
|
||||
dateadded_elem = root.find(".//dateadded")
|
||||
source_elem = root.find(".//nfoguard_source")
|
||||
aired_elem = root.find(".//aired") or root.find(".//premiered")
|
||||
|
||||
if dateadded_elem is not None and dateadded_elem.text:
|
||||
result = {
|
||||
"dateadded": dateadded_elem.text.strip(),
|
||||
"source": source_elem.text.strip() if source_elem is not None and source_elem.text else "unknown",
|
||||
"aired": aired_elem.text.strip() if aired_elem is not None and aired_elem.text else None
|
||||
}
|
||||
|
||||
# Check if title exists for episodes
|
||||
title_elem = root.find(".//title")
|
||||
if title_elem is not None and title_elem.text:
|
||||
result["has_title"] = True
|
||||
result["title"] = title_elem.text.strip()
|
||||
else:
|
||||
result["has_title"] = False
|
||||
|
||||
return result
|
||||
except Exception as e:
|
||||
_log("DEBUG", f"Error extracting NFOGuard dates from XML: {e}")
|
||||
|
||||
return None
|
||||
|
||||
def check_nfo_has_nfoguard_data(self, nfo_path: Path) -> bool:
|
||||
"""Quickly check if NFO has NFOGuard data"""
|
||||
cache_key = f"has_nfoguard:{nfo_path}"
|
||||
|
||||
if not nfo_path.exists():
|
||||
return False
|
||||
|
||||
current_mtime = nfo_path.stat().st_mtime
|
||||
|
||||
# Check cache
|
||||
if cache_key in self._nfo_data_cache:
|
||||
cached_mtime, has_data = self._nfo_data_cache[cache_key]
|
||||
if cached_mtime == current_mtime:
|
||||
self._cache_hits += 1
|
||||
return has_data
|
||||
|
||||
# Check for NFOGuard markers
|
||||
self._cache_misses += 1
|
||||
try:
|
||||
# Quick text search first (faster than XML parsing)
|
||||
with open(nfo_path, 'r', encoding='utf-8', errors='ignore') as f:
|
||||
content = f.read()
|
||||
has_data = '<nfoguard_source>' in content or '<!-- NFOGuard -->' in content
|
||||
|
||||
# Cache result
|
||||
self._nfo_data_cache[cache_key] = (current_mtime, has_data)
|
||||
return has_data
|
||||
|
||||
except (OSError, UnicodeDecodeError) as e:
|
||||
_log("DEBUG", f"Error checking NFOGuard markers in {nfo_path}: {e}")
|
||||
return False
|
||||
|
||||
def _cleanup_cache(self):
|
||||
"""Clean up caches when they get too large"""
|
||||
if len(self._xml_tree_cache) > self.max_cache_size:
|
||||
# Remove oldest 25% of entries
|
||||
remove_count = self.max_cache_size // 4
|
||||
old_keys = list(self._xml_tree_cache.keys())[:remove_count]
|
||||
for key in old_keys:
|
||||
del self._xml_tree_cache[key]
|
||||
|
||||
if len(self._nfo_data_cache) > self.max_cache_size:
|
||||
remove_count = self.max_cache_size // 4
|
||||
old_keys = list(self._nfo_data_cache.keys())[:remove_count]
|
||||
for key in old_keys:
|
||||
del self._nfo_data_cache[key]
|
||||
|
||||
def clear_cache(self):
|
||||
"""Clear all XML caches"""
|
||||
self._xml_tree_cache.clear()
|
||||
self._nfo_data_cache.clear()
|
||||
_log("INFO", "XML processing cache cleared")
|
||||
|
||||
def get_cache_stats(self) -> Dict[str, Any]:
|
||||
"""Get XML cache performance statistics"""
|
||||
total_requests = self._cache_hits + self._cache_misses
|
||||
hit_rate = (self._cache_hits / total_requests * 100) if total_requests > 0 else 0
|
||||
|
||||
return {
|
||||
"xml_cache_hits": self._cache_hits,
|
||||
"xml_cache_misses": self._cache_misses,
|
||||
"xml_hit_rate_percent": round(hit_rate, 2),
|
||||
"xml_tree_cache_size": len(self._xml_tree_cache),
|
||||
"nfo_data_cache_size": len(self._nfo_data_cache)
|
||||
}
|
||||
|
||||
|
||||
# Global XML cache instance
|
||||
xml_cache = XMLProcessingCache()
|
||||
|
||||
|
||||
@lru_cache(maxsize=1000)
|
||||
def parse_imdb_from_filename(filename: str) -> Optional[str]:
|
||||
"""Parse IMDb ID from filename with LRU caching"""
|
||||
import re
|
||||
match = re.search(r'\[imdb-([^]]+)\]', filename, re.IGNORECASE)
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
@lru_cache(maxsize=200)
|
||||
def clean_title_for_search(title: str) -> str:
|
||||
"""Clean title for searching with LRU caching"""
|
||||
return title.lower().replace(" ", "").replace("-", "").replace("_", "")
|
||||
|
||||
|
||||
def clear_xml_caches():
|
||||
"""Clear all XML-related caches"""
|
||||
xml_cache.clear_cache()
|
||||
parse_imdb_from_filename.cache_clear()
|
||||
clean_title_for_search.cache_clear()
|
||||
_log("INFO", "All XML caches cleared")
|
||||
@@ -13,6 +13,7 @@ from core.database import NFOGuardDatabase
|
||||
from core.nfo_manager import NFOManager
|
||||
from core.path_mapper import PathMapper
|
||||
from core.logging import _log
|
||||
from core.fs_cache import fs_cache, extract_imdb_from_path
|
||||
from clients.radarr_client import RadarrClient
|
||||
from clients.external_clients import ExternalClientManager
|
||||
from config.settings import config
|
||||
|
||||
@@ -14,6 +14,7 @@ from core.database import NFOGuardDatabase
|
||||
from core.nfo_manager import NFOManager
|
||||
from core.path_mapper import PathMapper
|
||||
from core.logging import _log
|
||||
from core.fs_cache import fs_cache, parse_episode_from_filename, extract_imdb_from_path
|
||||
from clients.sonarr_client import SonarrClient
|
||||
from clients.external_clients import ExternalClientManager
|
||||
from config.settings import config
|
||||
@@ -123,7 +124,10 @@ class TVProcessor:
|
||||
disk_episodes = {}
|
||||
video_exts = (".mkv", ".mp4", ".avi", ".mov", ".m4v")
|
||||
|
||||
for season_dir in series_path.iterdir():
|
||||
# Use cached directory listing
|
||||
season_dirs = fs_cache.get_directory_contents(series_path)
|
||||
|
||||
for season_dir in season_dirs:
|
||||
if not (season_dir.is_dir() and season_dir.name.lower().startswith(config.tv_season_dir_pattern)):
|
||||
continue
|
||||
|
||||
@@ -140,11 +144,14 @@ class TVProcessor:
|
||||
except (ValueError, IndexError):
|
||||
continue
|
||||
|
||||
for video_file in season_dir.iterdir():
|
||||
if video_file.is_file() and video_file.suffix.lower() in video_exts:
|
||||
match = re.search(r"S(\d{2})E(\d{2})", video_file.name, re.IGNORECASE)
|
||||
if match:
|
||||
file_season, file_episode = int(match.group(1)), int(match.group(2))
|
||||
# Use cached video file discovery
|
||||
video_files = fs_cache.find_video_files(season_dir)
|
||||
|
||||
for video_file in video_files:
|
||||
# Use cached episode parsing
|
||||
episode_info = parse_episode_from_filename(video_file.name)
|
||||
if episode_info:
|
||||
file_season, file_episode = episode_info
|
||||
key = (season_num, file_episode) # Use directory season number
|
||||
if key not in disk_episodes:
|
||||
disk_episodes[key] = []
|
||||
|
||||
Reference in New Issue
Block a user