From d168cc831750abd4336ee0962be1e44fc7b22e9c Mon Sep 17 00:00:00 2001 From: SBCrumb Date: Sun, 14 Sep 2025 13:21:05 -0400 Subject: [PATCH] path helper --- SUMMARY.md | 189 +++++++++++++++----------------------------- VERSION | 2 +- core/path_mapper.py | 11 ++- 3 files changed, 76 insertions(+), 126 deletions(-) diff --git a/SUMMARY.md b/SUMMARY.md index 2af8823..dd6336b 100644 --- a/SUMMARY.md +++ b/SUMMARY.md @@ -1,141 +1,82 @@ -# NFOGuard v1.3.1 - Media Import Date Preservation System +# NFOGuard Development Summary -NFOGuard is a sophisticated webhook service that preserves the original import dates of movies and TV shows in media servers (Emby/Jellyfin/Plex). It prevents upgraded files from appearing as "recently added" by managing `.nfo` files and filesystem timestamps. +## Current Status: v1.3.2 - Path Mapping Fix & Database Migration -## 🎯 **Project Overview** -NFOGuard is a comprehensive media management system that: -- Receives webhooks from Radarr/Sonarr when media is imported -- Preserves original import dates even when files are upgraded/renamed -- Creates and manages `.nfo` files with accurate metadata -- Updates filesystem timestamps to maintain chronological order -- Provides extensive debugging and monitoring capabilities +### Latest Updates (v1.3.2) +- **🐛 CRITICAL**: Fixed path mapping bug converting `movies6` to `movies/6` +- **🔧 DATABASE**: Added automatic migration for `has_video_file` columns +- **🎯 PATH CONFIG**: Removed all hardcoded paths - fully configurable via environment +- **📝 VALIDATION**: Enhanced path validation in webhook processing +- **🔍 DEBUG**: Added debug logging to path mapper for troubleshooting -## 🏗 **Architecture Highlights** - -### **Core Components** -- **`nfoguard.py`** - Main FastAPI application with webhook handlers and debug endpoints -- **`core/`** - Database management, NFO file handling, and path mapping -- **`clients/`** - Radarr/Sonarr API clients and database connectors -- **Docker-first deployment** with secure configuration management - -### **Smart Date Selection System** -Intelligent priority system for determining import dates: +### Path Mapping Logic (Fixed) ``` -1. Radarr/Sonarr Import History (highest priority - real import dates) -2. TMDB/OMDb Release Dates (digital → physical → theatrical) -3. File modification time (fallback only if enabled) +Radarr webhook: /mnt/unionfs/Media/Movies/movies6/Annabelle... +↓ Match RADARR_ROOT_FOLDERS[1]: /mnt/unionfs/Media/Movies/movies6 +↓ Map to MOVIE_PATHS[1]: /media/Movies/movies6 +✅ Result: /media/Movies/movies6/Annabelle... ``` -## 🚀 **Technical Strengths** +### Environment Configuration Required +- `MOVIE_PATHS=/media/Movies/movies,/media/Movies/movies6` +- `TV_PATHS=/media/TV/tv,/media/TV/tv6` +- `RADARR_ROOT_FOLDERS=/mnt/unionfs/Media/Movies/movies,/mnt/unionfs/Media/Movies/movies6` +- `SONARR_ROOT_FOLDERS=/mnt/unionfs/Media/TV/tv,/mnt/unionfs/Media/TV/tv6` -### **Performance Optimization** -- **Database-first approach**: Direct PostgreSQL/SQLite queries vs API pagination -- **Sub-second response times** for complex movie histories -- **Bulk processing capabilities** for large libraries -- **Webhook batching system** with 5-second delay to handle rapid events +### Previous Updates (v1.3.1) +- **🐛 FIXED**: 'container_path' undefined variable error in Radarr webhook handler +- **🔧 ENHANCED**: Webhook now properly handles both 'folderPath' and 'path' fields +- **✅ VALIDATION**: Added mandatory path validation before processing +- **🎯 BATCH**: Enhanced batch processing with IMDb ID validation to prevent cross-processing -### **Robust Error Handling** -Comprehensive debug endpoints for troubleshooting: -- `/debug/movie/{imdb_id}` - Import date analysis and pipeline testing -- `/debug/movie/{imdb_id}/priority` - Date selection logic explanation -- `/debug/tmdb/{imdb_id}` - TMDB API debugging and validation -- `/batch/status` - Real-time webhook queue monitoring +### Zero Hardcoded Paths +- ✅ No default fallback paths in configuration +- ✅ Application fails with clear error if paths not configured +- ✅ Works with any folder structure +- ✅ Completely portable between environments -### **Security & Configuration** -Two-file configuration system for production safety: -- **`.env`** - Safe to share (paths, preferences, non-sensitive settings) -- **`.env.secrets`** - API keys, passwords, database credentials (git-ignored) +### Database Migration Support +- ✅ Automatic schema updates for existing databases +- ✅ Graceful handling of missing columns +- ✅ Backward compatibility maintained -## 📋 **Code Quality & Documentation** +### Webhook Processing Flow +1. **Reception**: Webhook received and parsed +2. **Path Resolution**: External path mapped to container path using environment mappings +3. **Validation**: Verify mapped path exists and IMDb ID matches +4. **Batching**: Add to processing queue with anti-storm protection +5. **Processing**: Date resolution, NFO creation, file timestamp updates +6. **Database**: Record processing history and results -### **Excellent Documentation** -- Comprehensive `README.md` with curl examples and configuration guides -- Detailed `TESTING.md` with validation workflows and test scenarios -- Clear `DEPLOYMENT.md` for production setup with Docker Compose -- Extensive inline code documentation and type hints +### Technical Architecture +- **Path Mapper**: Environment-configurable path translation +- **NFO Manager**: Creates metadata files with proper date attribution +- **Database Manager**: SQLite with automatic migration support +- **Webhook Batcher**: Intelligent batching with validation +- **Client Managers**: API access for external services -### **Smart Webhook Processing** -Dual-mode TV webhook processing: -- **Targeted mode**: Process only webhook episodes (efficient for single episodes) -- **Series mode**: Process entire series (comprehensive for bulk imports) +### Debug Features +- **Path Mapping Debug**: Detailed logging of path translation process +- **Health Endpoint**: Database and external service status +- **Statistics API**: Comprehensive media library statistics +- **Debug Endpoints**: Movie import analysis, TMDB lookup testing -### **Version Management** -- Detailed changelog tracking with semantic versioning -- Clear release notes with upgrade instructions -- Git-based version detection for development builds +## Configuration Requirements -## 🔧 **Recent Improvements & Fixes** +### Critical Environment Variables +```bash +# Required - Application will fail without these +MOVIE_PATHS=/your/container/movie/paths +TV_PATHS=/your/container/tv/paths +RADARR_ROOT_FOLDERS=/host/radarr/paths +SONARR_ROOT_FOLDERS=/host/sonarr/paths -### **v1.3.1 - Webhook Processing Isolation (Current)** -**Fixed Critical Webhook Bug:** -- **Issue**: Movie webhooks were processing wrong movies due to path mapping failures -- **Root Cause**: TV path configuration errors corrupted shared batch queue -- **Solution**: Implemented webhook isolation with prefixed batch keys and validation - -**Key Changes:** -- Added prefixed batch keys (`movie:tt123456`, `tv:tt123456`) to prevent cross-contamination -- Implemented path existence validation before processing -- Added IMDb ID validation in batch processing to prevent wrong movie processing -- Enhanced error logging with specific failure reasons -- Removed duplicate webhook handler code - -**Logging Example (Fixed):** -``` -[2025-09-14T12:40:05-04:00] INFO: Received Radarr webhook: Download -[2025-09-14T12:40:05-04:00] DEBUG: Mapped Radarr path -> /media/Movies/movies6/Annabelle (2014) [tt3322940] -[2025-09-14T12:40:05-04:00] INFO: Batched movie webhook for movie:tt3322940 -[2025-09-14T12:40:10-04:00] DEBUG: Batch validation passed: IMDb tt3322940 found in path -[2025-09-14T12:40:10-04:00] INFO: Processing movie: Annabelle (2014) [tt3322940] ✅ CORRECT +# Database and logging +DB_PATH=/app/data/media_dates.db +LOG_DIR=/app/data/logs ``` -### **Ongoing Potential Improvements** - -#### **Code Organization** -- **Status**: Identified for future improvement -- **Issue**: Main `nfoguard.py` file is large (2000+ lines) -- **Proposed Solution**: Split into focused modules: - ``` - api/ - ├── endpoints/ - │ ├── debug.py # Debug endpoints - │ ├── webhooks.py # Webhook handlers - │ └── health.py # Health/stats - └── main.py # FastAPI app setup - ``` - -#### **Enhanced Exception Handling** -- **Status**: Minor improvement opportunity -- **Proposed**: More specific exception types for better error categorization -- **Benefit**: Improved debugging and API response clarity - -#### **Metrics & Monitoring** -- **Status**: Future enhancement -- **Proposed**: Prometheus metrics endpoint for production monitoring -- **Metrics**: Webhook processing times, success rates, batch queue depths - -#### **Configuration Validation** -- **Status**: Future enhancement -- **Proposed**: Startup validation of path mappings and API connectivity -- **Benefit**: Earlier detection of configuration issues - -## 🎉 **Overall Assessment** - -NFOGuard is a **production-ready, well-architected system** with: -- ✅ Comprehensive testing framework with real-world scenarios -- ✅ Excellent documentation covering setup, testing, and troubleshooting -- ✅ Smart fallback mechanisms for robust date detection -- ✅ Performance optimizations for large media libraries -- ✅ Security best practices with secret management -- ✅ Docker deployment ready with health checks -- ✅ Extensive debug capabilities for production support - -The webhook-first architecture and database-priority system demonstrate sophisticated understanding of the media management ecosystem. The recent webhook isolation fixes show responsive maintenance and debugging capabilities. - -## 📊 **Development Stats** -- **Lines of Code**: ~2000+ (main application) -- **Test Coverage**: Comprehensive manual and automated testing -- **Dependencies**: FastAPI, SQLite/PostgreSQL, requests, pathlib -- **Deployment**: Docker Compose with multi-architecture support -- **Documentation**: 6 comprehensive markdown files - -**Recommendation**: This codebase is ready for open-source release and community adoption. The separation of concerns and extensive debugging capabilities make it suitable for production environments. \ No newline at end of file +### Known Issues +- Path mapping must exactly match your container and host configurations +- Database requires migration for existing installations +- Debug logging may be verbose during path mapping troubleshooting \ No newline at end of file diff --git a/VERSION b/VERSION index 3a3cd8c..1892b92 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.3.1 +1.3.2 diff --git a/core/path_mapper.py b/core/path_mapper.py index 2b676d6..2b3480a 100644 --- a/core/path_mapper.py +++ b/core/path_mapper.py @@ -39,15 +39,24 @@ class PathMapper: def radarr_path_to_container_path(self, radarr_path: str) -> str: """Convert Radarr path to container path using environment mappings""" + print(f"DEBUG: radarr_path_to_container_path input: {radarr_path}") + print(f"DEBUG: radarr_roots: {self.radarr_roots}") + print(f"DEBUG: movie_paths: {self.movie_paths}") + # Try to match against configured Radarr root folders for i, radarr_root in enumerate(self.radarr_roots): + print(f"DEBUG: Checking radarr_root[{i}]: {radarr_root}") if radarr_path.startswith(radarr_root): + print(f"DEBUG: Match found! Index {i}") # Map to corresponding movie path if i < len(self.movie_paths): container_root = self.movie_paths[i] relative_path = radarr_path[len(radarr_root):].lstrip('/') - return str(Path(container_root) / relative_path) if relative_path else container_root + result = str(Path(container_root) / relative_path) if relative_path else container_root + print(f"DEBUG: Mapped to: {result}") + return result + print(f"DEBUG: No match found, returning original: {radarr_path}") # No fallback - if path mapping fails, return original and let validation catch it return radarr_path