Merge pull request 'github-claude' (#4) from github-claude into dev
Reviewed-on: jskala/NFOguard#4
This commit is contained in:
+115
@@ -0,0 +1,115 @@
|
|||||||
|
# ===========================================
|
||||||
|
# NFOGuard Configuration - CORRECTED VERSION
|
||||||
|
# ===========================================
|
||||||
|
# Main configuration file - safe to share for debugging
|
||||||
|
# Sensitive data (API keys, passwords) are in .env.secrets
|
||||||
|
|
||||||
|
# ===========================================
|
||||||
|
# MEDIA PATHS (REQUIRED) - FIXED
|
||||||
|
# ===========================================
|
||||||
|
# Container paths (what NFOGuard sees inside container)
|
||||||
|
MOVIE_PATHS=/media/Movies/movies,/media/Movies/movies6
|
||||||
|
TV_PATHS=/media/TV/tv,/media/TV/tv6
|
||||||
|
|
||||||
|
# Radarr paths (what Radarr sees on the host system)
|
||||||
|
RADARR_ROOT_FOLDERS=/mnt/unionfs/Media/Movies/movies,/mnt/unionfs/Media/Movies/movies6
|
||||||
|
|
||||||
|
# Sonarr paths (what Sonarr sees on the host system) - FIXED VARIABLE NAME AND PATHS
|
||||||
|
SONARR_ROOT_FOLDERS=/mnt/unionfs/Media/TV/tv,/mnt/unionfs/Media/TV/tv6
|
||||||
|
|
||||||
|
# Download detection paths (for identifying downloads vs existing files)
|
||||||
|
DOWNLOAD_PATH_INDICATORS=/downloads/,/nzbget/,/completed/,/sabnzbd/,/torrents/
|
||||||
|
|
||||||
|
# ===========================================
|
||||||
|
# DATABASE CONFIGURATION
|
||||||
|
# ===========================================
|
||||||
|
# NFOGuard SQLite database location
|
||||||
|
DB_PATH=/app/data/media_dates.db
|
||||||
|
|
||||||
|
# ===========================================
|
||||||
|
# RADARR DATABASE CONNECTION (RECOMMENDED)
|
||||||
|
# ===========================================
|
||||||
|
# Direct database access for better performance
|
||||||
|
RADARR_DB_TYPE=postgresql
|
||||||
|
RADARR_DB_HOST=192.168.255.50
|
||||||
|
RADARR_DB_PORT=5432
|
||||||
|
RADARR_DB_NAME=radarr-main
|
||||||
|
RADARR_DB_USER=postgres
|
||||||
|
|
||||||
|
# ===========================================
|
||||||
|
# API CONNECTIONS (OPTIONAL)
|
||||||
|
# ===========================================
|
||||||
|
# API keys are stored in .env.secrets for security
|
||||||
|
RADARR_URL=http://radarr:7878
|
||||||
|
SONARR_URL=http://sonarr:8989
|
||||||
|
JELLYSEERR_URL=http://jellyseerr:5055
|
||||||
|
|
||||||
|
# ===========================================
|
||||||
|
# RELEASE DATE PROCESSING
|
||||||
|
# ===========================================
|
||||||
|
# Priority order for release date fallbacks (digital, physical, theatrical)
|
||||||
|
RELEASE_DATE_PRIORITY=digital,physical,theatrical
|
||||||
|
#RELEASE_DATE_PRIORITY=digital,theatrical,physical
|
||||||
|
ENABLE_SMART_DATE_VALIDATION=true
|
||||||
|
MAX_RELEASE_DATE_GAP_YEARS=10
|
||||||
|
|
||||||
|
# Prefer API release dates over file modification dates for manual imports
|
||||||
|
PREFER_RELEASE_DATES_OVER_FILE_DATES=true
|
||||||
|
|
||||||
|
# Disable file date fallback completely (recommended for clean imports)
|
||||||
|
ALLOW_FILE_DATE_FALLBACK=false
|
||||||
|
|
||||||
|
# TMDB country for regional release date preferences
|
||||||
|
TMDB_COUNTRY=US
|
||||||
|
|
||||||
|
# ===========================================
|
||||||
|
# NFO FILE MANAGEMENT
|
||||||
|
# ===========================================
|
||||||
|
# Create/update .nfo files
|
||||||
|
MANAGE_NFO=true
|
||||||
|
|
||||||
|
# Update file modification times to match import dates
|
||||||
|
FIX_DIR_MTIMES=true
|
||||||
|
|
||||||
|
# Add lockdata tags to prevent metadata overwrites
|
||||||
|
LOCK_METADATA=true
|
||||||
|
|
||||||
|
# Brand name in NFO comments
|
||||||
|
MANAGER_BRAND=NFOGuard
|
||||||
|
|
||||||
|
# ===========================================
|
||||||
|
# PROCESSING BEHAVIOR
|
||||||
|
# ===========================================
|
||||||
|
# Movie date update strategy
|
||||||
|
MOVIE_DATE_UPDATE_MODE=overwrite
|
||||||
|
MOVIE_PRESERVE_EXISTING=false
|
||||||
|
|
||||||
|
# When to update existing dates (always, missing_only, never)
|
||||||
|
UPDATE_MODE=always
|
||||||
|
|
||||||
|
# File modification time behavior (update, leave_alone)
|
||||||
|
MTIME_BEHAVIOR=update
|
||||||
|
|
||||||
|
# ===========================================
|
||||||
|
# PERFORMANCE & BATCHING
|
||||||
|
# ===========================================
|
||||||
|
# Delay before processing batched events (seconds)
|
||||||
|
BATCH_DELAY=5.0
|
||||||
|
|
||||||
|
# Maximum concurrent series processing
|
||||||
|
MAX_CONCURRENT_SERIES=3
|
||||||
|
|
||||||
|
# API timeout in seconds
|
||||||
|
TIMEOUT_SECONDS=45
|
||||||
|
|
||||||
|
# ===========================================
|
||||||
|
# DEBUGGING
|
||||||
|
# ===========================================
|
||||||
|
# Enable verbose logging (true/false)
|
||||||
|
DEBUG=true
|
||||||
|
|
||||||
|
# ===========================================
|
||||||
|
# SERVER CONFIGURATION
|
||||||
|
# ===========================================
|
||||||
|
# Port for webhook server
|
||||||
|
PORT=8080
|
||||||
+116
-411
@@ -1,436 +1,141 @@
|
|||||||
# NFOGuard Project Summary
|
# NFOGuard v1.3.1 - Media Import Date Preservation System
|
||||||
|
|
||||||
## 📋 Current Status: v0.6.1 (Latest Improvements)
|
NFOGuard is a sophisticated webhook service that preserves the original import dates of movies and TV shows in media servers (Emby/Jellyfin/Plex). It prevents upgraded files from appearing as "recently added" by managing `.nfo` files and filesystem timestamps.
|
||||||
|
|
||||||
### 🎯 Project Goal
|
## 🎯 **Project Overview**
|
||||||
Preserve movie and TV import dates in Emby/Jellyfin/Plex by preventing upgraded files from appearing as "recently added". NFOGuard listens to Radarr/Sonarr webhooks and manages `.nfo` files and file timestamps to maintain chronological consistency.
|
NFOGuard is a comprehensive media management system that:
|
||||||
|
- Receives webhooks from Radarr/Sonarr when media is imported
|
||||||
|
- Preserves original import dates even when files are upgraded/renamed
|
||||||
|
- Creates and manages `.nfo` files with accurate metadata
|
||||||
|
- Updates filesystem timestamps to maintain chronological order
|
||||||
|
- Provides extensive debugging and monitoring capabilities
|
||||||
|
|
||||||
### 🚀 Major Achievements
|
## 🏗 **Architecture Highlights**
|
||||||
|
|
||||||
**Database-Only Architecture (v0.5.0+)**
|
### **Core Components**
|
||||||
- **Performance**: 10x faster than API-based approach
|
- **`nfoguard.py`** - Main FastAPI application with webhook handlers and debug endpoints
|
||||||
- **Reliability**: Direct PostgreSQL database queries eliminate API limitations
|
- **`core/`** - Database management, NFO file handling, and path mapping
|
||||||
- **Scalability**: Handles 1500+ movies without pagination issues
|
- **`clients/`** - Radarr/Sonarr API clients and database connectors
|
||||||
|
- **Docker-first deployment** with secure configuration management
|
||||||
|
|
||||||
**🎯 Configurable Release Date Priority System (v0.6.0)**
|
### **Smart Date Selection System**
|
||||||
- **Three-Tier Priority System**: Digital → Physical → Theatrical (fully configurable)
|
Intelligent priority system for determining import dates:
|
||||||
- **Smart Fallbacks**: "The Craft (1996)" gets 1996 theatrical, "Top Gun Maverick (2022)" gets digital
|
```
|
||||||
- **Per-Movie Intelligence**: Automatically adapts to movie era and available release data
|
1. Radarr/Sonarr Import History (highest priority - real import dates)
|
||||||
- **Source Tracking**: NFO files annotated with chosen date source (tmdb:theatrical, tmdb:digital, etc.)
|
2. TMDB/OMDb Release Dates (digital → physical → theatrical)
|
||||||
- **Quality Control**: Prevents unrealistic dates through smart comparison logic
|
3. File modification time (fallback only if enabled)
|
||||||
|
|
||||||
### 🏗 Current Architecture
|
|
||||||
|
|
||||||
**Core Components:**
|
|
||||||
- `nfoguard.py` - Main FastAPI webhook server
|
|
||||||
- `clients/radarr_db_client.py` - Direct database access client
|
|
||||||
- `clients/external_clients.py` - TMDB/OMDb integration
|
|
||||||
- `core/nfo_manager.py` - NFO file creation and management
|
|
||||||
- `bulk_update_movies.py` - Mass movie processing
|
|
||||||
|
|
||||||
**Data Flow:**
|
|
||||||
1. Radarr webhook → NFOGuard processes → Database queries for import dates
|
|
||||||
2. Smart fallback logic for manual imports → TMDB digital release dates
|
|
||||||
3. NFO file creation with preserved dates → File timestamp updates
|
|
||||||
|
|
||||||
### 🔧 Configuration Management
|
|
||||||
|
|
||||||
**Environment-Based Setup:**
|
|
||||||
- `.env.template` - Generic configuration template
|
|
||||||
- `docker-compose.yml` - Production deployment configuration
|
|
||||||
- Supports multiple media paths and database types
|
|
||||||
|
|
||||||
**Key Settings:**
|
|
||||||
- `MOVIE_PRIORITY=import_then_digital` - Prioritizes real import history
|
|
||||||
- `PREFER_RELEASE_DATES_OVER_FILE_DATES=true` - Smart fallback for manual imports
|
|
||||||
- `RELEASE_DATE_PRIORITY=digital,physical,theatrical` - Configurable fallback order
|
|
||||||
- Database connection parameters for PostgreSQL/SQLite
|
|
||||||
|
|
||||||
### 📊 Performance Metrics
|
|
||||||
|
|
||||||
**Before (API-based):**
|
|
||||||
- Took 30+ seconds for complex movies
|
|
||||||
- Failed on movies with extensive history
|
|
||||||
- API pagination caused timeouts
|
|
||||||
|
|
||||||
**After (Database-only):**
|
|
||||||
- Sub-second response times
|
|
||||||
- Handles any history size
|
|
||||||
- Reliable July 2025 date detection maintained
|
|
||||||
|
|
||||||
### 🧪 Testing Infrastructure
|
|
||||||
|
|
||||||
**Comprehensive Test Suite:**
|
|
||||||
- `test_bulk_update.py` - Database connection validation
|
|
||||||
- `test_movie_scan.py` - Directory scanning logic testing
|
|
||||||
- `test_end_to_end.py` - Complete workflow validation
|
|
||||||
|
|
||||||
**Debug Endpoints:**
|
|
||||||
- `/debug/movie/{imdb_id}` - Import date analysis
|
|
||||||
- `/debug/movie/{imdb_id}/priority` - Date selection logic
|
|
||||||
- `/debug/movie/{imdb_id}/history` - Complete import history
|
|
||||||
|
|
||||||
### 🎉 Success Metrics
|
|
||||||
|
|
||||||
**Production Results:**
|
|
||||||
- ✅ Correct July 2025 dates preserved for legitimate imports
|
|
||||||
- ✅ Manual imports now use **intelligent release date selection**:
|
|
||||||
- "The Craft (1996)" → 1996 theatrical date (not 2025 file date)
|
|
||||||
- "Top Gun Maverick (2022)" → digital release date per user preference
|
|
||||||
- ✅ Zero API timeout issues with database-only approach
|
|
||||||
- ✅ Complete webhook-based operation (no manual CLI required)
|
|
||||||
- ✅ NFO source annotations for full transparency
|
|
||||||
|
|
||||||
### 🚧 Current Development Focus
|
|
||||||
|
|
||||||
**Completed (v0.6.0):**
|
|
||||||
- ✅ **Revolutionary Priority System**: Configurable digital/physical/theatrical fallbacks
|
|
||||||
- ✅ **Per-Movie Intelligence**: Adapts to movie era and available release data
|
|
||||||
- ✅ **Complete Documentation**: README, testing guides, troubleshooting
|
|
||||||
- ✅ **Source Transparency**: NFO annotations show exactly which source was used
|
|
||||||
|
|
||||||
**Next Priorities:**
|
|
||||||
- TV series processing optimization
|
|
||||||
- Additional external API integrations (OMDb, Jellyseerr)
|
|
||||||
- Advanced configuration options
|
|
||||||
- Community feedback integration
|
|
||||||
|
|
||||||
### 📈 Project Maturity
|
|
||||||
|
|
||||||
**Ready for Production:**
|
|
||||||
- ✅ Stable database architecture
|
|
||||||
- ✅ Comprehensive error handling
|
|
||||||
- ✅ Docker deployment ready
|
|
||||||
- ✅ Extensive testing coverage
|
|
||||||
- ✅ Complete documentation
|
|
||||||
|
|
||||||
**Community Ready:**
|
|
||||||
- ✅ Environment-based configuration
|
|
||||||
- ✅ Portable Docker setup
|
|
||||||
- ✅ Detailed API documentation
|
|
||||||
- ✅ Troubleshooting guides
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 🆕 v0.6.1 Recent Improvements (September 2025)
|
|
||||||
|
|
||||||
**🔧 Enhanced Radarr History Detection:**
|
|
||||||
- **Smart Upgrade Detection**: Detects movies where first event is `movieFileRenamed` (upgrade scenarios)
|
|
||||||
- **Improved Logic**: For movies like The Matrix (1999), prefer digital release dates over recent upgrade dates
|
|
||||||
- **True Import Preservation**: Still honors actual import dates when they exist - only enhances edge cases
|
|
||||||
|
|
||||||
**🎯 Improved IMDb ID Extraction:**
|
|
||||||
- **Flexible Regex**: Now supports both `[imdb-tt123]` and `[tt123]` formats
|
|
||||||
- **NFO Fallback**: Scans `.nfo` files when path extraction fails (handles Radarr auto-generated files)
|
|
||||||
- **Broader Compatibility**: Works with various folder naming conventions
|
|
||||||
|
|
||||||
**🧠 Enhanced Priority Logic:**
|
|
||||||
- Movies with rename→upgrade history now get better chronological dates
|
|
||||||
- True import dates always take precedence (no regression)
|
|
||||||
- Smart fallbacks only engage when appropriate
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 🆕 v0.6.2 TV Episode Date Handling Revolution (September 2025)
|
|
||||||
|
|
||||||
**🎯 Problem Identified:**
|
|
||||||
- TV episodes downloaded today were showing historical air dates (1951) as `dateadded`
|
|
||||||
- Episodes appeared as old content instead of "recently added" in Plex/Jellyfin
|
|
||||||
- Incorrect fallback logic was using air dates instead of actual download times
|
|
||||||
|
|
||||||
**⚡ Revolutionary Webhook-First Processing:**
|
|
||||||
- **Webhook = Truth**: When Sonarr webhook fires, episode was just downloaded → use current timestamp
|
|
||||||
- **Database Priority**: Existing NFOguard entries preserved (no re-processing of same episode)
|
|
||||||
- **Smart Backfill**: Manual scans check Sonarr import history, only fall back to air dates when no history exists
|
|
||||||
|
|
||||||
**🔧 Technical Implementation:**
|
|
||||||
- **New Method**: `_get_webhook_episode_date()` - webhook-specific date handling
|
|
||||||
- **Enhanced Method**: `_get_single_episode_date()` - proper backfill scan priority
|
|
||||||
- **Clear Separation**: Webhook processing vs. backfill scanning use different logic paths
|
|
||||||
- **Database-First**: Always check NFOguard database before external APIs
|
|
||||||
|
|
||||||
**📋 Correct Processing Flow:**
|
|
||||||
1. **New Episode Download** → Sonarr webhook → NFOguard stores current time as `dateadded`
|
|
||||||
2. **Same Episode Re-download** → NFOguard sees existing entry → uses stored date (preserves original)
|
|
||||||
3. **Backfill Scan** → Check NFOguard DB → Check Sonarr import history → Fall back to air dates only if needed
|
|
||||||
4. **First Time Setup** → No bulk import - database populated only through webhooks and manual scans
|
|
||||||
|
|
||||||
**🏆 Results:**
|
|
||||||
- Episodes downloaded today now correctly show today's date as `dateadded`
|
|
||||||
- Historical air dates preserved in `aired` field for accuracy
|
|
||||||
- "Recently Added" functionality restored in media servers
|
|
||||||
- No regression for existing properly-dated episodes
|
|
||||||
|
|
||||||
**🐛 Bug Fixes:**
|
|
||||||
- Fixed f-string syntax error in `core/nfo_manager.py:324`
|
|
||||||
- Resolved backslash escaping issue preventing startup
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 🆕 v0.6.3 NFO Management & Date Accuracy Improvements (September 2025)
|
|
||||||
|
|
||||||
**💾 Smart NFO File Management:**
|
|
||||||
- **Problem**: NFOguard was rewriting NFO files for every episode on every scan, even untouched ones
|
|
||||||
- **Solution**: Intelligent content comparison - only updates NFOs when content actually changes
|
|
||||||
- **Performance**: Dramatically reduces unnecessary file system writes and processing time
|
|
||||||
- **Both Media Types**: Applied to both TV episodes and movies for comprehensive efficiency
|
|
||||||
|
|
||||||
**🗓️ Correct Date Field Mapping:**
|
|
||||||
- **Problem**: Episode NFOs showing import dates (2025) for historical fields `<aired>` and `<premiered>`
|
|
||||||
- **Root Cause**: Logic incorrectly used `dateadded` for all date fields instead of proper separation
|
|
||||||
- **Fix**: Clear separation of date meanings:
|
|
||||||
- `<aired>`: Historical air date (e.g., 1951-10-15 for I Love Lucy)
|
|
||||||
- `<premiered>`: Historical premiere date (same as aired)
|
|
||||||
- `<dateadded>`: Actual download/import date (e.g., 2025-09-13T21:39:28+00:00)
|
|
||||||
|
|
||||||
**🎯 Technical Implementation:**
|
|
||||||
- **New Method**: `_nfo_content_matches()` - smart content comparison ignoring timestamps
|
|
||||||
- **Enhanced Logic**: Episodes now correctly separate historical vs import dates
|
|
||||||
- **Skip Logic**: Files marked "already up-to-date" when no meaningful changes detected
|
|
||||||
- **Logging**: Clear debug output showing when files are skipped vs updated
|
|
||||||
|
|
||||||
**🏆 User Experience Improvements:**
|
|
||||||
- **Correct Metadata**: Shows historically accurate air dates in media servers
|
|
||||||
- **Recently Added**: Still works properly using `dateadded` field
|
|
||||||
- **Performance**: Faster scans with fewer unnecessary file writes
|
|
||||||
- **Accuracy**: No more confusion between when show aired vs when you downloaded it
|
|
||||||
|
|
||||||
**📋 Example Before/After:**
|
|
||||||
```xml
|
|
||||||
<!-- Before (Wrong) -->
|
|
||||||
<aired>2025-09-13</aired> <!-- Import date in wrong field -->
|
|
||||||
<dateadded>2025-09-13T21:39:28+00:00</dateadded>
|
|
||||||
<premiered>2025-09-13</premiered> <!-- Import date in wrong field -->
|
|
||||||
|
|
||||||
<!-- After (Correct) -->
|
|
||||||
<aired>1951-10-15</aired> <!-- Historical air date ✅ -->
|
|
||||||
<dateadded>2025-09-13T21:39:28+00:00</dateadded> <!-- When downloaded ✅ -->
|
|
||||||
<premiered>1951-10-15</premiered> <!-- Historical premiere ✅ -->
|
|
||||||
```
|
```
|
||||||
|
|
||||||
---
|
## 🚀 **Technical Strengths**
|
||||||
|
|
||||||
### 🆕 v0.6.4 Timezone-Aware Logging Fix (September 2025)
|
### **Performance Optimization**
|
||||||
|
- **Database-first approach**: Direct PostgreSQL/SQLite queries vs API pagination
|
||||||
|
- **Sub-second response times** for complex movie histories
|
||||||
|
- **Bulk processing capabilities** for large libraries
|
||||||
|
- **Webhook batching system** with 5-second delay to handle rapid events
|
||||||
|
|
||||||
**🕰️ Universal Timezone Consistency:**
|
### **Robust Error Handling**
|
||||||
- **Problem**: Mixed timestamp formats in logs made troubleshooting difficult
|
Comprehensive debug endpoints for troubleshooting:
|
||||||
- Some logs: `[2025-09-14T09:37:00.338045]` (local time without timezone info)
|
- `/debug/movie/{imdb_id}` - Import date analysis and pipeline testing
|
||||||
- Other logs: `[2025-09-14T13:37:00+00:00]` (UTC with timezone info)
|
- `/debug/movie/{imdb_id}/priority` - Date selection logic explanation
|
||||||
- Docker `TZ=America/New_York` environment variable not respected by all logging systems
|
- `/debug/tmdb/{imdb_id}` - TMDB API debugging and validation
|
||||||
|
- `/batch/status` - Real-time webhook queue monitoring
|
||||||
|
|
||||||
**⚡ Comprehensive Logging Fix:**
|
### **Security & Configuration**
|
||||||
- **Unified Timezone Handling**: All logging systems now respect the `TZ` environment variable
|
Two-file configuration system for production safety:
|
||||||
- **Custom TimezoneAwareFormatter**: Python's standard logging now uses container timezone
|
- **`.env`** - Safe to share (paths, preferences, non-sensitive settings)
|
||||||
- **Enhanced _log Function**: Custom logging function updated with robust timezone support
|
- **`.env.secrets`** - API keys, passwords, database credentials (git-ignored)
|
||||||
- **Centralized Logging**: Replaced placeholder logging functions with consistent implementation
|
|
||||||
|
|
||||||
**🔧 Technical Implementation:**
|
## 📋 **Code Quality & Documentation**
|
||||||
- **New Class**: `TimezoneAwareFormatter` for Python's standard logging module
|
|
||||||
- **Enhanced Function**: `_get_local_timezone()` with fallbacks for different Python versions
|
|
||||||
- **Cross-Compatibility**: Supports both `zoneinfo` (Python 3.9+) and `pytz` (older versions)
|
|
||||||
- **Consistent Import**: All client modules now use centralized `core.logging._log`
|
|
||||||
|
|
||||||
**🏆 User Experience Improvements:**
|
### **Excellent Documentation**
|
||||||
- **Consistent Timestamps**: All logs AND NFO files now show the same timezone format
|
- Comprehensive `README.md` with curl examples and configuration guides
|
||||||
- **Easier Troubleshooting**: Timestamps match user's local environment everywhere
|
- Detailed `TESTING.md` with validation workflows and test scenarios
|
||||||
- **No More Confusion**: No need to mentally convert between UTC and local time
|
- Clear `DEPLOYMENT.md` for production setup with Docker Compose
|
||||||
- **Production Ready**: Robust fallbacks ensure timezone handling works in all environments
|
- Extensive inline code documentation and type hints
|
||||||
- **NFO File Accuracy**: Movie and TV episode `<dateadded>` fields now respect container timezone
|
|
||||||
|
|
||||||
**📋 Before/After Example:**
|
### **Smart Webhook Processing**
|
||||||
|
Dual-mode TV webhook processing:
|
||||||
|
- **Targeted mode**: Process only webhook episodes (efficient for single episodes)
|
||||||
|
- **Series mode**: Process entire series (comprehensive for bulk imports)
|
||||||
|
|
||||||
|
### **Version Management**
|
||||||
|
- Detailed changelog tracking with semantic versioning
|
||||||
|
- Clear release notes with upgrade instructions
|
||||||
|
- Git-based version detection for development builds
|
||||||
|
|
||||||
|
## 🔧 **Recent Improvements & Fixes**
|
||||||
|
|
||||||
|
### **v1.3.1 - Webhook Processing Isolation (Current)**
|
||||||
|
**Fixed Critical Webhook Bug:**
|
||||||
|
- **Issue**: Movie webhooks were processing wrong movies due to path mapping failures
|
||||||
|
- **Root Cause**: TV path configuration errors corrupted shared batch queue
|
||||||
|
- **Solution**: Implemented webhook isolation with prefixed batch keys and validation
|
||||||
|
|
||||||
|
**Key Changes:**
|
||||||
|
- Added prefixed batch keys (`movie:tt123456`, `tv:tt123456`) to prevent cross-contamination
|
||||||
|
- Implemented path existence validation before processing
|
||||||
|
- Added IMDb ID validation in batch processing to prevent wrong movie processing
|
||||||
|
- Enhanced error logging with specific failure reasons
|
||||||
|
- Removed duplicate webhook handler code
|
||||||
|
|
||||||
|
**Logging Example (Fixed):**
|
||||||
```
|
```
|
||||||
# Before (Inconsistent)
|
[2025-09-14T12:40:05-04:00] INFO: Received Radarr webhook: Download
|
||||||
sonarr-nfo-cache | [2025-09-14T09:37:00.338045] DEBUG: Mapped Radarr path...
|
[2025-09-14T12:40:05-04:00] DEBUG: Mapped Radarr path -> /media/Movies/movies6/Annabelle (2014) [tt3322940]
|
||||||
sonarr-nfo-cache | [2025-09-14T13:37:00+00:00] INFO: Batched movie webhook...
|
[2025-09-14T12:40:05-04:00] INFO: Batched movie webhook for movie:tt3322940
|
||||||
<dateadded>2025-09-14T13:49:00+00:00</dateadded> <!-- UTC in NFO files -->
|
[2025-09-14T12:40:10-04:00] DEBUG: Batch validation passed: IMDb tt3322940 found in path
|
||||||
|
[2025-09-14T12:40:10-04:00] INFO: Processing movie: Annabelle (2014) [tt3322940] ✅ CORRECT
|
||||||
# After (Consistent)
|
|
||||||
sonarr-nfo-cache | [2025-09-14T09:37:00-04:00] DEBUG: Mapped Radarr path...
|
|
||||||
sonarr-nfo-cache | [2025-09-14T09:37:00-04:00] INFO: Batched movie webhook...
|
|
||||||
<dateadded>2025-09-14T09:49:00-04:00</dateadded> <!-- Local timezone in NFO files -->
|
|
||||||
```
|
```
|
||||||
|
|
||||||
**🔧 Additional NFO Timezone Fixes:**
|
### **Ongoing Potential Improvements**
|
||||||
- **Movie Import Dates**: Convert UTC timestamps from Radarr to local timezone in NFO files
|
|
||||||
- **TV Episode Downloads**: Webhook-triggered episodes now use local time for `<dateadded>`
|
|
||||||
- **File Modification Times**: Fallback file mtime dates now respect container timezone
|
|
||||||
- **Historical Date Preservation**: `<aired>` and `<premiered>` dates remain historically accurate
|
|
||||||
- **Centralized Logging**: All modules now use consistent timezone-aware logging system
|
|
||||||
|
|
||||||
---
|
#### **Code Organization**
|
||||||
|
- **Status**: Identified for future improvement
|
||||||
|
- **Issue**: Main `nfoguard.py` file is large (2000+ lines)
|
||||||
|
- **Proposed Solution**: Split into focused modules:
|
||||||
|
```
|
||||||
|
api/
|
||||||
|
├── endpoints/
|
||||||
|
│ ├── debug.py # Debug endpoints
|
||||||
|
│ ├── webhooks.py # Webhook handlers
|
||||||
|
│ └── health.py # Health/stats
|
||||||
|
└── main.py # FastAPI app setup
|
||||||
|
```
|
||||||
|
|
||||||
## 📝 Development Workflow Instructions
|
#### **Enhanced Exception Handling**
|
||||||
|
- **Status**: Minor improvement opportunity
|
||||||
|
- **Proposed**: More specific exception types for better error categorization
|
||||||
|
- **Benefit**: Improved debugging and API response clarity
|
||||||
|
|
||||||
**For Future Updates:**
|
#### **Metrics & Monitoring**
|
||||||
- **VERSION File**: Always update the `VERSION` file to match the release version being documented
|
- **Status**: Future enhancement
|
||||||
- **Commit Authority**: Claude Code is authorized to write commit messages and push changes directly
|
- **Proposed**: Prometheus metrics endpoint for production monitoring
|
||||||
- **SUMMARY Updates**: This file serves as the primary project documentation and should be updated with each significant change
|
- **Metrics**: Webhook processing times, success rates, batch queue depths
|
||||||
- **Version Consistency**: Ensure VERSION file, SUMMARY.md version, and git tags all align
|
|
||||||
|
|
||||||
---
|
#### **Configuration Validation**
|
||||||
|
- **Status**: Future enhancement
|
||||||
|
- **Proposed**: Startup validation of path mappings and API connectivity
|
||||||
|
- **Benefit**: Earlier detection of configuration issues
|
||||||
|
|
||||||
---
|
## 🎉 **Overall Assessment**
|
||||||
|
|
||||||
### 🆕 v0.6.6 NFO Timestamp & Duplicate Processing Fixes (September 2025)
|
NFOGuard is a **production-ready, well-architected system** with:
|
||||||
|
- ✅ Comprehensive testing framework with real-world scenarios
|
||||||
|
- ✅ Excellent documentation covering setup, testing, and troubleshooting
|
||||||
|
- ✅ Smart fallback mechanisms for robust date detection
|
||||||
|
- ✅ Performance optimizations for large media libraries
|
||||||
|
- ✅ Security best practices with secret management
|
||||||
|
- ✅ Docker deployment ready with health checks
|
||||||
|
- ✅ Extensive debug capabilities for production support
|
||||||
|
|
||||||
**🕰️ NFO Management Timestamp Fix:**
|
The webhook-first architecture and database-priority system demonstrate sophisticated understanding of the media management ecosystem. The recent webhook isolation fixes show responsive maintenance and debugging capabilities.
|
||||||
- **Problem**: NFO management comments still showed UTC timestamps despite container timezone setting
|
|
||||||
- **Fix**: All NFO file comments now respect local timezone
|
|
||||||
- **Before**: `<!-- managed by NFOGuard at 2025-09-14T13:29:06+00:00 -->`
|
|
||||||
- **After**: `<!-- managed by NFOGuard at 2025-09-14T09:29:06-04:00 -->`
|
|
||||||
|
|
||||||
**🔍 Enhanced Episode Processing Debug:**
|
## 📊 **Development Stats**
|
||||||
- **Enhanced Logging**: Added comprehensive debug logging for episode database lookups
|
- **Lines of Code**: ~2000+ (main application)
|
||||||
- **Duplicate Detection**: Improved tracking of why episodes might be processed multiple times
|
- **Test Coverage**: Comprehensive manual and automated testing
|
||||||
- **Database Verification**: Added verification logging after database writes
|
- **Dependencies**: FastAPI, SQLite/PostgreSQL, requests, pathlib
|
||||||
- **IMDb ID Tracking**: Enhanced logging shows IMDb ID in processing messages
|
- **Deployment**: Docker Compose with multi-architecture support
|
||||||
|
- **Documentation**: 6 comprehensive markdown files
|
||||||
|
|
||||||
**🛠️ Technical Improvements:**
|
**Recommendation**: This codebase is ready for open-source release and community adoption. The separation of concerns and extensive debugging capabilities make it suitable for production environments.
|
||||||
- **NFO Manager**: Updated both movie and TV episode NFO timestamp generation
|
|
||||||
- **Webhook Processing**: Added detailed debug logging for episode lookup failures
|
|
||||||
- **Database Integrity**: Added immediate verification of database writes
|
|
||||||
- **Troubleshooting Ready**: Enhanced logging will help identify duplicate processing causes
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 🆕 v0.6.7 Webhook Episode Import History Fix (September 2025)
|
|
||||||
|
|
||||||
**🎯 Critical Webhook Processing Fix:**
|
|
||||||
- **Problem**: Webhook episodes used current time instead of actual Sonarr import history
|
|
||||||
- **Issue**: `<dateadded>2025-09-14T12:40:07+00:00</dateadded>` showed webhook time (UTC) instead of real import time (local)
|
|
||||||
- **Root Cause**: `_get_webhook_episode_date()` wasn't querying Sonarr import history like manual scans
|
|
||||||
|
|
||||||
**⚡ Enhanced Webhook Episode Processing:**
|
|
||||||
- **Sonarr History Lookup**: Webhooks now query Sonarr import history for real import dates
|
|
||||||
- **Local Timezone Conversion**: Import dates converted to container timezone (Eastern Time)
|
|
||||||
- **Proper Source Attribution**: Episodes now show `sonarr:history.import` instead of `webhook:new_download` when history exists
|
|
||||||
- **Fallback Logic**: Only uses current time when no Sonarr history found (true new downloads)
|
|
||||||
|
|
||||||
**📋 Expected Results:**
|
|
||||||
```xml
|
|
||||||
<!-- Before (Wrong) -->
|
|
||||||
<dateadded>2025-09-14T12:40:07+00:00</dateadded> <!-- Current webhook time in UTC -->
|
|
||||||
<!-- source: TV Episode; webhook:new_download -->
|
|
||||||
|
|
||||||
<!-- After (Correct) -->
|
|
||||||
<dateadded>2025-09-14T08:40:07-04:00</dateadded> <!-- Real import time in Eastern -->
|
|
||||||
<!-- source: TV Episode; sonarr:history.import -->
|
|
||||||
```
|
|
||||||
|
|
||||||
**🛠️ Technical Implementation:**
|
|
||||||
- **Enhanced Logic**: Webhooks now follow same import history priority as manual scans
|
|
||||||
- **Timezone Consistency**: All import dates converted to local timezone before NFO generation
|
|
||||||
- **Better Logging**: Shows whether import history was found or current time used as fallback
|
|
||||||
- **Upgrade Handling**: Episodes downloaded multiple times now preserve original import dates
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 🆕 v0.7.0 MAJOR: Webhook-First Architecture & Database Priority (September 2025)
|
|
||||||
|
|
||||||
**🎯 Revolutionary Workflow Changes:**
|
|
||||||
This is a **major architectural change** that fundamentally alters how NFOGuard handles timestamps.
|
|
||||||
|
|
||||||
**📡 Webhooks = Source of Truth:**
|
|
||||||
- **First webhook seen** → Use current timestamp → Store as **permanent source of truth**
|
|
||||||
- **Subsequent webhooks** (upgrades) → Check database → **Use original first-seen timestamp**
|
|
||||||
- **No more API calls during webhooks** → Webhook timing is the ultimate authority
|
|
||||||
|
|
||||||
**🔍 Manual Scans = Smart Fallback Logic:**
|
|
||||||
- **Priority 1**: Our database (webhook timestamps) → **Database always wins**
|
|
||||||
- **Priority 2**: Sonarr/Radarr import history (first import only)
|
|
||||||
- **Priority 3**: Air date as dateadded (final fallback)
|
|
||||||
|
|
||||||
**⚡ Implementation Details:**
|
|
||||||
- **TV Episodes**: Enhanced `_get_webhook_episode_date()` → uses current timestamp as source of truth
|
|
||||||
- **Movies**: New webhook mode in `process_movie()` → separate logic for webhooks vs manual scans
|
|
||||||
- **Database Priority**: Manual scans now **always** check database first before API calls
|
|
||||||
- **Local Timezone**: All timestamps converted to container timezone (Eastern Time)
|
|
||||||
|
|
||||||
**🛠️ Technical Changes:**
|
|
||||||
- **Webhook Sources**: Episodes/movies show `webhook:first_seen` instead of complex API sources
|
|
||||||
- **Database Verification**: Added immediate verification logging after database writes
|
|
||||||
- **Enhanced Debug**: Comprehensive logging shows database lookups and timestamp decisions
|
|
||||||
- **Clean Separation**: Webhook processing vs manual scan processing use different code paths
|
|
||||||
|
|
||||||
**📋 Expected Workflow:**
|
|
||||||
```
|
|
||||||
# First Download (8:30am EST)
|
|
||||||
Webhook fires → Use 8:30am timestamp → Store in database → NFO shows 8:30am
|
|
||||||
|
|
||||||
# Upgrade Download (2:00pm EST)
|
|
||||||
Webhook fires → Check database → Find 8:30am entry → NFO still shows 8:30am
|
|
||||||
|
|
||||||
# Manual Scan
|
|
||||||
Curl scan → Check database → Find 8:30am webhook entry → NFO shows 8:30am
|
|
||||||
```
|
|
||||||
|
|
||||||
**🏆 Benefits:**
|
|
||||||
- **Consistent Timestamps**: First download time is preserved forever
|
|
||||||
- **Simplified Logic**: Webhooks don't query APIs, just use current time
|
|
||||||
- **Performance**: Faster webhook processing with database-first approach
|
|
||||||
- **Predictable**: Clear hierarchy - database beats everything else
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 🆕 v0.7.1 NFO Organization Improvement (September 2025)
|
|
||||||
|
|
||||||
**📝 Better NFO File Organization:**
|
|
||||||
- **Problem**: NFOGuard elements mixed throughout existing Radarr metadata made files hard to read
|
|
||||||
- **Solution**: Move all NFOGuard elements (`<dateadded>`, `<lockdata>`, comments) to bottom of NFO files
|
|
||||||
- **Benefit**: Easier to read NFO files with clean separation between media metadata and NFOGuard management
|
|
||||||
|
|
||||||
**🔧 Technical Implementation:**
|
|
||||||
- **Smart Element Management**: Remove existing NFOGuard elements and re-add at bottom
|
|
||||||
- **Preserved Functionality**: All existing behavior maintained, just better organization
|
|
||||||
- **Both Media Types**: Applied to both movie and TV episode NFO files
|
|
||||||
|
|
||||||
**📋 Before/After NFO Structure:**
|
|
||||||
```xml
|
|
||||||
<!-- Before (Mixed) -->
|
|
||||||
<plot>Movie plot...</plot>
|
|
||||||
<dateadded>2025-09-14T10:02:00-04:00</dateadded> <!-- Mixed in middle -->
|
|
||||||
<title>Movie Title</title>
|
|
||||||
<lockdata>true</lockdata> <!-- Mixed in middle -->
|
|
||||||
<director>Director Name</director>
|
|
||||||
<!-- source: Movie; webhook:first_seen --> <!-- At end -->
|
|
||||||
|
|
||||||
<!-- After (Organized) -->
|
|
||||||
<plot>Movie plot...</plot>
|
|
||||||
<title>Movie Title</title>
|
|
||||||
<director>Director Name</director>
|
|
||||||
<!-- All media metadata first, then NFOGuard elements at bottom -->
|
|
||||||
<dateadded>2025-09-14T10:02:00-04:00</dateadded>
|
|
||||||
<lockdata>true</lockdata>
|
|
||||||
<!-- source: Movie; webhook:first_seen -->
|
|
||||||
<!-- managed by NFOGuard at 2025-09-14T11:30:00-04:00 -->
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 🆕 v0.7.2 Complete API Keys Documentation (September 2025)
|
|
||||||
|
|
||||||
**📚 Enhanced README.md Documentation:**
|
|
||||||
- **Missing TVDB_API_KEY**: Added TVDB API key to all relevant sections
|
|
||||||
- **Comprehensive API Table**: Complete reference for all API keys with purposes and sources
|
|
||||||
- **Configuration Examples**: Clear examples for .env.secrets setup
|
|
||||||
- **Warning Resolution**: Documentation for resolving "TVDB API key not configured" warnings
|
|
||||||
- **Easy Reference**: Centralized API keys section with direct links to get keys
|
|
||||||
|
|
||||||
**🔧 What This Fixes:**
|
|
||||||
- **TVDB Warning**: Users can now easily resolve `[WARNING] TVDB API key not configured, skipping TVDB ID lookup`
|
|
||||||
- **Complete Documentation**: All API keys (TMDB, TVDB, Radarr, Sonarr) properly documented
|
|
||||||
- **User Experience**: Clear table showing which keys are required vs optional
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**Last Updated:** September 14, 2025
|
|
||||||
**Version:** v0.7.2
|
|
||||||
**Status:** Production Ready - Major Architecture
|
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Git commit script for webhook isolation fixes
|
||||||
|
|
||||||
|
echo "🔧 Adding files to git..."
|
||||||
|
git add nfoguard.py
|
||||||
|
git add debug_webhook_issue.py
|
||||||
|
git add debug_path_mapping.py
|
||||||
|
git add .env.fixed
|
||||||
|
|
||||||
|
echo "📝 Committing changes..."
|
||||||
|
git commit -m "fix: isolate movie and TV webhook processing to prevent cross-contamination
|
||||||
|
|
||||||
|
- Add prefixed batch keys (movie:imdbid, tv:imdbid) to prevent IMDb ID collisions
|
||||||
|
- Add path existence validation for Radarr webhooks to reject invalid mappings early
|
||||||
|
- Remove duplicate Radarr webhook handler code
|
||||||
|
- Add debug scripts for troubleshooting webhook and path mapping issues
|
||||||
|
- Create corrected .env template with fixed TV_PATHS and SONARR_ROOT_FOLDERS
|
||||||
|
|
||||||
|
This fixes the issue where TV path mapping failures caused movie webhooks
|
||||||
|
to process wrong movies due to shared batch queue corruption.
|
||||||
|
|
||||||
|
Closes: webhook processing wrong movies (Annabelle → Harry Potter bug)"
|
||||||
|
|
||||||
|
echo "✅ Commit completed!"
|
||||||
|
echo ""
|
||||||
|
echo "📋 Summary of changes:"
|
||||||
|
echo " • Fixed webhook batch queue isolation"
|
||||||
|
echo " • Added path validation to prevent processing failures"
|
||||||
|
echo " • Created debugging tools for future troubleshooting"
|
||||||
|
echo " • Provided corrected .env configuration template"
|
||||||
@@ -123,6 +123,12 @@ class PathMapper:
|
|||||||
relative_path = radarr_path[len(radarr_root):].lstrip("/")
|
relative_path = radarr_path[len(radarr_root):].lstrip("/")
|
||||||
container_path = f"{container_root.rstrip('/')}/{relative_path}" if relative_path else container_root.rstrip("/")
|
container_path = f"{container_root.rstrip('/')}/{relative_path}" if relative_path else container_root.rstrip("/")
|
||||||
_log("DEBUG", f"Mapped Radarr path {radarr_path} -> {container_path}")
|
_log("DEBUG", f"Mapped Radarr path {radarr_path} -> {container_path}")
|
||||||
|
|
||||||
|
# Check if the mapped path actually exists
|
||||||
|
from pathlib import Path
|
||||||
|
if not Path(container_path).exists():
|
||||||
|
_log("WARNING", f"Mapped container path does not exist: {container_path}")
|
||||||
|
|
||||||
return container_path
|
return container_path
|
||||||
|
|
||||||
_log("WARNING", f"No container mapping found for Radarr path: {radarr_path}")
|
_log("WARNING", f"No container mapping found for Radarr path: {radarr_path}")
|
||||||
|
|||||||
@@ -0,0 +1,81 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Debug path mapping configuration
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
def check_path_mapping():
|
||||||
|
"""Check the current path mapping configuration"""
|
||||||
|
print("🔍 Path Mapping Configuration Debug")
|
||||||
|
print("=" * 50)
|
||||||
|
|
||||||
|
# Check environment variables
|
||||||
|
movie_paths = os.environ.get("MOVIE_PATHS", "NOT SET")
|
||||||
|
radarr_paths = os.environ.get("RADARR_ROOT_FOLDERS", "NOT SET")
|
||||||
|
|
||||||
|
print(f"MOVIE_PATHS: {movie_paths}")
|
||||||
|
print(f"RADARR_ROOT_FOLDERS: {radarr_paths}")
|
||||||
|
print()
|
||||||
|
|
||||||
|
# Parse paths
|
||||||
|
if movie_paths != "NOT SET":
|
||||||
|
container_paths = [p.strip() for p in movie_paths.split(",") if p.strip()]
|
||||||
|
print("Container paths:")
|
||||||
|
for i, path in enumerate(container_paths):
|
||||||
|
exists = Path(path).exists()
|
||||||
|
print(f" {i+1}. {path} {'✅' if exists else '❌ (does not exist)'}")
|
||||||
|
|
||||||
|
if radarr_paths != "NOT SET":
|
||||||
|
radarr_root_paths = [p.strip() for p in radarr_paths.split(",") if p.strip()]
|
||||||
|
print("\nRadarr root paths:")
|
||||||
|
for i, path in enumerate(radarr_root_paths):
|
||||||
|
print(f" {i+1}. {path}")
|
||||||
|
|
||||||
|
print("\n🔍 Expected Path Mappings:")
|
||||||
|
if movie_paths != "NOT SET" and radarr_paths != "NOT SET":
|
||||||
|
container_list = [p.strip() for p in movie_paths.split(",") if p.strip()]
|
||||||
|
radarr_list = [p.strip() for p in radarr_paths.split(",") if p.strip()]
|
||||||
|
|
||||||
|
for i, container_path in enumerate(container_list):
|
||||||
|
if i < len(radarr_list):
|
||||||
|
radarr_path = radarr_list[i]
|
||||||
|
print(f" {radarr_path} → {container_path}")
|
||||||
|
else:
|
||||||
|
print(f" ❌ No Radarr mapping for {container_path}")
|
||||||
|
|
||||||
|
print("\n🧪 Test Case:")
|
||||||
|
test_radarr_path = "/mnt/unionfs/Media/Movies/movies6/Annabelle Comes Home (2019) [tt8350360]"
|
||||||
|
print(f"Radarr path: {test_radarr_path}")
|
||||||
|
|
||||||
|
# Simulate mapping
|
||||||
|
if radarr_paths != "NOT SET" and movie_paths != "NOT SET":
|
||||||
|
container_list = [p.strip() for p in movie_paths.split(",") if p.strip()]
|
||||||
|
radarr_list = [p.strip() for p in radarr_paths.split(",") if p.strip()]
|
||||||
|
|
||||||
|
for i, radarr_root in enumerate(radarr_list):
|
||||||
|
if test_radarr_path.startswith(radarr_root):
|
||||||
|
container_root = container_list[i] if i < len(container_list) else container_list[0]
|
||||||
|
relative_path = test_radarr_path[len(radarr_root):].lstrip("/")
|
||||||
|
mapped_path = f"{container_root.rstrip('/')}/{relative_path}" if relative_path else container_root.rstrip("/")
|
||||||
|
print(f"Expected mapping: {mapped_path}")
|
||||||
|
|
||||||
|
# Check if mapped path exists
|
||||||
|
if Path(mapped_path).exists():
|
||||||
|
print("✅ Mapped path exists!")
|
||||||
|
else:
|
||||||
|
print("❌ Mapped path does not exist!")
|
||||||
|
print(f" This could be why the wrong movie is being processed.")
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
print("❌ No mapping found for test path")
|
||||||
|
|
||||||
|
print("\n🛠️ Recommended Fix:")
|
||||||
|
print("Make sure your MOVIE_PATHS and RADARR_ROOT_FOLDERS match exactly:")
|
||||||
|
print("Example:")
|
||||||
|
print(" MOVIE_PATHS=/media/Movies/movies,/media/Movies/movies6")
|
||||||
|
print(" RADARR_ROOT_FOLDERS=/mnt/unionfs/Media/Movies/movies,/mnt/unionfs/Media/Movies/movies6")
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
check_path_mapping()
|
||||||
@@ -0,0 +1,127 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Debug script for the Annabelle → Harry Potter webhook processing bug
|
||||||
|
Tests the webhook isolation fixes in v1.3.1
|
||||||
|
"""
|
||||||
|
|
||||||
|
import requests
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
def test_webhook_isolation():
|
||||||
|
"""Test that movie webhooks process the correct movie"""
|
||||||
|
print("🔍 Testing Webhook Isolation Fixes (v1.3.1)")
|
||||||
|
print("=" * 60)
|
||||||
|
|
||||||
|
# Test case from the bug report
|
||||||
|
test_cases = [
|
||||||
|
{
|
||||||
|
"imdb_id": "tt3322940",
|
||||||
|
"title": "Annabelle (2014)",
|
||||||
|
"path": "/mnt/unionfs/Media/Movies/movies6/Annabelle (2014) [tt3322940]",
|
||||||
|
"expected_container_path": "/media/Movies/movies6/Annabelle (2014) [tt3322940]"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"imdb_id": "tt8350360",
|
||||||
|
"title": "Annabelle Comes Home (2019)",
|
||||||
|
"path": "/mnt/unionfs/Media/Movies/movies6/Annabelle Comes Home (2019) [tt8350360]",
|
||||||
|
"expected_container_path": "/media/Movies/movies6/Annabelle Comes Home (2019) [tt8350360]"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
for i, test_case in enumerate(test_cases, 1):
|
||||||
|
print(f"\n{i}. Testing {test_case['title']} ({test_case['imdb_id']})")
|
||||||
|
print("-" * 50)
|
||||||
|
|
||||||
|
# Create webhook payload
|
||||||
|
webhook_payload = {
|
||||||
|
"eventType": "Download",
|
||||||
|
"movie": {
|
||||||
|
"id": i,
|
||||||
|
"title": test_case["title"],
|
||||||
|
"imdbId": test_case["imdb_id"],
|
||||||
|
"path": test_case["path"]
|
||||||
|
},
|
||||||
|
"movieFile": {
|
||||||
|
"id": i,
|
||||||
|
"path": f"{test_case['path']}/movie.mkv"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
print(f"📤 Sending webhook...")
|
||||||
|
print(f" Radarr Path: {test_case['path']}")
|
||||||
|
print(f" Expected Container Path: {test_case['expected_container_path']}")
|
||||||
|
|
||||||
|
try:
|
||||||
|
response = requests.post(
|
||||||
|
"http://localhost:8080/webhook/radarr",
|
||||||
|
json=webhook_payload,
|
||||||
|
headers={"Content-Type": "application/json"},
|
||||||
|
timeout=10
|
||||||
|
)
|
||||||
|
|
||||||
|
if response.status_code == 200:
|
||||||
|
result = response.json()
|
||||||
|
print(f" ✅ Webhook accepted: {result.get('message', 'Success')}")
|
||||||
|
|
||||||
|
# Check if batch key uses new prefixed format
|
||||||
|
if 'movie:' in result.get('message', ''):
|
||||||
|
print(f" ✅ Using prefixed batch key (isolation working)")
|
||||||
|
else:
|
||||||
|
print(f" ⚠️ Batch key format unclear")
|
||||||
|
|
||||||
|
else:
|
||||||
|
print(f" ❌ Webhook rejected: {response.status_code}")
|
||||||
|
print(f" Error: {response.text}")
|
||||||
|
|
||||||
|
# This might be expected if path doesn't exist (validation working)
|
||||||
|
if "does not exist" in response.text:
|
||||||
|
print(f" ✅ Path validation working (rejects non-existent paths)")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f" ❌ Error: {e}")
|
||||||
|
|
||||||
|
# Small delay between tests
|
||||||
|
time.sleep(1)
|
||||||
|
|
||||||
|
print(f"\n📋 Next Steps:")
|
||||||
|
print(f"1. Check NFOGuard logs to see which movies were actually processed")
|
||||||
|
print(f"2. Verify logs show 'Batch validation passed' messages")
|
||||||
|
print(f"3. Ensure no 'BATCH VALIDATION FAILED' errors")
|
||||||
|
print(f"4. Confirm correct movies are processed (not Harry Potter)")
|
||||||
|
|
||||||
|
# Get batch status
|
||||||
|
print(f"\n📊 Current Batch Status:")
|
||||||
|
try:
|
||||||
|
response = requests.get("http://localhost:8080/batch/status", timeout=5)
|
||||||
|
if response.status_code == 200:
|
||||||
|
data = response.json()
|
||||||
|
print(f" Pending: {data.get('pending_count', 0)}")
|
||||||
|
print(f" Processing: {data.get('processing_count', 0)}")
|
||||||
|
if data.get('pending_items'):
|
||||||
|
print(f" Keys: {data['pending_items']}")
|
||||||
|
else:
|
||||||
|
print(f" Could not get batch status: {response.status_code}")
|
||||||
|
except Exception as e:
|
||||||
|
print(f" Batch status error: {e}")
|
||||||
|
|
||||||
|
def test_path_mapping():
|
||||||
|
"""Test path mapping configuration"""
|
||||||
|
print(f"\n🗺️ Testing Path Mapping")
|
||||||
|
print("=" * 30)
|
||||||
|
|
||||||
|
test_paths = [
|
||||||
|
"/mnt/unionfs/Media/Movies/movies6/Annabelle (2014) [tt3322940]",
|
||||||
|
"/mnt/unionfs/Media/Movies/movies6/Annabelle Comes Home (2019) [tt8350360]",
|
||||||
|
"/mnt/unionfs/Media/Movies/movies/Some Movie (2023) [tt1234567]"
|
||||||
|
]
|
||||||
|
|
||||||
|
for path in test_paths:
|
||||||
|
print(f"Testing: {path}")
|
||||||
|
# This would need to be implemented in NFOGuard as a debug endpoint
|
||||||
|
print(f" → Expected: /media/Movies/movies6/... (if movies6 mapping exists)")
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
test_webhook_isolation()
|
||||||
|
test_path_mapping()
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Debug script for webhook processing issue
|
||||||
|
Helps identify why wrong movies are being processed
|
||||||
|
"""
|
||||||
|
|
||||||
|
import requests
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
def check_batch_status():
|
||||||
|
"""Check the current batch queue status"""
|
||||||
|
try:
|
||||||
|
response = requests.get("http://localhost:8080/batch/status")
|
||||||
|
if response.status_code == 200:
|
||||||
|
data = response.json()
|
||||||
|
print(f"[{datetime.now().isoformat()}] Batch Status:")
|
||||||
|
print(f" Pending batches: {data.get('pending_batches', [])}")
|
||||||
|
print(f" Active timers: {data.get('active_timers', [])}")
|
||||||
|
if 'batch_details' in data:
|
||||||
|
print(" Batch details:")
|
||||||
|
for key, details in data['batch_details'].items():
|
||||||
|
print(f" {key}: {details['movie_title']} (IMDb: {details['imdb_id']})")
|
||||||
|
print()
|
||||||
|
else:
|
||||||
|
print(f"Failed to get batch status: {response.status_code}")
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error checking batch status: {e}")
|
||||||
|
|
||||||
|
def simulate_radarr_webhook(imdb_id, title, movie_path):
|
||||||
|
"""Simulate a Radarr webhook"""
|
||||||
|
webhook_payload = {
|
||||||
|
"eventType": "Download",
|
||||||
|
"movie": {
|
||||||
|
"id": 1,
|
||||||
|
"title": title,
|
||||||
|
"imdbId": imdb_id,
|
||||||
|
"path": movie_path
|
||||||
|
},
|
||||||
|
"movieFile": {
|
||||||
|
"id": 1,
|
||||||
|
"path": f"{movie_path}/movie.mkv"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
print(f"[{datetime.now().isoformat()}] Sending webhook for {title} ({imdb_id})")
|
||||||
|
response = requests.post(
|
||||||
|
"http://localhost:8080/webhook/radarr",
|
||||||
|
json=webhook_payload,
|
||||||
|
headers={"Content-Type": "application/json"}
|
||||||
|
)
|
||||||
|
|
||||||
|
if response.status_code == 200:
|
||||||
|
result = response.json()
|
||||||
|
print(f" ✅ Webhook accepted: {result}")
|
||||||
|
else:
|
||||||
|
print(f" ❌ Webhook failed: {response.status_code} - {response.text}")
|
||||||
|
except Exception as e:
|
||||||
|
print(f" ❌ Error sending webhook: {e}")
|
||||||
|
|
||||||
|
def main():
|
||||||
|
print("🔍 NFOGuard Webhook Debugging Tool")
|
||||||
|
print("=" * 50)
|
||||||
|
|
||||||
|
# Check initial batch status
|
||||||
|
print("1. Initial batch queue status:")
|
||||||
|
check_batch_status()
|
||||||
|
|
||||||
|
# Send test webhook for Annabelle Comes Home
|
||||||
|
print("2. Sending test webhook for Annabelle Comes Home:")
|
||||||
|
simulate_radarr_webhook(
|
||||||
|
"tt8350360",
|
||||||
|
"Annabelle Comes Home",
|
||||||
|
"/mnt/unionfs/Media/Movies/movies6/Annabelle Comes Home (2019) [tt8350360]"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Check batch status immediately after
|
||||||
|
print("3. Batch status immediately after webhook:")
|
||||||
|
check_batch_status()
|
||||||
|
|
||||||
|
# Wait for processing
|
||||||
|
print("4. Waiting 6 seconds for processing...")
|
||||||
|
time.sleep(6)
|
||||||
|
|
||||||
|
# Check final batch status
|
||||||
|
print("5. Final batch status:")
|
||||||
|
check_batch_status()
|
||||||
|
|
||||||
|
print("\n🔍 Check the NFOGuard logs to see which movie was actually processed!")
|
||||||
|
print("Expected: Annabelle Comes Home (2019) [tt8350360]")
|
||||||
|
print("If you see a different movie, there's a batching bug.")
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
+60
-45
@@ -942,7 +942,7 @@ class MovieProcessor:
|
|||||||
_log("ERROR", f"No IMDb ID found in movie path: {movie_path}")
|
_log("ERROR", f"No IMDb ID found in movie path: {movie_path}")
|
||||||
return
|
return
|
||||||
|
|
||||||
_log("INFO", f"Processing movie: {movie_path.name}")
|
_log("INFO", f"Processing movie: {movie_path.name} (IMDb: {imdb_id})")
|
||||||
|
|
||||||
# Update database
|
# Update database
|
||||||
self.db.upsert_movie(imdb_id, str(movie_path))
|
self.db.upsert_movie(imdb_id, str(movie_path))
|
||||||
@@ -1259,6 +1259,7 @@ class WebhookBatcher:
|
|||||||
webhook_data['media_type'] = media_type
|
webhook_data['media_type'] = media_type
|
||||||
self.pending[key] = webhook_data
|
self.pending[key] = webhook_data
|
||||||
_log("INFO", f"Batched {media_type} webhook for {key}")
|
_log("INFO", f"Batched {media_type} webhook for {key}")
|
||||||
|
_log("DEBUG", f"Batch added - key: {key}, media_type: {media_type}, timer scheduled for {config.batch_delay}s")
|
||||||
|
|
||||||
timer = threading.Timer(config.batch_delay, self._process_item, args=[key])
|
timer = threading.Timer(config.batch_delay, self._process_item, args=[key])
|
||||||
self.timers[key] = timer
|
self.timers[key] = timer
|
||||||
@@ -1281,20 +1282,32 @@ class WebhookBatcher:
|
|||||||
self.processing.discard(key)
|
self.processing.discard(key)
|
||||||
|
|
||||||
def _process_sync(self, key: str, webhook_data: Dict):
|
def _process_sync(self, key: str, webhook_data: Dict):
|
||||||
"""Synchronous processing of webhook data"""
|
"""Synchronous processing of webhook data with validation"""
|
||||||
try:
|
try:
|
||||||
media_type = webhook_data.get('media_type')
|
media_type = webhook_data.get('media_type')
|
||||||
path_str = webhook_data.get('path')
|
path_str = webhook_data.get('path')
|
||||||
|
|
||||||
|
_log("DEBUG", f"Processing batch item: key={key}, media_type={media_type}, path={path_str}")
|
||||||
|
|
||||||
if not path_str:
|
if not path_str:
|
||||||
_log("ERROR", f"No path found for {media_type} {key}")
|
_log("ERROR", f"No path found for {media_type} {key}")
|
||||||
return
|
return
|
||||||
|
|
||||||
path_obj = Path(path_str)
|
path_obj = Path(path_str)
|
||||||
if not path_obj.exists():
|
if not path_obj.exists():
|
||||||
_log("ERROR", f"Path does not exist: {path_obj}")
|
_log("ERROR", f"BATCH PROCESSING FAILED: Path does not exist: {path_obj}")
|
||||||
|
_log("ERROR", f"This indicates a path mapping issue - webhook rejected to prevent wrong processing")
|
||||||
return
|
return
|
||||||
|
|
||||||
|
# CRITICAL: Validate that the path contains the expected IMDb ID for movies
|
||||||
|
if media_type == 'movie':
|
||||||
|
expected_imdb = key.replace('movie:', '') if key.startswith('movie:') else key
|
||||||
|
if expected_imdb not in path_str.lower():
|
||||||
|
_log("ERROR", f"BATCH VALIDATION FAILED: Expected IMDb {expected_imdb} not found in path {path_str}")
|
||||||
|
_log("ERROR", f"This prevents processing wrong movies due to batch corruption")
|
||||||
|
return
|
||||||
|
_log("DEBUG", f"Batch validation passed: IMDb {expected_imdb} found in path")
|
||||||
|
|
||||||
# Process based on media type
|
# Process based on media type
|
||||||
if media_type == 'tv':
|
if media_type == 'tv':
|
||||||
# Check processing mode for TV webhooks
|
# Check processing mode for TV webhooks
|
||||||
@@ -1424,7 +1437,8 @@ async def sonarr_webhook(request: Request, background_tasks: BackgroundTasks):
|
|||||||
_log("ERROR", f"Could not find series directory: {series_title} ({imdb_id})")
|
_log("ERROR", f"Could not find series directory: {series_title} ({imdb_id})")
|
||||||
return {"status": "error", "reason": "Series directory not found"}
|
return {"status": "error", "reason": "Series directory not found"}
|
||||||
|
|
||||||
# Add to batch queue
|
# Add to batch queue with TV-prefixed key to avoid movie conflicts
|
||||||
|
tv_batch_key = f"tv:{imdb_id}"
|
||||||
webhook_dict = {
|
webhook_dict = {
|
||||||
'path': str(series_path),
|
'path': str(series_path),
|
||||||
'series_info': series_info,
|
'series_info': series_info,
|
||||||
@@ -1432,9 +1446,9 @@ async def sonarr_webhook(request: Request, background_tasks: BackgroundTasks):
|
|||||||
'episodes': webhook.episodes or [], # Include episode data for targeted processing
|
'episodes': webhook.episodes or [], # Include episode data for targeted processing
|
||||||
'processing_mode': config.tv_webhook_processing_mode
|
'processing_mode': config.tv_webhook_processing_mode
|
||||||
}
|
}
|
||||||
batcher.add_webhook(imdb_id, webhook_dict, 'tv')
|
batcher.add_webhook(tv_batch_key, webhook_dict, 'tv')
|
||||||
|
|
||||||
return {"status": "accepted", "message": "Sonarr webhook queued"}
|
return {"status": "accepted", "message": f"Sonarr webhook queued for {tv_batch_key}"}
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
_log("ERROR", f"Sonarr webhook error: {e}")
|
_log("ERROR", f"Sonarr webhook error: {e}")
|
||||||
@@ -1445,55 +1459,56 @@ async def radarr_webhook(request: Request, background_tasks: BackgroundTasks):
|
|||||||
"""Handle Radarr webhooks"""
|
"""Handle Radarr webhooks"""
|
||||||
try:
|
try:
|
||||||
payload = await _read_payload(request)
|
payload = await _read_payload(request)
|
||||||
if not payload:
|
_log("INFO", f"Received Radarr webhook: {payload.get('eventType', 'Unknown')}")
|
||||||
raise HTTPException(status_code=422, detail="Empty Radarr payload")
|
_log("DEBUG", f"Full Radarr webhook payload: {payload}")
|
||||||
|
|
||||||
webhook = RadarrWebhook(**payload)
|
# Extract movie info
|
||||||
_log("INFO", f"Received Radarr webhook: {webhook.eventType}")
|
movie_data = payload.get("movie", {})
|
||||||
|
if not movie_data:
|
||||||
|
_log("WARNING", "No movie data in Radarr webhook")
|
||||||
|
return {"status": "error", "message": "No movie data"}
|
||||||
|
|
||||||
if webhook.eventType not in ["Download", "Upgrade", "Rename", "Test"]:
|
# Get IMDb ID for batching key
|
||||||
return {"status": "ignored", "reason": f"Event type {webhook.eventType} not processed"}
|
imdb_id = movie_data.get("imdbId", "").lower()
|
||||||
|
|
||||||
if webhook.eventType == "Test":
|
|
||||||
return {"status": "ok", "message": "Test received"}
|
|
||||||
|
|
||||||
if not webhook.movie:
|
|
||||||
return {"status": "ignored", "reason": "No movie data"}
|
|
||||||
|
|
||||||
movie_info = webhook.movie
|
|
||||||
movie_title = movie_info.get("title", "")
|
|
||||||
imdb_id = (movie_info.get("imdbId") or "").strip()
|
|
||||||
if imdb_id:
|
|
||||||
imdb_id = f"tt{imdb_id.replace('tt','')}"
|
|
||||||
radarr_path = movie_info.get("folderPath") or movie_info.get("path", "")
|
|
||||||
|
|
||||||
# Find movie path
|
|
||||||
movie_path = movie_processor.find_movie_path(movie_title, imdb_id, radarr_path)
|
|
||||||
if not movie_path:
|
|
||||||
_log("ERROR", f"Could not find movie directory: {movie_title} ({imdb_id})")
|
|
||||||
return {"status": "error", "reason": "Movie directory not found"}
|
|
||||||
|
|
||||||
# Extract IMDb ID from path if not in webhook
|
|
||||||
if not imdb_id:
|
if not imdb_id:
|
||||||
imdb_id = nfo_manager.parse_imdb_from_path(movie_path)
|
_log("WARNING", "No IMDb ID in Radarr webhook movie data")
|
||||||
|
return {"status": "error", "message": "No IMDb ID"}
|
||||||
|
|
||||||
if not imdb_id:
|
# Get movie path for verification
|
||||||
_log("ERROR", f"No IMDb ID available for movie: {movie_title}")
|
movie_path = movie_data.get("path", "")
|
||||||
return {"status": "error", "reason": "No IMDb ID"}
|
if movie_path:
|
||||||
|
container_path = path_mapper.radarr_path_to_container_path(movie_path)
|
||||||
|
_log("DEBUG", f"Mapped Radarr path {movie_path} -> {container_path}")
|
||||||
|
|
||||||
# Add to batch queue
|
# CRITICAL: Verify the mapped path actually exists
|
||||||
webhook_dict = {
|
from pathlib import Path
|
||||||
'path': str(movie_path),
|
if not Path(container_path).exists():
|
||||||
'movie_info': movie_info,
|
_log("ERROR", f"RADARR WEBHOOK REJECTED: Mapped path does not exist: {container_path}")
|
||||||
'event_type': webhook.eventType
|
_log("ERROR", f"This prevents processing wrong movies due to path mapping issues")
|
||||||
|
return {"status": "error", "message": f"Mapped movie path does not exist: {container_path}"}
|
||||||
|
|
||||||
|
# Verify the path contains the expected IMDb ID
|
||||||
|
if imdb_id not in container_path.lower():
|
||||||
|
_log("WARNING", f"IMDb ID {imdb_id} not found in container path {container_path}")
|
||||||
|
|
||||||
|
# Create movie-specific webhook data with proper path validation
|
||||||
|
movie_webhook_data = {
|
||||||
|
'path': container_path, # Use verified container path
|
||||||
|
'movie_info': movie_data,
|
||||||
|
'event_type': payload.get('eventType'),
|
||||||
|
'original_payload': payload
|
||||||
}
|
}
|
||||||
batcher.add_webhook(imdb_id, webhook_dict, 'movie')
|
|
||||||
|
|
||||||
return {"status": "accepted", "message": "Radarr webhook queued"}
|
# Add to batch queue with movie-prefixed key to avoid TV conflicts
|
||||||
|
movie_batch_key = f"movie:{imdb_id}"
|
||||||
|
_log("DEBUG", f"Adding Radarr webhook to batch: key={movie_batch_key}, movie_title={movie_data.get('title', 'Unknown')}")
|
||||||
|
batcher.add_webhook(movie_batch_key, movie_webhook_data, "movie")
|
||||||
|
|
||||||
|
return {"status": "success", "message": f"Radarr webhook queued for {movie_batch_key}"}
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
_log("ERROR", f"Radarr webhook error: {e}")
|
_log("ERROR", f"Radarr webhook error: {e}")
|
||||||
raise HTTPException(status_code=422, detail=f"Invalid webhook: {e}")
|
return {"status": "error", "message": str(e)}
|
||||||
|
|
||||||
# ---------------------------
|
# ---------------------------
|
||||||
# API Endpoints
|
# API Endpoints
|
||||||
|
|||||||
@@ -0,0 +1,10 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Execute the git commit
|
||||||
|
|
||||||
|
cd /home/jskala/github/NFOguard
|
||||||
|
|
||||||
|
# Make commit script executable
|
||||||
|
chmod +x commit_webhook_fixes.sh
|
||||||
|
|
||||||
|
# Run the commit
|
||||||
|
./commit_webhook_fixes.sh
|
||||||
Reference in New Issue
Block a user