From 9f86b0266514fc6abee9719c329f6aeebdfbde2a Mon Sep 17 00:00:00 2001 From: sbcrumb Date: Thu, 2 Oct 2025 16:31:07 -0400 Subject: [PATCH] fix: fixing webhook processing --- README.md | 147 +++++++++++++++++++++++++++++++++--- VERSION | 2 +- main.py | 4 +- nfoguard.py | 2 +- webhooks/webhook_batcher.py | 32 ++++++-- 5 files changed, 165 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index dba03b0..15e613d 100644 --- a/README.md +++ b/README.md @@ -26,14 +26,37 @@ NFOGuard automatically updates movie and TV show NFO files with proper release d ## ✨ Features -- 🎬 **Movie & TV Support** - Works with both Radarr and Sonarr -- 📅 **Smart Date Handling** - Prioritizes digital, physical, and theatrical release dates -- 🔄 **Webhook Integration** - Triggers automatically on import, upgrade, and rename -- 🗄️ **Database Integration** - Direct PostgreSQL access for better performance -- 📝 **NFO Preservation** - Maintains existing metadata, adds fields cleanly at bottom -- 🔒 **Metadata Locking** - Prevents overwrites with lockdata tags -- ⚡ **Batch Processing** - Efficient handling of multiple files -- 🐳 **Docker Ready** - Easy deployment with Docker Compose +### **🎬 Core Media Management** +- **Movie & TV Support** - Works with both Radarr and Sonarr +- **Smart Date Handling** - Prioritizes digital, physical, and theatrical release dates +- **Webhook Integration** - Triggers automatically on import, upgrade, and rename +- **NFO Preservation** - Maintains existing metadata, adds fields cleanly at bottom +- **Metadata Locking** - Prevents overwrites with lockdata tags + +### **⚡ Performance & Scalability** +- **Async I/O Operations** - High-performance concurrent file processing +- **Batch Processing** - Efficient handling of multiple files simultaneously +- **Database Integration** - Direct PostgreSQL/SQLite access for better performance +- **Smart Concurrency** - Controlled parallel processing with rate limiting + +### **🔧 Configuration & Validation** +- **Comprehensive Config Validation** - Validates all settings before startup +- **Runtime Health Checks** - Monitors system health and dependencies +- **Path Validation** - Ensures media directories are accessible +- **Database Connectivity Tests** - Validates database connections + +### **📊 Monitoring & Observability** +- **Health Check APIs** - `/api/v1/health`, `/api/v1/health/ready`, `/api/v1/health/live` +- **Prometheus Metrics** - `/api/v1/metrics` endpoint for monitoring integration +- **Performance Monitoring** - Operation timing, error rates, and bottleneck detection +- **Structured Logging** - JSON logs with correlation IDs for request tracing +- **System Status APIs** - Real-time performance and error metrics + +### **🐳 Production Ready** +- **Docker & Kubernetes** - Health checks for orchestration platforms +- **Grafana Compatible** - Metrics endpoints for dashboard integration +- **Configuration CLI** - Validation tools for troubleshooting +- **Modular Architecture** - Clean separation of concerns for maintainability ## 🚀 Quick Start @@ -123,11 +146,13 @@ image: sbcrumb/nfoguard:latest image: sbcrumb/nfoguard:dev ``` -### Specific Version +### Specific Version ```yaml -image: sbcrumb/nfoguard:v1.5.5 +image: sbcrumb/nfoguard:v2.0.0 # Latest with monitoring & validation ``` +> **🚀 Version 2.0.0** includes major architecture improvements, async I/O performance enhancements, comprehensive monitoring, and configuration validation. + ## 🔗 Webhook Setup Configure these webhook URLs in your applications: @@ -162,13 +187,38 @@ curl -X POST "http://localhost:8080/bulk/update" ### API Endpoints +#### **Core Operations** | Endpoint | Method | Purpose | |----------|--------|---------| -| `/health` | GET | Health check | | `/webhook/radarr` | POST | Radarr webhook handler | | `/webhook/sonarr` | POST | Sonarr webhook handler | | `/manual/scan` | POST | Manual media scanning | -| `/bulk/update` | POST | Bulk movie updates from Radarr DB | +| `/bulk/update` | POST | Bulk update from Radarr database | + +#### **Health & Monitoring** +| Endpoint | Method | Purpose | +|----------|--------|---------| +| `/api/v1/health` | GET | Comprehensive health status | +| `/api/v1/health/ready` | GET | Kubernetes readiness probe | +| `/api/v1/health/live` | GET | Kubernetes liveness probe | +| `/api/v1/status` | GET | Complete system status | +| `/api/v1/status/brief` | GET | Quick system overview | + +#### **Metrics & Performance** +| Endpoint | Method | Purpose | +|----------|--------|---------| +| `/api/v1/metrics` | GET | Prometheus metrics (text format) | +| `/api/v1/metrics/json` | GET | Metrics in JSON format | +| `/api/v1/metrics/processing` | GET | Processing-specific metrics | +| `/api/v1/metrics/errors` | GET | Error metrics and recent failures | +| `/api/v1/metrics/system` | GET | System resource metrics | + +#### **Configuration & Debugging** +| Endpoint | Method | Purpose | +|----------|--------|---------| +| `/debug/movie/{imdb_id}/priority` | GET | Debug movie priority logic | +| `/debug/tmdb/{imdb_id}` | GET | Debug TMDB lookup | +| `/ping` | GET | Simple connectivity test | ### Manual Scan Parameters @@ -226,6 +276,79 @@ PATH_DEBUG=true curl http://localhost:8080/health ``` +## 📊 Monitoring & Observability + +NFOGuard provides comprehensive monitoring capabilities for production deployments: + +### **Health Checks** +```bash +# Basic health status +curl http://localhost:8080/api/v1/health + +# Kubernetes readiness probe +curl http://localhost:8080/api/v1/health/ready + +# Kubernetes liveness probe +curl http://localhost:8080/api/v1/health/live + +# Quick system overview +curl http://localhost:8080/api/v1/status/brief +``` + +### **Metrics for Grafana/Prometheus** +```bash +# Prometheus metrics format +curl http://localhost:8080/api/v1/metrics + +# JSON metrics for dashboards +curl http://localhost:8080/api/v1/metrics/json + +# Processing performance metrics +curl http://localhost:8080/api/v1/metrics/processing + +# Error rates and recent failures +curl http://localhost:8080/api/v1/metrics/errors + +# System resource usage +curl http://localhost:8080/api/v1/metrics/system +``` + +### **Configuration Validation** +```bash +# Validate configuration before deployment +python config/validation_cli.py + +# Include runtime tests (database, APIs, file access) +python config/validation_cli.py --runtime + +# JSON output for automation +python config/validation_cli.py --runtime --json +``` + +### **Docker Health Checks** +Add to your `docker-compose.yml`: +```yaml +healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/api/v1/health/live"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s +``` + +### **Structured Logging** +NFOGuard uses structured JSON logging with correlation IDs for request tracing: +```bash +# Enable structured logging +STRUCTURED_LOGGING=true + +# Log level configuration +LOG_LEVEL=INFO + +# Example log output with correlation ID +{"timestamp": "2024-01-01T12:00:00Z", "level": "INFO", "correlation_id": "req_123", "message": "Webhook received", "context": {"webhook_type": "radarr", "media_type": "movie"}} +``` + ### Common Issues 1. **Permission Errors**: Ensure NFOGuard can write to mounted directories diff --git a/VERSION b/VERSION index 359a5b9..10bf840 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.0 \ No newline at end of file +2.0.1 \ No newline at end of file diff --git a/main.py b/main.py index c2aabd7..4005224 100644 --- a/main.py +++ b/main.py @@ -94,8 +94,8 @@ def initialize_components(): tv_processor = TVProcessor(db, nfo_manager, path_mapper) movie_processor = MovieProcessor(db, nfo_manager, path_mapper) - # Initialize webhook batcher - batcher = WebhookBatcher() + # Initialize webhook batcher with nfo_manager for comprehensive IMDb detection + batcher = WebhookBatcher(nfo_manager) batcher.set_processors(tv_processor, movie_processor) return { diff --git a/nfoguard.py b/nfoguard.py index 9ffb8f6..5ccd72d 100644 --- a/nfoguard.py +++ b/nfoguard.py @@ -1614,7 +1614,7 @@ nfo_manager = NFOManager(config.manager_brand, config.debug) path_mapper = PathMapper(config) # FIXED: Pass config to PathMapper tv_processor = TVProcessor(db, nfo_manager, path_mapper) movie_processor = MovieProcessor(db, nfo_manager, path_mapper) -batcher = WebhookBatcher() +batcher = WebhookBatcher(nfo_manager) # --------------------------- # Webhook Handlers diff --git a/webhooks/webhook_batcher.py b/webhooks/webhook_batcher.py index f715d7b..3688057 100644 --- a/webhooks/webhook_batcher.py +++ b/webhooks/webhook_batcher.py @@ -14,7 +14,7 @@ from utils.logging import _log class WebhookBatcher: """Batches webhook events to avoid processing storms""" - def __init__(self): + def __init__(self, nfo_manager=None): self.pending: Dict[str, Dict] = {} self.timers: Dict[str, threading.Timer] = {} self.processing: Set[str] = set() @@ -23,6 +23,8 @@ class WebhookBatcher: # Will be set by the application when processors are available self.tv_processor = None self.movie_processor = None + # NFO manager for comprehensive IMDb detection + self.nfo_manager = nfo_manager def set_processors(self, tv_processor, movie_processor): """Set the processor instances""" @@ -81,11 +83,29 @@ class WebhookBatcher: # CRITICAL: Validate that the path contains the expected IMDb ID for movies if media_type == 'movie': expected_imdb = key.replace('movie:', '') if key.startswith('movie:') else key - if expected_imdb not in path_str.lower(): - _log("ERROR", f"BATCH VALIDATION FAILED: Expected IMDb {expected_imdb} not found in path {path_str}") - _log("ERROR", f"This prevents processing wrong movies due to batch corruption") - return - _log("DEBUG", f"Batch validation passed: IMDb {expected_imdb} found in path") + + # Use comprehensive IMDb detection (directory, filenames, NFO files) + if self.nfo_manager: + detected_imdb = self.nfo_manager.find_movie_imdb_id(path_obj) + imdb_match = False + if detected_imdb: + # Compare with and without 'tt' prefix for flexibility + if detected_imdb == expected_imdb or detected_imdb.replace('tt', '') == expected_imdb.replace('tt', ''): + imdb_match = True + + if not imdb_match: + _log("ERROR", f"BATCH VALIDATION FAILED: Expected IMDb {expected_imdb} not found via comprehensive detection in path {path_str}") + _log("ERROR", f"Detected IMDb: {detected_imdb}, Expected: {expected_imdb}") + _log("ERROR", f"This prevents processing wrong movies due to batch corruption") + return + _log("DEBUG", f"Batch validation passed: IMDb {expected_imdb} matches detected {detected_imdb}") + else: + # Fallback to simple string search if nfo_manager not available + if expected_imdb not in path_str.lower(): + _log("ERROR", f"BATCH VALIDATION FAILED: Expected IMDb {expected_imdb} not found in path {path_str} (fallback mode)") + _log("ERROR", f"This prevents processing wrong movies due to batch corruption") + return + _log("DEBUG", f"Batch validation passed: IMDb {expected_imdb} found in path (fallback mode)") # Process based on media type if media_type == 'tv': -- 2.39.5