From 7e56ac16012af83fb84c86edb80a9b1c758d9258 Mon Sep 17 00:00:00 2001 From: sbcrumb Date: Fri, 10 Oct 2025 15:47:53 -0400 Subject: [PATCH] Implement optimal caching hierarchy for both movies and TV shows Perfect the caching strategy as requested: 1. Database check first (fastest lookup) 2. NFO file extraction with database caching 3. API calls only when absolutely necessary Movies: Fixed order from NFO->DB->API to DB->NFO->API TV Shows: Added missing NFO extraction tier between database and API Benefits: - First scan: Extract existing NFOGuard data from NFOs and cache in DB - Subsequent scans: ~99% database cache hits, minimal API calls - NFO recovery: Rebuild database from existing NFO files - Webhook processing: Cache webhook dates in database immediately Expected performance: Manual scans now use 3-tier optimization --- VERSION | 2 +- processors/movie_processor.py | 105 ++++++++++++++-------------------- processors/tv_processor.py | 55 +++++++++++++++--- 3 files changed, 93 insertions(+), 69 deletions(-) diff --git a/VERSION b/VERSION index 7d5624f..1b8629f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.21 +2.0.22 diff --git a/processors/movie_processor.py b/processors/movie_processor.py index 9156061..6cefa05 100644 --- a/processors/movie_processor.py +++ b/processors/movie_processor.py @@ -114,20 +114,45 @@ class MovieProcessor: self.db.upsert_movie_dates(imdb_id, None, None, None, False) return - # TIER 1: Check if NFO file already has NFOGuard data (fastest - no DB or API calls) + # TIER 1: Check database first (fastest - local lookup) + existing = self.db.get_movie_dates(imdb_id) + _log("DEBUG", f"Database lookup for {imdb_id}: {existing}") + + # If we have complete data in database, use it and skip all other checks + if existing and existing.get("dateadded") and existing.get("source") != "no_valid_date_source": + _log("INFO", f"✅ TIER 1 - Using complete database data for {imdb_id}: {existing['dateadded']} (source: {existing['source']})") + dateadded, source, released = existing["dateadded"], existing["source"], existing.get("released") + + # Create NFO with existing data and update files + if config.manage_nfo: + self.nfo_manager.create_movie_nfo( + movie_path, imdb_id, dateadded, released, source, config.lock_metadata + ) + + if config.fix_dir_mtimes and dateadded: + self.nfo_manager.update_movie_files_mtime(movie_path, dateadded) + + _log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [database-cached]") + return + + # TIER 2: Check if NFO file has NFOGuard data and cache it in database nfo_path = movie_path / "movie.nfo" nfo_data = self.nfo_manager.extract_nfoguard_dates_from_nfo(nfo_path) if nfo_data: - _log("INFO", f"🚀 Using existing NFOGuard data from NFO file: {nfo_data['dateadded']} (source: {nfo_data['source']})") + _log("INFO", f"🚀 TIER 2 - Found NFOGuard data in NFO file: {nfo_data['dateadded']} (source: {nfo_data['source']})") dateadded = nfo_data["dateadded"] source = nfo_data["source"] released = nfo_data.get("released") + # Cache NFO data in database for future lookups + self.db.upsert_movie_dates(imdb_id, dateadded, released, source, True) + _log("INFO", f"✅ Cached NFO data in database for {imdb_id}") + # Update file mtimes if enabled (NFO is already correct) if config.fix_dir_mtimes and dateadded: self.nfo_manager.update_movie_files_mtime(movie_path, dateadded) - _log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [nfo-only]") + _log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [nfo-cached]") return # TIER 1.5: Special handling for TMDB-only movies - extract dates from existing NFO @@ -155,68 +180,26 @@ class MovieProcessor: _log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [tmdb-nfo]") return - # TIER 2: Check database for existing data - existing = self.db.get_movie_dates(imdb_id) - _log("DEBUG", f"Database lookup for {imdb_id}: {existing}") + # TIER 3: No cached data found - proceed with API lookups and verification - # If we have complete data in database, use it and skip expensive API calls - if existing and existing.get("dateadded") and existing.get("source") != "no_valid_date_source": - _log("INFO", f"✅ Using complete database data for {imdb_id}: {existing['dateadded']} (source: {existing['source']})") - # Still create NFO and update files but skip API queries - dateadded, source, released = existing["dateadded"], existing["source"], existing.get("released") - - # Create NFO with existing data - print(f"🔍 TIER2 - config.manage_nfo: {config.manage_nfo}") - if config.manage_nfo: - print(f"🔍 TIER2 - Calling create_movie_nfo with dateadded: {dateadded}") - self.nfo_manager.create_movie_nfo( - movie_path, imdb_id, dateadded, released, source, config.lock_metadata - ) - else: - print(f"❌ TIER2 - manage_nfo is disabled, skipping NFO creation") - - # Update file mtimes if enabled - if config.fix_dir_mtimes and dateadded: - self.nfo_manager.update_movie_files_mtime(movie_path, dateadded) - - _log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [database-only]") - return - - # Handle webhook mode - prioritize database, then use proper date logic + # TIER 3: No cached data found - determine if we should query APIs if webhook_mode: - _log("DEBUG", f"Webhook mode: existing={bool(existing)}, has_dateadded={bool(existing and existing.get('dateadded')) if existing else 'N/A'}") - if existing and existing.get("dateadded"): - _log("INFO", f"Webhook processing - using existing database entry: {existing['dateadded']} (source: {existing.get('source', 'unknown')})") - dateadded, source, released = existing["dateadded"], existing["source"], existing.get("released") - else: - if existing: - _log("INFO", f"Webhook processing - database entry exists but no dateadded field: {existing}") - else: - _log("INFO", f"Webhook processing - no database entry found for {imdb_id}") - _log("INFO", f"Using full date decision logic") - # Use same logic as manual scan to check Radarr import dates, release dates, etc. - should_query = True # Always query for webhooks when no database entry exists - dateadded, source, released = self._decide_movie_dates(imdb_id, movie_path, should_query, existing) - - # Only if ALL date sources fail, fall back to current timestamp - if dateadded is None: - local_tz = _get_local_timezone() - current_time = datetime.now(local_tz).isoformat(timespec="seconds") - _log("INFO", f"Webhook processing - all date sources failed, using current timestamp as last resort: {current_time}") - dateadded, source = current_time, "webhook:fallback_timestamp" + _log("INFO", f"Webhook processing - no cached data found, using full date decision logic") + should_query = True # Always query for webhooks when no cached data exists else: # Manual scan mode - determine if we should query APIs - should_query = ( - config.movie_poll_mode == "always" or - (config.movie_poll_mode == "if_missing" and not existing) or - (config.movie_poll_mode == "if_missing" and existing and existing.get("source") == "file:mtime") or - (config.movie_poll_mode == "if_missing" and existing and not existing.get("dateadded")) - ) - - _log("DEBUG", f"Movie {imdb_id}: should_query={should_query}, poll_mode={config.movie_poll_mode}, existing={bool(existing)}, has_dateadded={bool(existing and existing.get('dateadded')) if existing else False}") - - # Use existing movie date decision logic - dateadded, source, released = self._decide_movie_dates(imdb_id, movie_path, should_query, existing) + should_query = config.movie_poll_mode == "always" + _log("DEBUG", f"Movie {imdb_id}: should_query={should_query}, poll_mode={config.movie_poll_mode}") + + # Use existing movie date decision logic + dateadded, source, released = self._decide_movie_dates(imdb_id, movie_path, should_query, None) + + # Webhook fallback: if ALL date sources fail, use current timestamp + if webhook_mode and dateadded is None: + local_tz = _get_local_timezone() + current_time = datetime.now(local_tz).isoformat(timespec="seconds") + _log("INFO", f"Webhook processing - all date sources failed, using current timestamp as last resort: {current_time}") + dateadded, source = current_time, "webhook:fallback_timestamp" # If we don't have an import/download date but we have a release date, use it as dateadded # This ensures we save digital release dates, theatrical dates, etc. to the database diff --git a/processors/tv_processor.py b/processors/tv_processor.py index 6d11216..a233ddf 100644 --- a/processors/tv_processor.py +++ b/processors/tv_processor.py @@ -106,9 +106,10 @@ class TVProcessor: episode_dates = {} episodes_needing_lookup = [] - # OPTIMIZATION: Check database first for existing dates - _log("DEBUG", f"Checking database for existing episode dates for {len(disk_episodes)} episodes") + # TIER 1: Check database first for existing dates (fastest) + _log("DEBUG", f"TIER 1 - Checking database for existing episode dates for {len(disk_episodes)} episodes") db_cache_hits = 0 + episodes_needing_nfo_check = [] for (season, episode) in disk_episodes: # Try database first - this is much faster than API calls @@ -123,14 +124,54 @@ class TVProcessor: db_cache_hits += 1 _log("DEBUG", f"Database cache hit for S{season:02d}E{episode:02d}: {dateadded}") else: - # Not in database or incomplete - needs lookup - episodes_needing_lookup.append((season, episode)) + # Not in database or incomplete - needs NFO check + episodes_needing_nfo_check.append((season, episode)) - _log("INFO", f"Database cache hits: {db_cache_hits}/{len(disk_episodes)} episodes. Need API lookup: {len(episodes_needing_lookup)}") + _log("INFO", f"Database cache hits: {db_cache_hits}/{len(disk_episodes)} episodes. Need NFO check: {len(episodes_needing_nfo_check)}") - # Only call Sonarr API for episodes not in database + # TIER 2: Check NFO files for NFOGuard dates and cache them in database + nfo_cache_hits = 0 + episodes_needing_lookup = [] + + if episodes_needing_nfo_check: + _log("DEBUG", f"TIER 2 - Checking NFO files for NFOGuard dates for {len(episodes_needing_nfo_check)} episodes") + + for (season, episode) in episodes_needing_nfo_check: + # Look for existing NFO files for this episode + season_dir = series_path / config.tv_season_dir_format.format(season=season) + episode_files = disk_episodes[(season, episode)] + + nfo_found = False + for episode_file in episode_files: + # Try to find matching NFO file + nfo_path = episode_file.with_suffix('.nfo') + if nfo_path.exists(): + # Extract NFOGuard data from episode NFO + nfo_data = self.nfo_manager.extract_nfoguard_dates_from_nfo(nfo_path) + if nfo_data: + aired = nfo_data.get('aired') + dateadded = nfo_data.get('dateadded') + source = nfo_data.get('source', 'nfo_cache') + + if dateadded: + episode_dates[(season, episode)] = (aired, dateadded, source) + nfo_cache_hits += 1 + nfo_found = True + + # Cache NFO data in database for future lookups + self.db.upsert_episode_date(imdb_id, season, episode, aired, dateadded, source, True) + _log("DEBUG", f"NFO cache hit for S{season:02d}E{episode:02d}: {dateadded} - cached in DB") + break + + if not nfo_found: + # No NFO data found - needs API lookup + episodes_needing_lookup.append((season, episode)) + + _log("INFO", f"NFO cache hits: {nfo_cache_hits}/{len(episodes_needing_nfo_check)} episodes. Need API lookup: {len(episodes_needing_lookup)}") + + # TIER 3: Only call Sonarr API for episodes not in database or NFO files if episodes_needing_lookup: - _log("DEBUG", f"Querying Sonarr for {len(episodes_needing_lookup)} episodes missing from database") + _log("DEBUG", f"TIER 3 - Querying Sonarr for {len(episodes_needing_lookup)} episodes missing from database and NFO files") sonarr_episodes = self._get_sonarr_episodes(imdb_id, episodes_needing_lookup) # Process episodes that needed lookup