Merge pull request 'Implement optimal caching hierarchy for both movies and TV shows' (#43) from sonarrissues into dev

Reviewed-on: #43

VERSION

@@ -1 +1 @@
-2.0.21
+2.0.22

processors/movie_processor.py

@@ -114,20 +114,45 @@ class MovieProcessor:
             self.db.upsert_movie_dates(imdb_id, None, None, None, False)
             return
         
-        # TIER 1: Check if NFO file already has NFOGuard data (fastest - no DB or API calls)
+        # TIER 1: Check database first (fastest - local lookup)
+        existing = self.db.get_movie_dates(imdb_id)
+        _log("DEBUG", f"Database lookup for {imdb_id}: {existing}")
+        
+        # If we have complete data in database, use it and skip all other checks
+        if existing and existing.get("dateadded") and existing.get("source") != "no_valid_date_source":
+            _log("INFO", f"✅ TIER 1 - Using complete database data for {imdb_id}: {existing['dateadded']} (source: {existing['source']})")
+            dateadded, source, released = existing["dateadded"], existing["source"], existing.get("released")
+            
+            # Create NFO with existing data and update files
+            if config.manage_nfo:
+                self.nfo_manager.create_movie_nfo(
+                    movie_path, imdb_id, dateadded, released, source, config.lock_metadata
+                )
+            
+            if config.fix_dir_mtimes and dateadded:
+                self.nfo_manager.update_movie_files_mtime(movie_path, dateadded)
+            
+            _log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [database-cached]")
+            return
+        
+        # TIER 2: Check if NFO file has NFOGuard data and cache it in database
         nfo_path = movie_path / "movie.nfo"
         nfo_data = self.nfo_manager.extract_nfoguard_dates_from_nfo(nfo_path)
         if nfo_data:
-            _log("INFO", f"🚀 Using existing NFOGuard data from NFO file: {nfo_data['dateadded']} (source: {nfo_data['source']})")
+            _log("INFO", f"🚀 TIER 2 - Found NFOGuard data in NFO file: {nfo_data['dateadded']} (source: {nfo_data['source']})")
             dateadded = nfo_data["dateadded"]
             source = nfo_data["source"] 
             released = nfo_data.get("released")
             
+            # Cache NFO data in database for future lookups
+            self.db.upsert_movie_dates(imdb_id, dateadded, released, source, True)
+            _log("INFO", f"✅ Cached NFO data in database for {imdb_id}")
+            
             # Update file mtimes if enabled (NFO is already correct)
             if config.fix_dir_mtimes and dateadded:
                 self.nfo_manager.update_movie_files_mtime(movie_path, dateadded)
             
-            _log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [nfo-only]")
+            _log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [nfo-cached]")
             return
             
         # TIER 1.5: Special handling for TMDB-only movies - extract dates from existing NFO
@@ -155,68 +180,26 @@ class MovieProcessor:
                 _log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [tmdb-nfo]")
                 return
             
-        # TIER 2: Check database for existing data
-        existing = self.db.get_movie_dates(imdb_id)
-        _log("DEBUG", f"Database lookup for {imdb_id}: {existing}")
+        # TIER 3: No cached data found - proceed with API lookups and verification
         
-        # If we have complete data in database, use it and skip expensive API calls
-        if existing and existing.get("dateadded") and existing.get("source") != "no_valid_date_source":
-            _log("INFO", f"✅ Using complete database data for {imdb_id}: {existing['dateadded']} (source: {existing['source']})")
-            # Still create NFO and update files but skip API queries
-            dateadded, source, released = existing["dateadded"], existing["source"], existing.get("released")
-            
-            # Create NFO with existing data
-            print(f"🔍 TIER2 - config.manage_nfo: {config.manage_nfo}")
-            if config.manage_nfo:
-                print(f"🔍 TIER2 - Calling create_movie_nfo with dateadded: {dateadded}")
-                self.nfo_manager.create_movie_nfo(
-                    movie_path, imdb_id, dateadded, released, source, config.lock_metadata
-                )
-            else:
-                print(f"❌ TIER2 - manage_nfo is disabled, skipping NFO creation")
-            
-            # Update file mtimes if enabled
-            if config.fix_dir_mtimes and dateadded:
-                self.nfo_manager.update_movie_files_mtime(movie_path, dateadded)
-            
-            _log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [database-only]")
-            return
-        
-        # Handle webhook mode - prioritize database, then use proper date logic
+        # TIER 3: No cached data found - determine if we should query APIs
         if webhook_mode:
-            _log("DEBUG", f"Webhook mode: existing={bool(existing)}, has_dateadded={bool(existing and existing.get('dateadded')) if existing else 'N/A'}")
-            if existing and existing.get("dateadded"):
-                _log("INFO", f"Webhook processing - using existing database entry: {existing['dateadded']} (source: {existing.get('source', 'unknown')})")
-                dateadded, source, released = existing["dateadded"], existing["source"], existing.get("released")
+            _log("INFO", f"Webhook processing - no cached data found, using full date decision logic")
+            should_query = True  # Always query for webhooks when no cached data exists
         else:
-                if existing:
-                    _log("INFO", f"Webhook processing - database entry exists but no dateadded field: {existing}")
-                else:
-                    _log("INFO", f"Webhook processing - no database entry found for {imdb_id}")
-                _log("INFO", f"Using full date decision logic")
-                # Use same logic as manual scan to check Radarr import dates, release dates, etc.
-                should_query = True  # Always query for webhooks when no database entry exists
-                dateadded, source, released = self._decide_movie_dates(imdb_id, movie_path, should_query, existing)
+            # Manual scan mode - determine if we should query APIs
+            should_query = config.movie_poll_mode == "always"
+            _log("DEBUG", f"Movie {imdb_id}: should_query={should_query}, poll_mode={config.movie_poll_mode}")
         
-                # Only if ALL date sources fail, fall back to current timestamp
-                if dateadded is None:
+        # Use existing movie date decision logic
+        dateadded, source, released = self._decide_movie_dates(imdb_id, movie_path, should_query, None)
+        
+        # Webhook fallback: if ALL date sources fail, use current timestamp
+        if webhook_mode and dateadded is None:
             local_tz = _get_local_timezone()
             current_time = datetime.now(local_tz).isoformat(timespec="seconds")
             _log("INFO", f"Webhook processing - all date sources failed, using current timestamp as last resort: {current_time}")
             dateadded, source = current_time, "webhook:fallback_timestamp"
-        else:
-            # Manual scan mode - determine if we should query APIs
-            should_query = (
-                config.movie_poll_mode == "always" or
-                (config.movie_poll_mode == "if_missing" and not existing) or
-                (config.movie_poll_mode == "if_missing" and existing and existing.get("source") == "file:mtime") or
-                (config.movie_poll_mode == "if_missing" and existing and not existing.get("dateadded"))
-            )
-            
-            _log("DEBUG", f"Movie {imdb_id}: should_query={should_query}, poll_mode={config.movie_poll_mode}, existing={bool(existing)}, has_dateadded={bool(existing and existing.get('dateadded')) if existing else False}")
-            
-            # Use existing movie date decision logic
-            dateadded, source, released = self._decide_movie_dates(imdb_id, movie_path, should_query, existing)
         
         # If we don't have an import/download date but we have a release date, use it as dateadded
         # This ensures we save digital release dates, theatrical dates, etc. to the database

processors/tv_processor.py

@@ -106,9 +106,10 @@ class TVProcessor:
         episode_dates = {}
         episodes_needing_lookup = []
         
-        # OPTIMIZATION: Check database first for existing dates
-        _log("DEBUG", f"Checking database for existing episode dates for {len(disk_episodes)} episodes")
+        # TIER 1: Check database first for existing dates (fastest)
+        _log("DEBUG", f"TIER 1 - Checking database for existing episode dates for {len(disk_episodes)} episodes")
         db_cache_hits = 0
+        episodes_needing_nfo_check = []
         
         for (season, episode) in disk_episodes:
             # Try database first - this is much faster than API calls
@@ -123,14 +124,54 @@ class TVProcessor:
                 db_cache_hits += 1
                 _log("DEBUG", f"Database cache hit for S{season:02d}E{episode:02d}: {dateadded}")
             else:
-                # Not in database or incomplete - needs lookup
+                # Not in database or incomplete - needs NFO check
+                episodes_needing_nfo_check.append((season, episode))
+        
+        _log("INFO", f"Database cache hits: {db_cache_hits}/{len(disk_episodes)} episodes. Need NFO check: {len(episodes_needing_nfo_check)}")
+        
+        # TIER 2: Check NFO files for NFOGuard dates and cache them in database
+        nfo_cache_hits = 0
+        episodes_needing_lookup = []
+        
+        if episodes_needing_nfo_check:
+            _log("DEBUG", f"TIER 2 - Checking NFO files for NFOGuard dates for {len(episodes_needing_nfo_check)} episodes")
+            
+            for (season, episode) in episodes_needing_nfo_check:
+                # Look for existing NFO files for this episode
+                season_dir = series_path / config.tv_season_dir_format.format(season=season)
+                episode_files = disk_episodes[(season, episode)]
+                
+                nfo_found = False
+                for episode_file in episode_files:
+                    # Try to find matching NFO file
+                    nfo_path = episode_file.with_suffix('.nfo')
+                    if nfo_path.exists():
+                        # Extract NFOGuard data from episode NFO
+                        nfo_data = self.nfo_manager.extract_nfoguard_dates_from_nfo(nfo_path)
+                        if nfo_data:
+                            aired = nfo_data.get('aired') 
+                            dateadded = nfo_data.get('dateadded')
+                            source = nfo_data.get('source', 'nfo_cache')
+                            
+                            if dateadded:
+                                episode_dates[(season, episode)] = (aired, dateadded, source)
+                                nfo_cache_hits += 1
+                                nfo_found = True
+                                
+                                # Cache NFO data in database for future lookups
+                                self.db.upsert_episode_date(imdb_id, season, episode, aired, dateadded, source, True)
+                                _log("DEBUG", f"NFO cache hit for S{season:02d}E{episode:02d}: {dateadded} - cached in DB")
+                                break
+                
+                if not nfo_found:
+                    # No NFO data found - needs API lookup
                     episodes_needing_lookup.append((season, episode))
             
-        _log("INFO", f"Database cache hits: {db_cache_hits}/{len(disk_episodes)} episodes. Need API lookup: {len(episodes_needing_lookup)}")
+            _log("INFO", f"NFO cache hits: {nfo_cache_hits}/{len(episodes_needing_nfo_check)} episodes. Need API lookup: {len(episodes_needing_lookup)}")
         
-        # Only call Sonarr API for episodes not in database
+        # TIER 3: Only call Sonarr API for episodes not in database or NFO files
         if episodes_needing_lookup:
-            _log("DEBUG", f"Querying Sonarr for {len(episodes_needing_lookup)} episodes missing from database")
+            _log("DEBUG", f"TIER 3 - Querying Sonarr for {len(episodes_needing_lookup)} episodes missing from database and NFO files")
             sonarr_episodes = self._get_sonarr_episodes(imdb_id, episodes_needing_lookup)
             
             # Process episodes that needed lookup