From 035f2f6e02d694796b2ded144dc05f6de461ec3f Mon Sep 17 00:00:00 2001 From: sbcrumb Date: Fri, 10 Oct 2025 15:27:06 -0400 Subject: [PATCH 1/2] Optimize TV processor performance with database-first episode lookup Major performance improvement for manual scans: - Check NFOGuard database before making Sonarr API calls - Only query Sonarr history for episodes missing from database - Filter Sonarr episode processing to only needed episodes - Add detailed logging for cache hits vs API calls Expected improvement: 172 API calls -> 0-10 API calls for typical scans --- VERSION | 2 +- processors/tv_processor.py | 100 +++++++++++++++++++++++++------------ 2 files changed, 69 insertions(+), 33 deletions(-) diff --git a/VERSION b/VERSION index ed2af5c..d17d1f3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.19 +2.0.20 diff --git a/processors/tv_processor.py b/processors/tv_processor.py index 0e54de8..6d11216 100644 --- a/processors/tv_processor.py +++ b/processors/tv_processor.py @@ -102,47 +102,70 @@ class TVProcessor: def _gather_episode_dates(self, series_path: Path, imdb_id: str, disk_episodes: Dict[Tuple[int, int], List[Path]]) -> Dict[Tuple[int, int], Tuple[Optional[str], Optional[str], str]]: - """Gather episode air dates and date added information""" + """Gather episode air dates and date added information with database-first optimization""" episode_dates = {} + episodes_needing_lookup = [] - # Get data from Sonarr first if available - sonarr_episodes = self._get_sonarr_episodes(imdb_id) + # OPTIMIZATION: Check database first for existing dates + _log("DEBUG", f"Checking database for existing episode dates for {len(disk_episodes)} episodes") + db_cache_hits = 0 - # Get data from external sources for missing information for (season, episode) in disk_episodes: - aired = None - dateadded = None - source = "unknown" + # Try database first - this is much faster than API calls + db_result = self.db.get_episode_date(imdb_id, season, episode) - # Try Sonarr first - if (season, episode) in sonarr_episodes: - sonarr_data = sonarr_episodes[(season, episode)] - aired = sonarr_data.get('airDate') - dateadded = sonarr_data.get('dateAdded') - if dateadded: - # Use more specific source to indicate this came from Sonarr import history - source = "sonarr:history.import" + if db_result and db_result.get('dateadded'): + # Found in database - use cached data + aired = db_result.get('aired') + dateadded = db_result.get('dateadded') + source = db_result.get('source', 'database_cache') + episode_dates[(season, episode)] = (aired, dateadded, source) + db_cache_hits += 1 + _log("DEBUG", f"Database cache hit for S{season:02d}E{episode:02d}: {dateadded}") + else: + # Not in database or incomplete - needs lookup + episodes_needing_lookup.append((season, episode)) + + _log("INFO", f"Database cache hits: {db_cache_hits}/{len(disk_episodes)} episodes. Need API lookup: {len(episodes_needing_lookup)}") + + # Only call Sonarr API for episodes not in database + if episodes_needing_lookup: + _log("DEBUG", f"Querying Sonarr for {len(episodes_needing_lookup)} episodes missing from database") + sonarr_episodes = self._get_sonarr_episodes(imdb_id, episodes_needing_lookup) - # Fallback to external sources if needed - if not aired: - external_aired = self.external_clients.get_episode_air_date(imdb_id, season, episode) - if external_aired: - aired = external_aired - if not dateadded: - source = "external" - - # ONLY use air date as fallback for dateadded if we have no Sonarr import date at all - # and if we couldn't get any import information from Sonarr history - if not dateadded and aired and source != "sonarr:history.import": - dateadded = aired - source = f"{source}_fallback" if source != "unknown" else "aired_fallback" - - episode_dates[(season, episode)] = (aired, dateadded, source) + # Process episodes that needed lookup + for (season, episode) in episodes_needing_lookup: + aired = None + dateadded = None + source = "unknown" + + # Try Sonarr first + if (season, episode) in sonarr_episodes: + sonarr_data = sonarr_episodes[(season, episode)] + aired = sonarr_data.get('airDate') + dateadded = sonarr_data.get('dateAdded') + if dateadded: + source = "sonarr:history.import" + + # Fallback to external sources if needed + if not aired: + external_aired = self.external_clients.get_episode_air_date(imdb_id, season, episode) + if external_aired: + aired = external_aired + if not dateadded: + source = "external" + + # Use air date as fallback for dateadded if no import date found + if not dateadded and aired and source != "sonarr:history.import": + dateadded = aired + source = f"{source}_fallback" if source != "unknown" else "aired_fallback" + + episode_dates[(season, episode)] = (aired, dateadded, source) return episode_dates - def _get_sonarr_episodes(self, imdb_id: str) -> Dict[Tuple[int, int], Dict[str, Any]]: - """Get episode information from Sonarr including import history""" + def _get_sonarr_episodes(self, imdb_id: str, episodes_filter: List[Tuple[int, int]] = None) -> Dict[Tuple[int, int], Dict[str, Any]]: + """Get episode information from Sonarr including import history - optimized to only fetch needed episodes""" try: series_data = self.sonarr.series_by_imdb(imdb_id) if not series_data: @@ -154,11 +177,20 @@ class TVProcessor: episodes = self.sonarr.episodes_for_series(series_id) + # Convert episodes_filter to set for faster lookup + filter_set = set(episodes_filter) if episodes_filter else None + episode_map = {} + api_calls_made = 0 + for episode in episodes: season = episode.get('seasonNumber', 0) episode_num = episode.get('episodeNumber', 0) + # Skip episodes not in filter if filter is provided + if filter_set and (season, episode_num) not in filter_set: + continue + if season > 0 and episode_num > 0: # Get basic episode info episode_data = { @@ -170,6 +202,7 @@ class TVProcessor: episode_id = episode.get('id') if episode_id and episode.get('hasFile'): import_date = self.sonarr.get_episode_import_history(episode_id) + api_calls_made += 1 if import_date: episode_data['dateAdded'] = import_date _log("DEBUG", f"Got import date from history for S{season:02d}E{episode_num:02d}: {import_date}") @@ -183,6 +216,9 @@ class TVProcessor: episode_map[(season, episode_num)] = episode_data + if filter_set: + _log("DEBUG", f"Made {api_calls_made} Sonarr history API calls for filtered episodes (instead of all episodes)") + return episode_map except Exception as e: From 88b75230a78da71b30ef12cdb18d6f2995ef14de Mon Sep 17 00:00:00 2001 From: sbcrumb Date: Fri, 10 Oct 2025 15:28:45 -0400 Subject: [PATCH 2/2] Bump version to 2.0.21 to resolve merge conflict with dev branch --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index d17d1f3..7d5624f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.20 +2.0.21