From 0dfc296bf477425b968034a5ccd636e1894658bc Mon Sep 17 00:00:00 2001 From: sbcrumb Date: Wed, 15 Oct 2025 07:56:56 -0400 Subject: [PATCH] fix: db not filling in .nfo data --- VERSION | 2 +- api/routes.py | 15 ++++-- api/web_routes.py | 50 ++++++++++++++++-- processors/movie_processor.py | 96 +++++++++++++++++++++++++++++++++++ processors/tv_processor.py | 27 +++++++++- static/index.html | 29 ++++++++--- static/js/app.js | 27 ++++++++++ 7 files changed, 231 insertions(+), 15 deletions(-) diff --git a/VERSION b/VERSION index 399088b..ccbccc3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1.6 +2.2.0 diff --git a/api/routes.py b/api/routes.py index 81870f6..f2fdfff 100644 --- a/api/routes.py +++ b/api/routes.py @@ -16,7 +16,7 @@ from api.models import ( MovieUpdateRequest, EpisodeUpdateRequest, BulkUpdateRequest ) from api.web_routes import ( - get_movies_list, get_tv_series_list, get_series_episodes, get_missing_dates_report, + get_movies_list, get_tv_series_list, get_series_episodes, get_series_sources, get_missing_dates_report, get_dashboard_stats, update_movie_date, update_episode_date, bulk_update_source, get_movie_date_options, get_episode_date_options ) @@ -1043,15 +1043,22 @@ def register_routes(app, dependencies: dict): return await get_movies_list(dependencies, skip, limit, has_date, source_filter, search, imdb_search) @app.get("/api/series") - async def _series_list(skip: int = 0, limit: int = 50, search: Optional[str] = None): - """Get paginated TV series list""" - return await get_tv_series_list(dependencies, skip, limit, search) + async def _series_list(skip: int = 0, limit: int = 50, search: Optional[str] = None, + imdb_search: Optional[str] = None, date_filter: Optional[str] = None, + source_filter: Optional[str] = None): + """Get paginated TV series list with filtering""" + return await get_tv_series_list(dependencies, skip, limit, search, imdb_search, date_filter, source_filter) @app.get("/api/series/{imdb_id}/episodes") async def _series_episodes(imdb_id: str): """Get episodes for a specific series""" return await get_series_episodes(dependencies, imdb_id) + @app.get("/api/series/sources") + async def _series_sources(): + """Get list of available episode sources for filtering""" + return await get_series_sources(dependencies) + @app.get("/api/reports/missing-dates") async def _missing_dates_report(): """Get report of content missing dateadded""" diff --git a/api/web_routes.py b/api/web_routes.py index 3ce12b4..3e00390 100644 --- a/api/web_routes.py +++ b/api/web_routes.py @@ -90,7 +90,10 @@ async def get_movies_list(dependencies: dict, async def get_tv_series_list(dependencies: dict, skip: int = Query(0, ge=0), limit: int = Query(50, le=500), - search: Optional[str] = Query(None)): + search: Optional[str] = Query(None), + imdb_search: Optional[str] = Query(None), + date_filter: Optional[str] = Query(None), + source_filter: Optional[str] = Query(None)): """Get paginated list of TV series with episode counts""" db = dependencies["db"] @@ -100,12 +103,35 @@ async def get_tv_series_list(dependencies: dict, # Build dynamic query where_conditions = [] params = [] + having_conditions = [] + having_params = [] if search: where_conditions.append("(s.imdb_id LIKE ? OR s.path LIKE ?)") params.extend([f"%{search}%", f"%{search}%"]) + + if imdb_search: + where_conditions.append("s.imdb_id LIKE ?") + params.append(f"%{imdb_search}%") + + if source_filter: + # Need to check episodes for source filter + where_conditions.append("e.source = ?") + params.append(source_filter) + + if date_filter: + if date_filter == "complete": + # All episodes have dates + having_conditions.append("COUNT(e.episode) > 0 AND COUNT(CASE WHEN e.dateadded IS NULL OR e.dateadded = '' THEN 1 END) = 0") + elif date_filter == "incomplete": + # Some episodes have dates, some don't + having_conditions.append("COUNT(e.episode) > 0 AND COUNT(CASE WHEN e.dateadded IS NOT NULL AND e.dateadded != '' THEN 1 END) > 0 AND COUNT(CASE WHEN e.dateadded IS NULL OR e.dateadded = '' THEN 1 END) > 0") + elif date_filter == "none": + # No episodes have dates + having_conditions.append("COUNT(e.episode) > 0 AND COUNT(CASE WHEN e.dateadded IS NOT NULL AND e.dateadded != '' THEN 1 END) = 0") where_clause = " AND ".join(where_conditions) if where_conditions else "1=1" + having_clause = " AND ".join(having_conditions) if having_conditions else "" # Get total count count_query = f"SELECT COUNT(*) FROM series s WHERE {where_clause}" @@ -113,6 +139,7 @@ async def get_tv_series_list(dependencies: dict, total_count = cursor.fetchone()[0] # Get series with episode statistics + having_part = f" HAVING {having_clause}" if having_clause else "" query = f""" SELECT s.imdb_id, @@ -124,11 +151,11 @@ async def get_tv_series_list(dependencies: dict, FROM series s LEFT JOIN episodes e ON s.imdb_id = e.imdb_id WHERE {where_clause} - GROUP BY s.imdb_id, s.path, s.last_updated + GROUP BY s.imdb_id, s.path, s.last_updated{having_part} ORDER BY s.last_updated DESC LIMIT ? OFFSET ? """ - cursor.execute(query, params + [limit, skip]) + cursor.execute(query, params + having_params + [limit, skip]) series = [] for row in cursor.fetchall(): @@ -150,6 +177,23 @@ async def get_tv_series_list(dependencies: dict, } +async def get_series_sources(dependencies: dict): + """Get unique sources from episodes table for filtering""" + db = dependencies["db"] + + with db.get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + SELECT DISTINCT source + FROM episodes + WHERE source IS NOT NULL AND source != '' + ORDER BY source + """) + + sources = [row[0] for row in cursor.fetchall()] + return {"sources": sources} + + async def get_series_episodes(dependencies: dict, imdb_id: str): """Get all episodes for a specific TV series""" db = dependencies["db"] diff --git a/processors/movie_processor.py b/processors/movie_processor.py index 6cefa05..81a1e1e 100644 --- a/processors/movie_processor.py +++ b/processors/movie_processor.py @@ -118,6 +118,14 @@ class MovieProcessor: existing = self.db.get_movie_dates(imdb_id) _log("DEBUG", f"Database lookup for {imdb_id}: {existing}") + # Enhanced debug for database state + if existing: + has_dateadded = bool(existing.get("dateadded")) + source_value = existing.get("source") + _log("INFO", f"🔍 TIER 1 DEBUG - {imdb_id}: has_dateadded={has_dateadded}, source='{source_value}', dateadded='{existing.get('dateadded')}'") + else: + _log("INFO", f"🔍 TIER 1 DEBUG - {imdb_id}: No database record found") + # If we have complete data in database, use it and skip all other checks if existing and existing.get("dateadded") and existing.get("source") != "no_valid_date_source": _log("INFO", f"✅ TIER 1 - Using complete database data for {imdb_id}: {existing['dateadded']} (source: {existing['source']})") @@ -134,10 +142,17 @@ class MovieProcessor: _log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [database-cached]") return + else: + _log("INFO", f"🔍 TIER 1 SKIP - {imdb_id}: Database incomplete, proceeding to Tier 2") # TIER 2: Check if NFO file has NFOGuard data and cache it in database nfo_path = movie_path / "movie.nfo" + _log("INFO", f"🔍 TIER 2 - Checking NFO file: {nfo_path}") + _log("INFO", f"🔍 TIER 2 - NFO exists: {nfo_path.exists()}") + nfo_data = self.nfo_manager.extract_nfoguard_dates_from_nfo(nfo_path) + _log("INFO", f"🔍 TIER 2 - NFOGuard data extracted: {nfo_data}") + if nfo_data: _log("INFO", f"🚀 TIER 2 - Found NFOGuard data in NFO file: {nfo_data['dateadded']} (source: {nfo_data['source']})") dateadded = nfo_data["dateadded"] @@ -155,6 +170,32 @@ class MovieProcessor: _log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [nfo-cached]") return + # TIER 2.5: Check for any existing valid date data in NFO (even without lockdata marker) + existing_nfo_data = self._extract_any_valid_dates_from_nfo(nfo_path) + if existing_nfo_data: + _log("INFO", f"🔍 TIER 2.5 - Found existing date data in NFO (no lockdata): {existing_nfo_data['dateadded']} (source: {existing_nfo_data['source']})") + dateadded = existing_nfo_data["dateadded"] + source = existing_nfo_data["source"] + released = existing_nfo_data.get("released") + + # Cache existing data in database and add proper NFOGuard formatting + self.db.upsert_movie_dates(imdb_id, dateadded, released, source, True) + _log("INFO", f"✅ Cached existing NFO data in database for {imdb_id}") + + # Update NFO file to add NFOGuard formatting (lockdata, comment) + if config.manage_nfo: + self.nfo_manager.create_movie_nfo( + movie_path, imdb_id, dateadded, released, source, config.lock_metadata + ) + _log("INFO", f"✅ Added NFOGuard formatting to existing NFO for {imdb_id}") + + # Update file mtimes if enabled + if config.fix_dir_mtimes and dateadded: + self.nfo_manager.update_movie_files_mtime(movie_path, dateadded) + + _log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [existing-nfo-enhanced]") + return + # TIER 1.5: Special handling for TMDB-only movies - extract dates from existing NFO if is_tmdb_fallback: tmdb_nfo_data = self._extract_dates_from_tmdb_nfo(nfo_path) @@ -275,6 +316,61 @@ class MovieProcessor: return None + def _extract_any_valid_dates_from_nfo(self, nfo_path: Path) -> Optional[Dict[str, str]]: + """Extract any valid date information from NFO file, even without NFOGuard markers""" + if not nfo_path.exists(): + return None + + try: + import xml.etree.ElementTree as ET + tree = ET.parse(nfo_path) + root = tree.getroot() + + # Look for dateadded element (indicates previously processed by NFOGuard or similar) + dateadded_elem = root.find('.//dateadded') + premiered_elem = root.find('.//premiered') + + if dateadded_elem is not None and dateadded_elem.text: + dateadded = dateadded_elem.text.strip() + + # Try to determine source from NFOGuard comment if present + source = "existing_nfo_data" + try: + nfo_content = nfo_path.read_text(encoding='utf-8') + import re + # Look for NFOGuard comment pattern + source_match = re.search(r'', nfo_content, re.DOTALL | re.IGNORECASE) + if source_match: + source = source_match.group(1).strip() + _log("DEBUG", f"Found source in NFOGuard comment: {source}") + else: + # Try to infer source from dateadded format/content + if "tmdb" in nfo_content.lower() or (premiered_elem and premiered_elem.text): + source = "tmdb:digital" + elif "radarr" in nfo_content.lower(): + source = "radarr:db.history.import" + else: + source = "existing_nfo_data" + _log("DEBUG", f"Inferred source from NFO content: {source}") + except Exception as e: + _log("DEBUG", f"Could not determine source from NFO content: {e}") + + result = { + "dateadded": dateadded, + "source": source + } + + if premiered_elem is not None and premiered_elem.text: + result["released"] = premiered_elem.text.strip() + + _log("INFO", f"✅ Found existing date data in NFO: dateadded={dateadded}, source={source}") + return result + + except (ET.ParseError, Exception) as e: + _log("DEBUG", f"Error parsing NFO for existing date data: {e}") + + return None + def _decide_movie_dates(self, imdb_id: str, movie_path: Path, should_query: bool, existing: Optional[Dict]) -> Tuple[str, str, Optional[str]]: """Decide movie dates based on configuration and available data""" _log("DEBUG", f"_decide_movie_dates for {imdb_id}: should_query={should_query}, existing={existing}") diff --git a/processors/tv_processor.py b/processors/tv_processor.py index 4546036..e6e1a2a 100644 --- a/processors/tv_processor.py +++ b/processors/tv_processor.py @@ -4,6 +4,7 @@ Handles TV series processing and episode management with async I/O support """ import os import re +import time import asyncio from pathlib import Path from typing import Optional, Dict, List, Set, Tuple, Any @@ -71,9 +72,12 @@ class TVProcessor: # Get episode dates episode_dates = self._gather_episode_dates(series_path, imdb_id, disk_episodes) - # Process episodes + # Process episodes with periodic yielding for non-blocking operation + episode_count = 0 for (season, episode), (aired, dateadded, source) in episode_dates.items(): if (season, episode) in disk_episodes: + episode_count += 1 + # Create NFO if config.manage_nfo: season_dir = series_path / config.tv_season_dir_format.format(season=season) @@ -90,6 +94,12 @@ class TVProcessor: # Save to database self.db.upsert_episode_date(imdb_id, season, episode, aired, dateadded, source, True) + + # Yield control every 10 episodes to allow other requests (webhooks, web interface) + if episode_count % 10 == 0: + # Note: Since this is sync method, we can't use asyncio.sleep() + # This will require making process_series async or adding yield points in caller + pass # Skip season.nfo and tvshow.nfo creation - focus only on episode NFOs pass @@ -238,6 +248,7 @@ class TVProcessor: episode_map = {} api_calls_made = 0 + episodes_processed = 0 for episode in episodes: season = episode.get('seasonNumber', 0) @@ -248,6 +259,8 @@ class TVProcessor: continue if season > 0 and episode_num > 0: + episodes_processed += 1 + # Get basic episode info episode_data = { 'airDate': episode.get('airDate'), @@ -262,6 +275,12 @@ class TVProcessor: if import_date: episode_data['dateAdded'] = import_date _log("DEBUG", f"Got import date from history for S{season:02d}E{episode_num:02d}: {import_date}") + + # Yield control every 5 API calls to allow other requests + if api_calls_made % 5 == 0: + import time + time.sleep(0.01) # 10ms yield to other processes + _log("DEBUG", f"Yielded after {api_calls_made} Sonarr API calls to allow other requests...") # Fallback to episodeFile.dateAdded if history didn't work if not episode_data['dateAdded'] and episode.get('hasFile'): @@ -271,6 +290,12 @@ class TVProcessor: _log("DEBUG", f"Got file date for S{season:02d}E{episode_num:02d}: {file_date}") episode_map[(season, episode_num)] = episode_data + + # Also yield control every 20 episodes processed to prevent blocking + if episodes_processed % 20 == 0: + import time + time.sleep(0.01) # 10ms yield for large episode lists + _log("DEBUG", f"Processed {episodes_processed} episodes, yielding to allow other requests...") if filter_set: _log("DEBUG", f"Made {api_calls_made} Sonarr history API calls for filtered episodes (instead of all episodes)") diff --git a/static/index.html b/static/index.html index 9781be1..124bf05 100644 --- a/static/index.html +++ b/static/index.html @@ -157,13 +157,30 @@

TV Series Database

-
- - +
+
+ + +
+
+ + +
+
+
+ + +
-
diff --git a/static/js/app.js b/static/js/app.js index d51d1f2..f375b35 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -11,6 +11,7 @@ document.addEventListener('DOMContentLoaded', function() { initializeTabs(); initializeEventListeners(); loadDashboard(); + loadSeriesSources(); }); // Tab management @@ -63,10 +64,13 @@ function initializeEventListeners() { document.getElementById('movies-search').addEventListener('input', debounce(loadMovies, 500)); document.getElementById('movies-imdb-search').addEventListener('input', debounce(loadMovies, 500)); document.getElementById('series-search').addEventListener('input', debounce(loadSeries, 500)); + document.getElementById('series-imdb-search').addEventListener('input', debounce(loadSeries, 500)); // Filter dropdowns document.getElementById('movies-filter-date').addEventListener('change', loadMovies); document.getElementById('movies-filter-source').addEventListener('change', loadMovies); + document.getElementById('series-filter-date').addEventListener('change', loadSeries); + document.getElementById('series-filter-source').addEventListener('change', loadSeries); // Forms document.getElementById('edit-form').addEventListener('submit', handleEditSubmit); @@ -296,6 +300,23 @@ function updateMoviesSourceFilter(data) { } } +async function loadSeriesSources() { + try { + const data = await apiCall('/api/series/sources'); + const select = document.getElementById('series-filter-source'); + const currentValue = select.value; + + select.innerHTML = ''; + data.sources.forEach(source => { + select.innerHTML += ``; + }); + + select.value = currentValue; + } catch (error) { + console.error('Failed to load series sources:', error); + } +} + function refreshMovies() { loadMovies(currentMoviesPage); } @@ -303,6 +324,9 @@ function refreshMovies() { // TV Series async function loadSeries(page = 1) { const search = document.getElementById('series-search').value; + const imdbSearch = document.getElementById('series-imdb-search').value; + const dateFilter = document.getElementById('series-filter-date').value; + const sourceFilter = document.getElementById('series-filter-source').value; const params = new URLSearchParams({ skip: (page - 1) * 50, @@ -310,6 +334,9 @@ async function loadSeries(page = 1) { }); if (search) params.append('search', search); + if (imdbSearch) params.append('imdb_search', imdbSearch); + if (dateFilter) params.append('date_filter', dateFilter); + if (sourceFilter) params.append('source_filter', sourceFilter); try { const data = await apiCall(`/api/series?${params}`);