fix: db not filling in .nfo data

VERSION

@@ -1 +1 @@
-2.1.6
+2.2.0

api/routes.py

@@ -16,7 +16,7 @@ from api.models import (
     MovieUpdateRequest, EpisodeUpdateRequest, BulkUpdateRequest
 )
 from api.web_routes import (
-    get_movies_list, get_tv_series_list, get_series_episodes, get_missing_dates_report,
+    get_movies_list, get_tv_series_list, get_series_episodes, get_series_sources, get_missing_dates_report,
     get_dashboard_stats, update_movie_date, update_episode_date, bulk_update_source,
     get_movie_date_options, get_episode_date_options
 )
@@ -1043,15 +1043,22 @@ def register_routes(app, dependencies: dict):
         return await get_movies_list(dependencies, skip, limit, has_date, source_filter, search, imdb_search)
     
     @app.get("/api/series")
-    async def _series_list(skip: int = 0, limit: int = 50, search: Optional[str] = None):
-        """Get paginated TV series list"""
-        return await get_tv_series_list(dependencies, skip, limit, search)
+    async def _series_list(skip: int = 0, limit: int = 50, search: Optional[str] = None, 
+                          imdb_search: Optional[str] = None, date_filter: Optional[str] = None, 
+                          source_filter: Optional[str] = None):
+        """Get paginated TV series list with filtering"""
+        return await get_tv_series_list(dependencies, skip, limit, search, imdb_search, date_filter, source_filter)
     
     @app.get("/api/series/{imdb_id}/episodes")
     async def _series_episodes(imdb_id: str):
         """Get episodes for a specific series"""
         return await get_series_episodes(dependencies, imdb_id)
     
+    @app.get("/api/series/sources")
+    async def _series_sources():
+        """Get list of available episode sources for filtering"""
+        return await get_series_sources(dependencies)
+    
     @app.get("/api/reports/missing-dates")
     async def _missing_dates_report():
         """Get report of content missing dateadded"""

api/web_routes.py

@@ -90,7 +90,10 @@ async def get_movies_list(dependencies: dict,
 async def get_tv_series_list(dependencies: dict,
                            skip: int = Query(0, ge=0), 
                            limit: int = Query(50, le=500),
-                           search: Optional[str] = Query(None)):
+                           search: Optional[str] = Query(None),
+                           imdb_search: Optional[str] = Query(None),
+                           date_filter: Optional[str] = Query(None),
+                           source_filter: Optional[str] = Query(None)):
     """Get paginated list of TV series with episode counts"""
     db = dependencies["db"]
     
@@ -100,12 +103,35 @@ async def get_tv_series_list(dependencies: dict,
         # Build dynamic query
         where_conditions = []
         params = []
+        having_conditions = []
+        having_params = []
         
         if search:
             where_conditions.append("(s.imdb_id LIKE ? OR s.path LIKE ?)")
             params.extend([f"%{search}%", f"%{search}%"])
+            
+        if imdb_search:
+            where_conditions.append("s.imdb_id LIKE ?")
+            params.append(f"%{imdb_search}%")
+            
+        if source_filter:
+            # Need to check episodes for source filter
+            where_conditions.append("e.source = ?")
+            params.append(source_filter)
+            
+        if date_filter:
+            if date_filter == "complete":
+                # All episodes have dates
+                having_conditions.append("COUNT(e.episode) > 0 AND COUNT(CASE WHEN e.dateadded IS NULL OR e.dateadded = '' THEN 1 END) = 0")
+            elif date_filter == "incomplete":
+                # Some episodes have dates, some don't
+                having_conditions.append("COUNT(e.episode) > 0 AND COUNT(CASE WHEN e.dateadded IS NOT NULL AND e.dateadded != '' THEN 1 END) > 0 AND COUNT(CASE WHEN e.dateadded IS NULL OR e.dateadded = '' THEN 1 END) > 0")
+            elif date_filter == "none":
+                # No episodes have dates
+                having_conditions.append("COUNT(e.episode) > 0 AND COUNT(CASE WHEN e.dateadded IS NOT NULL AND e.dateadded != '' THEN 1 END) = 0")
         
         where_clause = " AND ".join(where_conditions) if where_conditions else "1=1"
+        having_clause = " AND ".join(having_conditions) if having_conditions else ""
         
         # Get total count
         count_query = f"SELECT COUNT(*) FROM series s WHERE {where_clause}"
@@ -113,6 +139,7 @@ async def get_tv_series_list(dependencies: dict,
         total_count = cursor.fetchone()[0]
         
         # Get series with episode statistics
+        having_part = f" HAVING {having_clause}" if having_clause else ""
         query = f"""
             SELECT 
                 s.imdb_id, 
@@ -124,11 +151,11 @@ async def get_tv_series_list(dependencies: dict,
             FROM series s
             LEFT JOIN episodes e ON s.imdb_id = e.imdb_id
             WHERE {where_clause}
-            GROUP BY s.imdb_id, s.path, s.last_updated
+            GROUP BY s.imdb_id, s.path, s.last_updated{having_part}
             ORDER BY s.last_updated DESC
             LIMIT ? OFFSET ?
         """
-        cursor.execute(query, params + [limit, skip])
+        cursor.execute(query, params + having_params + [limit, skip])
         
         series = []
         for row in cursor.fetchall():
@@ -150,6 +177,23 @@ async def get_tv_series_list(dependencies: dict,
         }
 
 
+async def get_series_sources(dependencies: dict):
+    """Get unique sources from episodes table for filtering"""
+    db = dependencies["db"]
+    
+    with db.get_connection() as conn:
+        cursor = conn.cursor()
+        cursor.execute("""
+            SELECT DISTINCT source 
+            FROM episodes 
+            WHERE source IS NOT NULL AND source != '' 
+            ORDER BY source
+        """)
+        
+        sources = [row[0] for row in cursor.fetchall()]
+        return {"sources": sources}
+
+
 async def get_series_episodes(dependencies: dict, imdb_id: str):
     """Get all episodes for a specific TV series"""
     db = dependencies["db"]

processors/movie_processor.py

@@ -118,6 +118,14 @@ class MovieProcessor:
         existing = self.db.get_movie_dates(imdb_id)
         _log("DEBUG", f"Database lookup for {imdb_id}: {existing}")
         
+        # Enhanced debug for database state
+        if existing:
+            has_dateadded = bool(existing.get("dateadded"))
+            source_value = existing.get("source")
+            _log("INFO", f"🔍 TIER 1 DEBUG - {imdb_id}: has_dateadded={has_dateadded}, source='{source_value}', dateadded='{existing.get('dateadded')}'")
+        else:
+            _log("INFO", f"🔍 TIER 1 DEBUG - {imdb_id}: No database record found")
+        
         # If we have complete data in database, use it and skip all other checks
         if existing and existing.get("dateadded") and existing.get("source") != "no_valid_date_source":
             _log("INFO", f"✅ TIER 1 - Using complete database data for {imdb_id}: {existing['dateadded']} (source: {existing['source']})")
@@ -134,10 +142,17 @@ class MovieProcessor:
             
             _log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [database-cached]")
             return
+        else:
+            _log("INFO", f"🔍 TIER 1 SKIP - {imdb_id}: Database incomplete, proceeding to Tier 2")
         
         # TIER 2: Check if NFO file has NFOGuard data and cache it in database
         nfo_path = movie_path / "movie.nfo"
+        _log("INFO", f"🔍 TIER 2 - Checking NFO file: {nfo_path}")
+        _log("INFO", f"🔍 TIER 2 - NFO exists: {nfo_path.exists()}")
+        
         nfo_data = self.nfo_manager.extract_nfoguard_dates_from_nfo(nfo_path)
+        _log("INFO", f"🔍 TIER 2 - NFOGuard data extracted: {nfo_data}")
+        
         if nfo_data:
             _log("INFO", f"🚀 TIER 2 - Found NFOGuard data in NFO file: {nfo_data['dateadded']} (source: {nfo_data['source']})")
             dateadded = nfo_data["dateadded"]
@@ -155,6 +170,32 @@ class MovieProcessor:
             _log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [nfo-cached]")
             return
             
+        # TIER 2.5: Check for any existing valid date data in NFO (even without lockdata marker)
+        existing_nfo_data = self._extract_any_valid_dates_from_nfo(nfo_path)
+        if existing_nfo_data:
+            _log("INFO", f"🔍 TIER 2.5 - Found existing date data in NFO (no lockdata): {existing_nfo_data['dateadded']} (source: {existing_nfo_data['source']})")
+            dateadded = existing_nfo_data["dateadded"]
+            source = existing_nfo_data["source"]
+            released = existing_nfo_data.get("released")
+            
+            # Cache existing data in database and add proper NFOGuard formatting
+            self.db.upsert_movie_dates(imdb_id, dateadded, released, source, True)
+            _log("INFO", f"✅ Cached existing NFO data in database for {imdb_id}")
+            
+            # Update NFO file to add NFOGuard formatting (lockdata, comment)
+            if config.manage_nfo:
+                self.nfo_manager.create_movie_nfo(
+                    movie_path, imdb_id, dateadded, released, source, config.lock_metadata
+                )
+                _log("INFO", f"✅ Added NFOGuard formatting to existing NFO for {imdb_id}")
+            
+            # Update file mtimes if enabled
+            if config.fix_dir_mtimes and dateadded:
+                self.nfo_manager.update_movie_files_mtime(movie_path, dateadded)
+            
+            _log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [existing-nfo-enhanced]")
+            return
+            
         # TIER 1.5: Special handling for TMDB-only movies - extract dates from existing NFO
         if is_tmdb_fallback:
             tmdb_nfo_data = self._extract_dates_from_tmdb_nfo(nfo_path)
@@ -275,6 +316,61 @@ class MovieProcessor:
             
         return None
     
+    def _extract_any_valid_dates_from_nfo(self, nfo_path: Path) -> Optional[Dict[str, str]]:
+        """Extract any valid date information from NFO file, even without NFOGuard markers"""
+        if not nfo_path.exists():
+            return None
+            
+        try:
+            import xml.etree.ElementTree as ET
+            tree = ET.parse(nfo_path)
+            root = tree.getroot()
+            
+            # Look for dateadded element (indicates previously processed by NFOGuard or similar)
+            dateadded_elem = root.find('.//dateadded')
+            premiered_elem = root.find('.//premiered')
+            
+            if dateadded_elem is not None and dateadded_elem.text:
+                dateadded = dateadded_elem.text.strip()
+                
+                # Try to determine source from NFOGuard comment if present
+                source = "existing_nfo_data"
+                try:
+                    nfo_content = nfo_path.read_text(encoding='utf-8')
+                    import re
+                    # Look for NFOGuard comment pattern
+                    source_match = re.search(r'<!--.*?NFOGuard.*?Source:\s*([^-\s]+).*?-->', nfo_content, re.DOTALL | re.IGNORECASE)
+                    if source_match:
+                        source = source_match.group(1).strip()
+                        _log("DEBUG", f"Found source in NFOGuard comment: {source}")
+                    else:
+                        # Try to infer source from dateadded format/content
+                        if "tmdb" in nfo_content.lower() or (premiered_elem and premiered_elem.text):
+                            source = "tmdb:digital"
+                        elif "radarr" in nfo_content.lower():
+                            source = "radarr:db.history.import"
+                        else:
+                            source = "existing_nfo_data"
+                        _log("DEBUG", f"Inferred source from NFO content: {source}")
+                except Exception as e:
+                    _log("DEBUG", f"Could not determine source from NFO content: {e}")
+                
+                result = {
+                    "dateadded": dateadded,
+                    "source": source
+                }
+                
+                if premiered_elem is not None and premiered_elem.text:
+                    result["released"] = premiered_elem.text.strip()
+                
+                _log("INFO", f"✅ Found existing date data in NFO: dateadded={dateadded}, source={source}")
+                return result
+                
+        except (ET.ParseError, Exception) as e:
+            _log("DEBUG", f"Error parsing NFO for existing date data: {e}")
+            
+        return None
+    
     def _decide_movie_dates(self, imdb_id: str, movie_path: Path, should_query: bool, existing: Optional[Dict]) -> Tuple[str, str, Optional[str]]:
         """Decide movie dates based on configuration and available data"""
         _log("DEBUG", f"_decide_movie_dates for {imdb_id}: should_query={should_query}, existing={existing}")

processors/tv_processor.py

@@ -4,6 +4,7 @@ Handles TV series processing and episode management with async I/O support
 """
 import os
 import re
+import time
 import asyncio
 from pathlib import Path
 from typing import Optional, Dict, List, Set, Tuple, Any
@@ -71,9 +72,12 @@ class TVProcessor:
         # Get episode dates
         episode_dates = self._gather_episode_dates(series_path, imdb_id, disk_episodes)
         
-        # Process episodes
+        # Process episodes with periodic yielding for non-blocking operation
+        episode_count = 0
         for (season, episode), (aired, dateadded, source) in episode_dates.items():
             if (season, episode) in disk_episodes:
+                episode_count += 1
+                
                 # Create NFO
                 if config.manage_nfo:
                     season_dir = series_path / config.tv_season_dir_format.format(season=season)
@@ -90,6 +94,12 @@ class TVProcessor:
                 
                 # Save to database
                 self.db.upsert_episode_date(imdb_id, season, episode, aired, dateadded, source, True)
+                
+                # Yield control every 10 episodes to allow other requests (webhooks, web interface)
+                if episode_count % 10 == 0:
+                    # Note: Since this is sync method, we can't use asyncio.sleep()
+                    # This will require making process_series async or adding yield points in caller
+                    pass
         
         # Skip season.nfo and tvshow.nfo creation - focus only on episode NFOs
         pass
@@ -238,6 +248,7 @@ class TVProcessor:
             
             episode_map = {}
             api_calls_made = 0
+            episodes_processed = 0
             
             for episode in episodes:
                 season = episode.get('seasonNumber', 0)
@@ -248,6 +259,8 @@ class TVProcessor:
                     continue
                 
                 if season > 0 and episode_num > 0:
+                    episodes_processed += 1
+                    
                     # Get basic episode info
                     episode_data = {
                         'airDate': episode.get('airDate'),
@@ -262,6 +275,12 @@ class TVProcessor:
                         if import_date:
                             episode_data['dateAdded'] = import_date
                             _log("DEBUG", f"Got import date from history for S{season:02d}E{episode_num:02d}: {import_date}")
+                        
+                        # Yield control every 5 API calls to allow other requests
+                        if api_calls_made % 5 == 0:
+                            import time
+                            time.sleep(0.01)  # 10ms yield to other processes
+                            _log("DEBUG", f"Yielded after {api_calls_made} Sonarr API calls to allow other requests...")
                     
                     # Fallback to episodeFile.dateAdded if history didn't work
                     if not episode_data['dateAdded'] and episode.get('hasFile'):
@@ -271,6 +290,12 @@ class TVProcessor:
                             _log("DEBUG", f"Got file date for S{season:02d}E{episode_num:02d}: {file_date}")
                     
                     episode_map[(season, episode_num)] = episode_data
+                    
+                    # Also yield control every 20 episodes processed to prevent blocking
+                    if episodes_processed % 20 == 0:
+                        import time
+                        time.sleep(0.01)  # 10ms yield for large episode lists
+                        _log("DEBUG", f"Processed {episodes_processed} episodes, yielding to allow other requests...")
             
             if filter_set:
                 _log("DEBUG", f"Made {api_calls_made} Sonarr history API calls for filtered episodes (instead of all episodes)")

static/index.html

@@ -157,13 +157,30 @@
                 <div class="content-header">
                     <h2><i class="fas fa-tv"></i> TV Series Database</h2>
                     <div class="content-controls">
-                        <div class="search-box">
-                            <i class="fas fa-search"></i>
-                            <input type="text" id="series-search" placeholder="Search series...">
+                        <div class="search-controls">
+                            <div class="search-box">
+                                <i class="fas fa-search"></i>
+                                <input type="text" id="series-search" placeholder="Search title/path...">
+                            </div>
+                            <div class="search-box">
+                                <i class="fas fa-hashtag"></i>
+                                <input type="text" id="series-imdb-search" placeholder="Search IMDb ID...">
+                            </div>
+                        </div>
+                        <div class="filter-controls">
+                            <select id="series-filter-date">
+                                <option value="">All Series</option>
+                                <option value="complete">Fully Dated</option>
+                                <option value="incomplete">Missing Dates</option>
+                                <option value="none">No Dates</option>
+                            </select>
+                            <select id="series-filter-source">
+                                <option value="">All Sources</option>
+                            </select>
+                            <button class="btn btn-primary" onclick="refreshSeries()">
+                                <i class="fas fa-sync"></i> Refresh
+                            </button>
                         </div>
-                        <button class="btn btn-primary" onclick="refreshSeries()">
-                            <i class="fas fa-sync"></i> Refresh
-                        </button>
                     </div>
                 </div>
                 

static/js/app.js

@@ -11,6 +11,7 @@ document.addEventListener('DOMContentLoaded', function() {
     initializeTabs();
     initializeEventListeners();
     loadDashboard();
+    loadSeriesSources();
 });
 
 // Tab management
@@ -63,10 +64,13 @@ function initializeEventListeners() {
     document.getElementById('movies-search').addEventListener('input', debounce(loadMovies, 500));
     document.getElementById('movies-imdb-search').addEventListener('input', debounce(loadMovies, 500));
     document.getElementById('series-search').addEventListener('input', debounce(loadSeries, 500));
+    document.getElementById('series-imdb-search').addEventListener('input', debounce(loadSeries, 500));
     
     // Filter dropdowns
     document.getElementById('movies-filter-date').addEventListener('change', loadMovies);
     document.getElementById('movies-filter-source').addEventListener('change', loadMovies);
+    document.getElementById('series-filter-date').addEventListener('change', loadSeries);
+    document.getElementById('series-filter-source').addEventListener('change', loadSeries);
     
     // Forms
     document.getElementById('edit-form').addEventListener('submit', handleEditSubmit);
@@ -296,6 +300,23 @@ function updateMoviesSourceFilter(data) {
     }
 }
 
+async function loadSeriesSources() {
+    try {
+        const data = await apiCall('/api/series/sources');
+        const select = document.getElementById('series-filter-source');
+        const currentValue = select.value;
+        
+        select.innerHTML = '<option value="">All Sources</option>';
+        data.sources.forEach(source => {
+            select.innerHTML += `<option value="${source}">${source}</option>`;
+        });
+        
+        select.value = currentValue;
+    } catch (error) {
+        console.error('Failed to load series sources:', error);
+    }
+}
+
 function refreshMovies() {
     loadMovies(currentMoviesPage);
 }
@@ -303,6 +324,9 @@ function refreshMovies() {
 // TV Series
 async function loadSeries(page = 1) {
     const search = document.getElementById('series-search').value;
+    const imdbSearch = document.getElementById('series-imdb-search').value;
+    const dateFilter = document.getElementById('series-filter-date').value;
+    const sourceFilter = document.getElementById('series-filter-source').value;
     
     const params = new URLSearchParams({
         skip: (page - 1) * 50,
@@ -310,6 +334,9 @@ async function loadSeries(page = 1) {
     });
     
     if (search) params.append('search', search);
+    if (imdbSearch) params.append('imdb_search', imdbSearch);
+    if (dateFilter) params.append('date_filter', dateFilter);
+    if (sourceFilter) params.append('source_filter', sourceFilter);
     
     try {
         const data = await apiCall(`/api/series?${params}`);