improv: phase 1 of faster scan and tracking of scans

VERSION

@@ -1 +1 @@
-2.4.7
+2.5.0

api/routes.py

@@ -530,8 +530,8 @@ async def debug_movie_history(imdb_id: str, dependencies: dict):
         }
 
 
-async def manual_scan(background_tasks: BackgroundTasks, path: Optional[str] = None, scan_type: str = "both", dependencies: dict = None):
-    """Manual scan endpoint"""
+async def manual_scan(background_tasks: BackgroundTasks, path: Optional[str] = None, scan_type: str = "both", scan_mode: str = "smart", dependencies: dict = None):
+    """Manual scan endpoint with smart optimization modes"""
     config = dependencies["config"]
     nfo_manager = dependencies["nfo_manager"]
     tv_processor = dependencies["tv_processor"]
@@ -540,6 +540,9 @@ async def manual_scan(background_tasks: BackgroundTasks, path: Optional[str] = N
     if scan_type not in ["both", "tv", "movies"]:
         raise HTTPException(status_code=400, detail="scan_type must be 'both', 'tv', or 'movies'")
     
+    if scan_mode not in ["smart", "full", "incomplete"]:
+        raise HTTPException(status_code=400, detail="scan_mode must be 'smart', 'full', or 'incomplete'")
+    
     async def run_scan():
         from datetime import datetime, timezone
         import time
@@ -564,7 +567,12 @@ async def manual_scan(background_tasks: BackgroundTasks, path: Optional[str] = N
             local_start = start_time
             tz_display = " (container time)"
             
-        print(f"🚀 MANUAL SCAN STARTED: {scan_type} scan initiated at {local_start.strftime('%Y-%m-%d %H:%M:%S')}{tz_display}")
+        print(f"🚀 MANUAL SCAN STARTED: {scan_type} scan (mode: {scan_mode}) initiated at {local_start.strftime('%Y-%m-%d %H:%M:%S')}{tz_display}")
+        
+        # Initialize counters for scan statistics
+        tv_series_total = 0
+        tv_series_skipped = 0
+        tv_series_processed = 0
         
         paths_to_scan = []
         if path:
@@ -604,9 +612,17 @@ async def manual_scan(background_tasks: BackgroundTasks, path: Optional[str] = N
                     # Check if this path itself is a series (has IMDb ID in the directory name)
                     if nfo_manager.parse_imdb_from_path(scan_path):
                         try:
-                            tv_processor.process_series(scan_path)
+                            # Determine force_scan based on scan mode
+                            force_scan = (scan_mode == "full")
+                            result = tv_processor.process_series(scan_path, force_scan=force_scan)
+                            tv_series_total += 1
+                            if result == "skipped":
+                                tv_series_skipped += 1
+                            elif result == "processed":
+                                tv_series_processed += 1
                         except Exception as e:
                             print(f"ERROR: Failed processing TV series {scan_path}: {e}")
+                            tv_series_total += 1
                     else:
                         # Full series processing - scan subdirectories
                         import re
@@ -618,9 +634,17 @@ async def manual_scan(background_tasks: BackgroundTasks, path: Optional[str] = N
                                 nfo_manager.parse_imdb_from_path(item)):
                                 tv_count += 1
                                 try:
-                                    tv_processor.process_series(item)
+                                    # Determine force_scan based on scan mode
+                                    force_scan = (scan_mode == "full")
+                                    result = tv_processor.process_series(item, force_scan=force_scan)
+                                    tv_series_total += 1
+                                    if result == "skipped":
+                                        tv_series_skipped += 1
+                                    elif result == "processed":
+                                        tv_series_processed += 1
                                 except Exception as e:
                                     print(f"ERROR: Failed processing TV series {item}: {e}")
+                                    tv_series_total += 1
                                 
                                 # Yield control every TV series to allow other requests  
                                 if tv_count % 1 == 0:
@@ -668,11 +692,18 @@ async def manual_scan(background_tasks: BackgroundTasks, path: Optional[str] = N
             local_end = end_time
             tz_display = " (container time)"
             
-        print(f"✅ MANUAL SCAN COMPLETED: {scan_type} scan finished at {local_end.strftime('%Y-%m-%d %H:%M:%S')}{tz_display}")
+        print(f"✅ MANUAL SCAN COMPLETED: {scan_type} scan (mode: {scan_mode}) finished at {local_end.strftime('%Y-%m-%d %H:%M:%S')}{tz_display}")
         print(f"⏱️ MANUAL SCAN DURATION: {duration_str} (total time: {duration.total_seconds():.1f} seconds)")
         
+        # Print optimization statistics for TV scans
+        if scan_type in ["both", "tv"] and tv_series_total > 0:
+            print(f"📊 SCAN OPTIMIZATION STATS: Total: {tv_series_total}, Processed: {tv_series_processed}, Skipped: {tv_series_skipped}")
+            if tv_series_skipped > 0:
+                skip_percentage = (tv_series_skipped / tv_series_total) * 100
+                print(f"⚡ PERFORMANCE BOOST: {tv_series_skipped}/{tv_series_total} series skipped ({skip_percentage:.1f}% time saved!)")
+    
     background_tasks.add_task(run_scan)
-    return {"status": "started", "message": f"Manual {scan_type} scan started"}
+    return {"status": "started", "message": f"Manual {scan_type} scan started (mode: {scan_mode})"}
 
 
 async def scan_tv_season(background_tasks: BackgroundTasks, request: TVSeasonRequest, dependencies: dict):
@@ -1041,8 +1072,8 @@ def register_routes(app, dependencies: dict):
         return await debug_movie_history(imdb_id, dependencies)
 
     @app.post("/manual/scan")
-    async def _manual_scan(background_tasks: BackgroundTasks, path: Optional[str] = None, scan_type: str = "both"):
-        return await manual_scan(background_tasks, path, scan_type, dependencies)
+    async def _manual_scan(background_tasks: BackgroundTasks, path: Optional[str] = None, scan_type: str = "both", scan_mode: str = "smart"):
+        return await manual_scan(background_tasks, path, scan_type, scan_mode, dependencies)
 
     @app.post("/tv/scan-season")
     async def _scan_tv_season(background_tasks: BackgroundTasks, request: TVSeasonRequest):

processors/tv_processor.py

@@ -53,22 +53,96 @@ class TVProcessor:
             path_mapper=self.path_mapper
         )
     
-    def process_series(self, series_path: Path) -> None:
+    def should_skip_series(self, imdb_id: str, episodes_on_disk: int, series_name: str = "") -> Tuple[bool, str]:
+        """
+        Determine if we should skip processing this series based on completion status
+        
+        Args:
+            imdb_id: Series IMDb ID
+            episodes_on_disk: Number of episodes found on disk
+            series_name: Series name for logging
+            
+        Returns:
+            (should_skip: bool, reason: str)
+        """
+        try:
+            with self.db.get_connection() as conn:
+                cursor = conn.cursor()
+                
+                if self.db.db_type == "postgresql":
+                    cursor.execute("""
+                        SELECT 
+                            COUNT(*) as total_in_db,
+                            COUNT(CASE WHEN dateadded IS NOT NULL AND source IS NOT NULL AND source != 'unknown' AND source != 'no_valid_date_source' THEN 1 END) as complete_episodes
+                        FROM episodes 
+                        WHERE imdb_id = %s
+                    """, (imdb_id,))
+                else:
+                    cursor.execute("""
+                        SELECT 
+                            COUNT(*) as total_in_db,
+                            COUNT(CASE WHEN dateadded IS NOT NULL AND dateadded != '' AND source IS NOT NULL AND source != 'unknown' AND source != 'no_valid_date_source' THEN 1 END) as complete_episodes
+                        FROM episodes 
+                        WHERE imdb_id = ?
+                    """, (imdb_id,))
+                
+                result = cursor.fetchone()
+                if not result:
+                    return False, "No database records found"
+                
+                if self.db.db_type == "postgresql":
+                    total_in_db = result['total_in_db']
+                    complete_episodes = result['complete_episodes'] 
+                else:
+                    total_in_db = result[0]
+                    complete_episodes = result[1]
+                
+                # Skip if:
+                # 1. We have episodes in database
+                # 2. Database count matches disk count (no missing episodes)
+                # 3. All episodes have valid dates and sources
+                if total_in_db > 0 and total_in_db == episodes_on_disk and complete_episodes == episodes_on_disk:
+                    return True, f"Complete: {complete_episodes}/{episodes_on_disk} episodes have valid dates"
+                elif total_in_db == 0:
+                    return False, f"New series: No episodes in database"
+                elif total_in_db != episodes_on_disk:
+                    return False, f"Disk mismatch: {total_in_db} in DB vs {episodes_on_disk} on disk"
+                else:
+                    return False, f"Incomplete: {complete_episodes}/{episodes_on_disk} episodes have valid dates"
+                    
+        except Exception as e:
+            _log("ERROR", f"Error checking series completion for {imdb_id}: {e}")
+            return False, f"Error checking completion: {e}"
+    
+    def process_series(self, series_path: Path, force_scan: bool = False) -> str:
         """Process a TV series directory"""
         imdb_id = self.nfo_manager.parse_imdb_from_path(series_path)
         if not imdb_id:
             _log("ERROR", f"No IMDb ID found in series path: {series_path}")
-            return
+            return "error"
         
         _log("INFO", f"Processing TV series: {series_path.name}")
         
-        # Update database
-        self.db.upsert_series(imdb_id, str(series_path))
-        
-        # Find video files
+        # Find video files first to check completion
         disk_episodes = find_episodes_on_disk(series_path)
         _log("INFO", f"Found {len(disk_episodes)} episodes on disk")
         
+        # Check if we should skip this series (unless forced)
+        if not force_scan:
+            should_skip, reason = self.should_skip_series(imdb_id, len(disk_episodes), series_path.name)
+            if should_skip:
+                _log("INFO", f"⏭️ SKIPPING SERIES: {series_path.name} [{imdb_id}] - {reason}")
+                # Still update the series record to track that we've seen it
+                self.db.upsert_series(imdb_id, str(series_path))
+                return "skipped"
+            else:
+                _log("INFO", f"📺 PROCESSING SERIES: {series_path.name} [{imdb_id}] - {reason}")
+        else:
+            _log("INFO", f"🔄 FORCE PROCESSING SERIES: {series_path.name} [{imdb_id}] - Force scan enabled")
+        
+        # Update database
+        self.db.upsert_series(imdb_id, str(series_path))
+        
         # Get episode dates
         episode_dates = self._gather_episode_dates(series_path, imdb_id, disk_episodes)
         
@@ -110,6 +184,7 @@ class TVProcessor:
         pass
         
         _log("INFO", f"Completed processing TV series: {series_path.name}")
+        return "processed"
     
     def _extract_series_title_from_path(self, series_path: Path) -> Optional[str]:
         """Extract series title from directory path using unified file utilities"""