From 55d5f891b514fa46c69634c9acac3bb0d967ed80 Mon Sep 17 00:00:00 2001 From: sbcrumb Date: Sat, 18 Oct 2025 15:16:02 -0400 Subject: [PATCH] improv: phase 1 of faster scan and tracking of scans --- VERSION | 2 +- api/routes.py | 49 +++++++++++++++++---- processors/tv_processor.py | 87 +++++++++++++++++++++++++++++++++++--- 3 files changed, 122 insertions(+), 16 deletions(-) diff --git a/VERSION b/VERSION index e30309f..437459c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.7 +2.5.0 diff --git a/api/routes.py b/api/routes.py index e63eadf..777f59a 100644 --- a/api/routes.py +++ b/api/routes.py @@ -530,8 +530,8 @@ async def debug_movie_history(imdb_id: str, dependencies: dict): } -async def manual_scan(background_tasks: BackgroundTasks, path: Optional[str] = None, scan_type: str = "both", dependencies: dict = None): - """Manual scan endpoint""" +async def manual_scan(background_tasks: BackgroundTasks, path: Optional[str] = None, scan_type: str = "both", scan_mode: str = "smart", dependencies: dict = None): + """Manual scan endpoint with smart optimization modes""" config = dependencies["config"] nfo_manager = dependencies["nfo_manager"] tv_processor = dependencies["tv_processor"] @@ -540,6 +540,9 @@ async def manual_scan(background_tasks: BackgroundTasks, path: Optional[str] = N if scan_type not in ["both", "tv", "movies"]: raise HTTPException(status_code=400, detail="scan_type must be 'both', 'tv', or 'movies'") + if scan_mode not in ["smart", "full", "incomplete"]: + raise HTTPException(status_code=400, detail="scan_mode must be 'smart', 'full', or 'incomplete'") + async def run_scan(): from datetime import datetime, timezone import time @@ -564,7 +567,12 @@ async def manual_scan(background_tasks: BackgroundTasks, path: Optional[str] = N local_start = start_time tz_display = " (container time)" - print(f"🚀 MANUAL SCAN STARTED: {scan_type} scan initiated at {local_start.strftime('%Y-%m-%d %H:%M:%S')}{tz_display}") + print(f"🚀 MANUAL SCAN STARTED: {scan_type} scan (mode: {scan_mode}) initiated at {local_start.strftime('%Y-%m-%d %H:%M:%S')}{tz_display}") + + # Initialize counters for scan statistics + tv_series_total = 0 + tv_series_skipped = 0 + tv_series_processed = 0 paths_to_scan = [] if path: @@ -604,9 +612,17 @@ async def manual_scan(background_tasks: BackgroundTasks, path: Optional[str] = N # Check if this path itself is a series (has IMDb ID in the directory name) if nfo_manager.parse_imdb_from_path(scan_path): try: - tv_processor.process_series(scan_path) + # Determine force_scan based on scan mode + force_scan = (scan_mode == "full") + result = tv_processor.process_series(scan_path, force_scan=force_scan) + tv_series_total += 1 + if result == "skipped": + tv_series_skipped += 1 + elif result == "processed": + tv_series_processed += 1 except Exception as e: print(f"ERROR: Failed processing TV series {scan_path}: {e}") + tv_series_total += 1 else: # Full series processing - scan subdirectories import re @@ -618,9 +634,17 @@ async def manual_scan(background_tasks: BackgroundTasks, path: Optional[str] = N nfo_manager.parse_imdb_from_path(item)): tv_count += 1 try: - tv_processor.process_series(item) + # Determine force_scan based on scan mode + force_scan = (scan_mode == "full") + result = tv_processor.process_series(item, force_scan=force_scan) + tv_series_total += 1 + if result == "skipped": + tv_series_skipped += 1 + elif result == "processed": + tv_series_processed += 1 except Exception as e: print(f"ERROR: Failed processing TV series {item}: {e}") + tv_series_total += 1 # Yield control every TV series to allow other requests if tv_count % 1 == 0: @@ -668,11 +692,18 @@ async def manual_scan(background_tasks: BackgroundTasks, path: Optional[str] = N local_end = end_time tz_display = " (container time)" - print(f"✅ MANUAL SCAN COMPLETED: {scan_type} scan finished at {local_end.strftime('%Y-%m-%d %H:%M:%S')}{tz_display}") + print(f"✅ MANUAL SCAN COMPLETED: {scan_type} scan (mode: {scan_mode}) finished at {local_end.strftime('%Y-%m-%d %H:%M:%S')}{tz_display}") print(f"⏱️ MANUAL SCAN DURATION: {duration_str} (total time: {duration.total_seconds():.1f} seconds)") + + # Print optimization statistics for TV scans + if scan_type in ["both", "tv"] and tv_series_total > 0: + print(f"📊 SCAN OPTIMIZATION STATS: Total: {tv_series_total}, Processed: {tv_series_processed}, Skipped: {tv_series_skipped}") + if tv_series_skipped > 0: + skip_percentage = (tv_series_skipped / tv_series_total) * 100 + print(f"⚡ PERFORMANCE BOOST: {tv_series_skipped}/{tv_series_total} series skipped ({skip_percentage:.1f}% time saved!)") background_tasks.add_task(run_scan) - return {"status": "started", "message": f"Manual {scan_type} scan started"} + return {"status": "started", "message": f"Manual {scan_type} scan started (mode: {scan_mode})"} async def scan_tv_season(background_tasks: BackgroundTasks, request: TVSeasonRequest, dependencies: dict): @@ -1041,8 +1072,8 @@ def register_routes(app, dependencies: dict): return await debug_movie_history(imdb_id, dependencies) @app.post("/manual/scan") - async def _manual_scan(background_tasks: BackgroundTasks, path: Optional[str] = None, scan_type: str = "both"): - return await manual_scan(background_tasks, path, scan_type, dependencies) + async def _manual_scan(background_tasks: BackgroundTasks, path: Optional[str] = None, scan_type: str = "both", scan_mode: str = "smart"): + return await manual_scan(background_tasks, path, scan_type, scan_mode, dependencies) @app.post("/tv/scan-season") async def _scan_tv_season(background_tasks: BackgroundTasks, request: TVSeasonRequest): diff --git a/processors/tv_processor.py b/processors/tv_processor.py index 48bb37d..edc837f 100644 --- a/processors/tv_processor.py +++ b/processors/tv_processor.py @@ -53,22 +53,96 @@ class TVProcessor: path_mapper=self.path_mapper ) - def process_series(self, series_path: Path) -> None: + def should_skip_series(self, imdb_id: str, episodes_on_disk: int, series_name: str = "") -> Tuple[bool, str]: + """ + Determine if we should skip processing this series based on completion status + + Args: + imdb_id: Series IMDb ID + episodes_on_disk: Number of episodes found on disk + series_name: Series name for logging + + Returns: + (should_skip: bool, reason: str) + """ + try: + with self.db.get_connection() as conn: + cursor = conn.cursor() + + if self.db.db_type == "postgresql": + cursor.execute(""" + SELECT + COUNT(*) as total_in_db, + COUNT(CASE WHEN dateadded IS NOT NULL AND source IS NOT NULL AND source != 'unknown' AND source != 'no_valid_date_source' THEN 1 END) as complete_episodes + FROM episodes + WHERE imdb_id = %s + """, (imdb_id,)) + else: + cursor.execute(""" + SELECT + COUNT(*) as total_in_db, + COUNT(CASE WHEN dateadded IS NOT NULL AND dateadded != '' AND source IS NOT NULL AND source != 'unknown' AND source != 'no_valid_date_source' THEN 1 END) as complete_episodes + FROM episodes + WHERE imdb_id = ? + """, (imdb_id,)) + + result = cursor.fetchone() + if not result: + return False, "No database records found" + + if self.db.db_type == "postgresql": + total_in_db = result['total_in_db'] + complete_episodes = result['complete_episodes'] + else: + total_in_db = result[0] + complete_episodes = result[1] + + # Skip if: + # 1. We have episodes in database + # 2. Database count matches disk count (no missing episodes) + # 3. All episodes have valid dates and sources + if total_in_db > 0 and total_in_db == episodes_on_disk and complete_episodes == episodes_on_disk: + return True, f"Complete: {complete_episodes}/{episodes_on_disk} episodes have valid dates" + elif total_in_db == 0: + return False, f"New series: No episodes in database" + elif total_in_db != episodes_on_disk: + return False, f"Disk mismatch: {total_in_db} in DB vs {episodes_on_disk} on disk" + else: + return False, f"Incomplete: {complete_episodes}/{episodes_on_disk} episodes have valid dates" + + except Exception as e: + _log("ERROR", f"Error checking series completion for {imdb_id}: {e}") + return False, f"Error checking completion: {e}" + + def process_series(self, series_path: Path, force_scan: bool = False) -> str: """Process a TV series directory""" imdb_id = self.nfo_manager.parse_imdb_from_path(series_path) if not imdb_id: _log("ERROR", f"No IMDb ID found in series path: {series_path}") - return + return "error" _log("INFO", f"Processing TV series: {series_path.name}") - # Update database - self.db.upsert_series(imdb_id, str(series_path)) - - # Find video files + # Find video files first to check completion disk_episodes = find_episodes_on_disk(series_path) _log("INFO", f"Found {len(disk_episodes)} episodes on disk") + # Check if we should skip this series (unless forced) + if not force_scan: + should_skip, reason = self.should_skip_series(imdb_id, len(disk_episodes), series_path.name) + if should_skip: + _log("INFO", f"⏭️ SKIPPING SERIES: {series_path.name} [{imdb_id}] - {reason}") + # Still update the series record to track that we've seen it + self.db.upsert_series(imdb_id, str(series_path)) + return "skipped" + else: + _log("INFO", f"📺 PROCESSING SERIES: {series_path.name} [{imdb_id}] - {reason}") + else: + _log("INFO", f"🔄 FORCE PROCESSING SERIES: {series_path.name} [{imdb_id}] - Force scan enabled") + + # Update database + self.db.upsert_series(imdb_id, str(series_path)) + # Get episode dates episode_dates = self._gather_episode_dates(series_path, imdb_id, disk_episodes) @@ -110,6 +184,7 @@ class TVProcessor: pass _log("INFO", f"Completed processing TV series: {series_path.name}") + return "processed" def _extract_series_title_from_path(self, series_path: Path) -> Optional[str]: """Extract series title from directory path using unified file utilities"""