sbcrumb/nfoguard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update to skiped files
Local Docker Build (Dev) / build-dev (push) Successful in 5s
Details

Browse Source
This commit is contained in:
SBCrumb
2025-11-03 16:37:42 -05:00
parent c9b099a218
commit 7932b9cfba
6 changed files with 209 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
VERSION
+1 -1
View File
@@ -1 +1 @@
2.9.10-version-strings-fix
2.10.0-skipped-tracking
api/web_routes.py
+33 -14
View File
@@ -82,13 +82,14 @@ def map_source_to_description(source: str) -> str:
# Database Query Endpoints
# ---------------------------
async def get_movies_list(dependencies: dict,
async def get_movies_list(dependencies: dict,
skip: int = Query(0, ge=0),
limit: int = Query(100, le=1000),
has_date: Optional[bool] = Query(None),
source_filter: Optional[str] = Query(None),
search: Optional[str] = Query(None),
imdb_search: Optional[str] = Query(None)):
imdb_search: Optional[str] = Query(None),
skipped: Optional[bool] = Query(None)):
"""Get paginated list of movies with filtering options"""
db = dependencies["db"]
@@ -98,7 +99,13 @@ async def get_movies_list(dependencies: dict,
# Build dynamic query
where_conditions = []
params = []
if skipped is not None:
if skipped:
where_conditions.append("skipped = TRUE")
else:
where_conditions.append("(skipped = FALSE OR skipped IS NULL)")
if has_date is not None:
if has_date:
# PostgreSQL - NULL handling
@@ -106,15 +113,15 @@ async def get_movies_list(dependencies: dict,
else:
# PostgreSQL - NULL handling
where_conditions.append("dateadded IS NULL")
if source_filter:
where_conditions.append("source = %s")
params.append(source_filter)
if search:
where_conditions.append("(imdb_id ILIKE %s OR path ILIKE %s)")
params.extend([f"%{search}%", f"%{search}%"])
where_conditions.append("(imdb_id ILIKE %s OR path ILIKE %s OR title ILIKE %s)")
params.extend([f"%{search}%", f"%{search}%", f"%{search}%"])
if imdb_search:
where_conditions.append("imdb_id ILIKE %s")
params.append(f"%{imdb_search}%")
@@ -128,7 +135,7 @@ async def get_movies_list(dependencies: dict,
# Get paginated results - PostgreSQL
query = f"""
SELECT imdb_id, title, year, path, released, dateadded, source, has_video_file, last_updated
SELECT imdb_id, title, year, path, released, dateadded, source, has_video_file, last_updated, skipped, skip_reason
FROM movies
WHERE {where_clause}
ORDER BY last_updated DESC
@@ -488,7 +495,14 @@ async def get_dashboard_stats(dependencies: dict):
cursor.execute("SELECT COUNT(*) FROM episodes WHERE source = 'no_valid_date_source'")
episodes_no_valid_source = db._get_first_value(cursor.fetchone())
# Skipped items
cursor.execute("SELECT COUNT(*) FROM movies WHERE skipped = TRUE")
movies_skipped = db._get_first_value(cursor.fetchone())
cursor.execute("SELECT COUNT(*) FROM episodes WHERE skipped = TRUE")
episodes_skipped = db._get_first_value(cursor.fetchone())
# Recent activity (last 7 days)
cursor.execute("""
SELECT COUNT(*) FROM processing_history
@@ -518,7 +532,8 @@ async def get_dashboard_stats(dependencies: dict):
# Calculate total missing dates (movies + episodes)
total_missing_dates = movies_without_dates + episodes_without_dates
total_skipped = movies_skipped + episodes_skipped
# Combine with enhanced stats
stats.update({
"movies_with_dates": movies_with_dates,
@@ -530,6 +545,9 @@ async def get_dashboard_stats(dependencies: dict):
"total_missing_dates": total_missing_dates,
"movies_no_valid_source": movies_no_valid_source,
"episodes_no_valid_source": episodes_no_valid_source,
"movies_skipped": movies_skipped,
"episodes_skipped": episodes_skipped,
"total_skipped": total_skipped,
"recent_activity_count": recent_activity,
"movie_sources": movie_sources,
"episode_sources": episode_sources
@@ -1461,9 +1479,10 @@ def register_web_routes(app, dependencies):
# Movies endpoints
@app.get("/api/movies")
async def api_movies_list(skip: int = 0, limit: int = 100, has_date: bool = None,
source_filter: str = None, search: str = None, imdb_search: str = None):
return await get_movies_list(dependencies, skip, limit, has_date, source_filter, search, imdb_search)
async def api_movies_list(skip: int = 0, limit: int = 100, has_date: bool = None,
source_filter: str = None, search: str = None, imdb_search: str = None,
skipped: bool = None):
return await get_movies_list(dependencies, skip, limit, has_date, source_filter, search, imdb_search, skipped)
@app.post("/api/movies/{imdb_id}/update-date")
async def api_update_movie_date(imdb_id: str, dateadded: str = None, source: str = "manual"):
core/database.py
+92 -1
View File
@@ -133,6 +133,29 @@ class NFOGuardDatabase:
END IF;
END $$;
""")
# Add skipped and skip_reason columns if they don't exist (migration for skip tracking)
cursor.execute("""
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
WHERE table_name='movies' AND column_name='skipped') THEN
ALTER TABLE movies ADD COLUMN skipped BOOLEAN DEFAULT FALSE;
END IF;
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
WHERE table_name='movies' AND column_name='skip_reason') THEN
ALTER TABLE movies ADD COLUMN skip_reason TEXT;
END IF;
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
WHERE table_name='episodes' AND column_name='skipped') THEN
ALTER TABLE episodes ADD COLUMN skipped BOOLEAN DEFAULT FALSE;
END IF;
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
WHERE table_name='episodes' AND column_name='skip_reason') THEN
ALTER TABLE episodes ADD COLUMN skip_reason TEXT;
END IF;
END $$;
""")
# Processing history table
cursor.execute("""
@@ -301,7 +324,75 @@ class NFOGuardDatabase:
import os
if os.environ.get("DEBUG", "false").lower() == "true":
print(f"🔍 DATABASE VERIFY: After upsert, found dateadded={result['dateadded'] if result else 'NOT_FOUND'}, source={result['source'] if result else 'NOT_FOUND'}")
def mark_movie_skipped(self, imdb_id: str, title: str, year: int, path: str, reason: str):
"""Mark a movie as skipped with reason"""
with self.get_connection() as conn:
cursor = conn.cursor()
timestamp = datetime.utcnow()
cursor.execute("""
INSERT INTO movies (imdb_id, title, year, path, skipped, skip_reason, has_video_file, last_updated)
VALUES (%s, %s, %s, %s, TRUE, %s, FALSE, %s)
ON CONFLICT (imdb_id) DO UPDATE SET
title = EXCLUDED.title,
year = EXCLUDED.year,
path = EXCLUDED.path,
skipped = TRUE,
skip_reason = EXCLUDED.skip_reason,
last_updated = EXCLUDED.last_updated
""", (imdb_id, title, year, path, reason, timestamp))
def mark_episode_skipped(self, imdb_id: str, season: int, episode: int, reason: str):
"""Mark an episode as skipped with reason"""
with self.get_connection() as conn:
cursor = conn.cursor()
timestamp = datetime.utcnow()
cursor.execute("""
INSERT INTO episodes (imdb_id, season, episode, skipped, skip_reason, has_video_file, last_updated)
VALUES (%s, %s, %s, TRUE, %s, FALSE, %s)
ON CONFLICT (imdb_id, season, episode) DO UPDATE SET
skipped = TRUE,
skip_reason = EXCLUDED.skip_reason,
last_updated = EXCLUDED.last_updated
""", (imdb_id, season, episode, reason, timestamp))
def clear_movie_skipped(self, imdb_id: str):
"""Clear skipped flag for a movie"""
with self.get_connection() as conn:
cursor = conn.cursor()
cursor.execute("""
UPDATE movies SET skipped = FALSE, skip_reason = NULL
WHERE imdb_id = %s
""", (imdb_id,))
def clear_episode_skipped(self, imdb_id: str, season: int, episode: int):
"""Clear skipped flag for an episode"""
with self.get_connection() as conn:
cursor = conn.cursor()
cursor.execute("""
UPDATE episodes SET skipped = FALSE, skip_reason = NULL
WHERE imdb_id = %s AND season = %s AND episode = %s
""", (imdb_id, season, episode))
def get_skipped_counts(self) -> Dict:
"""Get counts of skipped movies and episodes"""
with self.get_connection() as conn:
cursor = conn.cursor()
# Count skipped movies
cursor.execute("SELECT COUNT(*) as count FROM movies WHERE skipped = TRUE")
skipped_movies = cursor.fetchone()['count']
# Count skipped episodes
cursor.execute("SELECT COUNT(*) as count FROM episodes WHERE skipped = TRUE")
skipped_episodes = cursor.fetchone()['count']
return {
'movies': skipped_movies,
'episodes': skipped_episodes,
'total': skipped_movies + skipped_episodes
}
def get_series_episodes(self, imdb_id: str, has_video_file_only: bool = False) -> List[Dict]:
"""Get all episodes for a series"""
with self.get_connection() as conn:
core/database_populator.py
+38 -30
View File
@@ -82,29 +82,24 @@ class DatabasePopulator:
_log("DEBUG", f"Extracted IMDb ID {imdb_id} from path for: {movie.get('title')}")
if not imdb_id:
skip_reason = 'No IMDb ID found'
skip_info = {
'title': movie.get('title', 'Unknown'),
'year': movie.get('year'),
'path': path,
'reason': 'No IMDb ID found'
'reason': skip_reason
}
stats['skipped_items'].append(skip_info)
_log("DEBUG", f"Skipping movie without IMDb ID: {movie.get('title')} (path: {path})")
# Note: Cannot mark in DB because no IMDb ID (primary key)
stats['skipped'] += 1
continue
# Check if movie already exists in database
existing = self.db.get_movie_dates(imdb_id)
if existing and existing.get('dateadded'):
skip_info = {
'title': movie.get('title', 'Unknown'),
'year': movie.get('year'),
'imdb_id': imdb_id,
'reason': 'Already in database'
}
stats['skipped_items'].append(skip_info)
# Already in database - skip silently (not an error)
_log("DEBUG", f"Movie {imdb_id} already in database, skipping")
stats['skipped'] += 1
continue
# Get release date
@@ -134,14 +129,24 @@ class DatabasePopulator:
dateadded = released
source = f'{source_type}_fallback'
else:
skip_reason = 'No import date in Radarr history and no release dates available'
skip_info = {
'title': movie.get('title', 'Unknown'),
'year': movie.get('year'),
'imdb_id': imdb_id,
'reason': 'No import date in Radarr history and no release dates available'
'reason': skip_reason
}
stats['skipped_items'].append(skip_info)
_log("DEBUG", f"No date available for movie {imdb_id}, skipping")
# Mark as skipped in database for troubleshooting
self.db.mark_movie_skipped(
imdb_id=imdb_id,
title=movie.get('title', 'Unknown'),
year=movie.get('year', 0),
path=path or 'unknown',
reason=skip_reason
)
stats['skipped'] += 1
continue
elif released:
@@ -149,14 +154,24 @@ class DatabasePopulator:
dateadded = released
source = f'{source_type}_fallback'
else:
skip_reason = 'No Radarr movie ID and no release dates available'
skip_info = {
'title': movie.get('title', 'Unknown'),
'year': movie.get('year'),
'imdb_id': imdb_id,
'reason': 'No Radarr movie ID and no release dates available'
'reason': skip_reason
}
stats['skipped_items'].append(skip_info)
_log("DEBUG", f"No date available for movie {imdb_id}, skipping")
# Mark as skipped in database for troubleshooting
self.db.mark_movie_skipped(
imdb_id=imdb_id,
title=movie.get('title', 'Unknown'),
year=movie.get('year', 0),
path=path or 'unknown',
reason=skip_reason
)
stats['skipped'] += 1
continue
@@ -278,29 +293,13 @@ class DatabasePopulator:
# Check if episode already exists
existing = self.db.get_episode_date(imdb_id, season_num, episode_num)
if existing and existing.get('dateadded'):
skip_info = {
'title': series_title,
'episode_title': episode_title,
'season': season_num,
'episode': episode_num,
'reason': 'Already in database'
}
stats['skipped_items'].append(skip_info)
stats['skipped'] += 1
# Already in database - skip silently (not an error)
continue
# Only process episodes that have video files
has_file = episode.get('hasFile', False)
if not has_file:
skip_info = {
'title': series_title,
'episode_title': episode_title,
'season': season_num,
'episode': episode_num,
'reason': 'No video file'
}
stats['skipped_items'].append(skip_info)
stats['skipped'] += 1
# No video file - skip silently (intentionally filtered)
continue
# Get air date
@@ -328,14 +327,23 @@ class DatabasePopulator:
source = 'sonarr:aired_fallback'
elif not dateadded:
# No date available
skip_reason = 'No import date from Sonarr history and no air date available'
skip_info = {
'title': series_title,
'episode_title': episode_title,
'season': season_num,
'episode': episode_num,
'reason': 'No import date from Sonarr history and no air date available'
'reason': skip_reason
}
stats['skipped_items'].append(skip_info)
# Mark as skipped in database for troubleshooting
self.db.mark_episode_skipped(
imdb_id=imdb_id,
season=season_num,
episode=episode_num,
reason=skip_reason
)
stats['skipped'] += 1
continue
nfoguard-web/static/index.html
+15 -3
View File
@@ -4,7 +4,7 @@
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>NFOGuard - Database Management</title>
<link rel="stylesheet" href="/static/css/styles.css?v=2.9.10-version-strings-fix">
<link rel="stylesheet" href="/static/css/styles.css?v=2.10.0-skipped-tracking">
<link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0/css/all.min.css" rel="stylesheet">
</head>
<body>
@@ -12,7 +12,7 @@
<!-- Header -->
<header class="app-header">
<div class="header-content">
<h1><i class="fas fa-shield-alt"></i> NFOGuard <span style="font-size: 0.5em; color: #888;">v2.9.10-version-strings-fix</span></h1>
<h1><i class="fas fa-shield-alt"></i> NFOGuard <span style="font-size: 0.5em; color: #888;">v2.10.0-skipped-tracking</span></h1>
<p>Database Management & Reporting</p>
</div>
<div class="auth-status" id="auth-status" style="display: none;">
@@ -93,6 +93,17 @@
<small>Last 7 days</small>
</div>
</div>
<div class="stat-card" style="cursor: pointer;" onclick="showSkippedItems()">
<div class="stat-icon" style="background: linear-gradient(135deg, #f39c12 0%, #e67e22 100%);">
<i class="fas fa-ban"></i>
</div>
<div class="stat-info">
<h3 id="skipped-total">-</h3>
<p>Skipped Items</p>
<small id="skipped-breakdown">- movies, - episodes</small>
</div>
</div>
</div>
<div class="dashboard-charts">
@@ -127,6 +138,7 @@
<option value="">All Movies</option>
<option value="true">With Dates</option>
<option value="false">Missing Dates</option>
<option value="skipped">Skipped</option>
</select>
<select id="movies-filter-source">
<option value="">All Sources</option>
@@ -711,6 +723,6 @@
<!-- Toast Notifications -->
<div class="toast-container" id="toast-container"></div>
<script src="/static/js/app.js?v=2.9.10-version-strings-fix"></script>
<script src="/static/js/app.js?v=2.10.0-skipped-tracking">
</body>
</html>
nfoguard-web/static/js/app.js
+30 -5
View File
@@ -141,6 +141,24 @@ function updateDashboardStats() {
document.getElementById('no-valid-source-total').textContent = `${moviesWithoutDates} movies, ${episodesWithoutDates} episodes without dates`;
document.getElementById('recent-activity').textContent = dashboardData.recent_activity_count || 0;
// Skipped items
const skippedMovies = dashboardData.movies_skipped || 0;
const skippedEpisodes = dashboardData.episodes_skipped || 0;
const skippedTotal = dashboardData.total_skipped || (skippedMovies + skippedEpisodes);
document.getElementById('skipped-total').textContent = skippedTotal;
document.getElementById('skipped-breakdown').textContent = `${skippedMovies} movies, ${skippedEpisodes} episodes`;
}
function showSkippedItems() {
// Switch to Movies tab and filter to show only skipped items
switchTab('movies');
// Set the filter dropdown to "skipped"
const moviesFilter = document.getElementById('movies-filter-date');
if (moviesFilter) {
moviesFilter.value = 'skipped';
refreshMovies();
}
}
function updateDashboardCharts() {
@@ -196,20 +214,27 @@ async function loadMovies(page = 1) {
const search = document.getElementById('movies-search').value;
const imdbSearch = document.getElementById('movies-imdb-search').value;
const hasDate = document.getElementById('movies-filter-date').value;
const dateFilter = document.getElementById('movies-filter-date').value;
const sourceFilter = document.getElementById('movies-filter-source').value;
const skip = (page - 1) * 100;
console.log(`DEBUG: loadMovies called with page=${page}, calculated skip=${skip}`);
const params = new URLSearchParams({
skip: skip,
limit: 100
});
if (search) params.append('search', search);
if (imdbSearch) params.append('imdb_search', imdbSearch);
if (hasDate) params.append('has_date', hasDate);
// Handle different filter values
if (dateFilter === 'skipped') {
params.append('skipped', 'true');
} else if (dateFilter) {
params.append('has_date', dateFilter);
}
if (sourceFilter) params.append('source_filter', sourceFilter);
try {