sbcrumb/nfoguard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: sbcrumb/nfoguard:dev
Branches Tags
sbcrumb/nfoguard:main sbcrumb/nfoguard:dev
..
compare: sbcrumb/nfoguard:5a1633490ef19637a7013f6352e545d98ebdbce9
Branches Tags
sbcrumb/nfoguard:dev sbcrumb/nfoguard:main

1 Commits
dev .. 5a1633490e

Author SHA1 Message Date
sbcrumb 5a1633490e Merge branch 'dev' into improvements
Build & Push DEV to DockerHub / docker (pull_request) Failing after 20s
Details
2025-09-27 13:02:11 -04:00
68 changed files with 4157 additions and 25272 deletions
Expand all files Collapse all files
.env.example
+20 -179
View File
@@ -1,187 +1,28 @@
# ===========================================
# NFOGuard Configuration - CORRECTED VERSION
# ===========================================
# Main configuration file - safe to share for debugging
# Sensitive data (API keys, passwords) are in .env.secrets
# NFOGuard Environment Variables
# ===========================================
# MEDIA PATHS (REQUIRED) - FIXED
# ===========================================
# Container paths (what NFOGuard sees inside container)
MOVIE_PATHS=/media/Movies/movies,/media/Movies/movies6
TV_PATHS=/media/TV/tv,/media/TV/tv6
# User and Group IDs
PUID=1000
PGID=1000
# Radarr paths (what Radarr sees on the host system)
RADARR_ROOT_FOLDERS=/mnt/unionfs/Media/Movies/movies,/mnt/unionfs/Media/Movies/movies6
# Timezone
TZ=UTC
# Sonarr paths (what Sonarr sees on the host system) - FIXED VARIABLE NAME AND PATHS
SONARR_ROOT_FOLDERS=/mnt/unionfs/Media/TV/tv,/mnt/unionfs/Media/TV/tv6
# Media path (for read-only access to scan NFO files)
MEDIA_PATH=/path/to/your/media
# Download detection paths (for identifying downloads vs existing files)
DOWNLOAD_PATH_INDICATORS=/downloads/,/nzbget/,/completed/,/sabnzbd/,/torrents/
# Database settings (if using PostgreSQL)
# DB_USER=nfoguard
# DB_PASSWORD=your_secure_password
# ===========================================
# SERVER CONFIGURATION
# ===========================================
# Timezone for container logging and timestamps
TZ=America/New_York
# Emby Plugin Deployment - Bind Mount Method (Recommended)
# Set this to the path where Emby plugins are installed on your host
# Common paths:
# - /var/lib/emby/plugins (native Emby install)
# - /path/to/emby/config/plugins (Docker Emby)
# - /config/plugins (linuxserver.io Emby)
EMBY_PLUGINS_PATH=/path/to/emby/plugins
# NFOGuard Core API (webhooks, processing, database management)
CORE_API_HOST=0.0.0.0
CORE_API_PORT=8080
# NFOGuard Web Interface (dashboard, series/movie management)
WEB_API_HOST=0.0.0.0
WEB_API_PORT=8081
# External port where web interface is accessible (for dynamic port reference)
WEB_EXTERNAL_PORT=8081
# ===========================================
# DATABASE CONFIGURATION
# ===========================================
# NFOGuard Database Configuration (PostgreSQL - v2.6+)
# NOTE: For new installations, PostgreSQL is strongly recommended
# Set these values in .env.secrets for security:
# - DB_PASSWORD=your_secure_password
# - DB_USER=nfoguard (default)
# PostgreSQL Database Settings
DB_TYPE=postgresql
DB_HOST=nfoguard-db # Container name from docker-compose
DB_PORT=5432
DB_NAME=nfoguard
DB_USER=nfoguard
# Legacy SQLite Configuration (Deprecated in v2.6+)
# Only use for existing SQLite installations
# DB_PATH=/app/data/media_dates.db
# Log file directory
LOG_DIR=/app/data/logs
# ===========================================
# RADARR DATABASE CONNECTION (RECOMMENDED)
# ===========================================
# Direct database access for better performance
RADARR_DB_TYPE=postgresql
RADARR_DB_HOST=192.168.255.50
RADARR_DB_PORT=5432
RADARR_DB_NAME=radarr-main
RADARR_DB_USER=postgres
# ===========================================
# SONARR DATABASE CONNECTION (RECOMMENDED)
# ===========================================
# Direct database access for better performance
# Phase 7: High-performance TV episode import date detection
SONARR_DB_TYPE=postgresql
SONARR_DB_HOST=192.168.255.50
SONARR_DB_PORT=5432
SONARR_DB_NAME=sonarr-main
SONARR_DB_USER=postgres
# Alternative: SQLite (if Sonarr uses SQLite)
# SONARR_DB_TYPE=sqlite
# SONARR_DB_PATH=/path/to/sonarr.db
# ===========================================
# API CONNECTIONS (OPTIONAL)
# ===========================================
# API keys are stored in .env.secrets for security
RADARR_URL=http://radarr:7878
SONARR_URL=http://sonarr:8989
JELLYSEERR_URL=http://jellyseerr:5055
# ===========================================
# RELEASE DATE PROCESSING
# ===========================================
# Priority order for release date fallbacks (digital, physical, theatrical)
RELEASE_DATE_PRIORITY=digital,physical,theatrical
#RELEASE_DATE_PRIORITY=digital,theatrical,physical
ENABLE_SMART_DATE_VALIDATION=true
MAX_RELEASE_DATE_GAP_YEARS=10
# Prefer API release dates over file modification dates for manual imports
PREFER_RELEASE_DATES_OVER_FILE_DATES=true
# Disable file date fallback completely (recommended for clean imports)
ALLOW_FILE_DATE_FALLBACK=false
# TMDB country for regional release date preferences
TMDB_COUNTRY=US
# ===========================================
# NFO FILE MANAGEMENT (DEPRECATED - Phase 1 Migration)
# ===========================================
# NFO file operations have been removed in favor of database-only architecture
# These settings are preserved for backward compatibility but no longer have effect
# The PostgreSQL database is now the single source of truth for all metadata
# Create/update .nfo files (DEPRECATED - no longer used)
MANAGE_NFO=false
# Update file modification times to match import dates (DEPRECATED - no longer used)
FIX_DIR_MTIMES=false
# Add lockdata tags to prevent metadata overwrites (DEPRECATED - no longer used)
LOCK_METADATA=false
# Brand name in NFO comments (DEPRECATED - no longer used)
MANAGER_BRAND=NFOGuard
# ===========================================
# PROCESSING BEHAVIOR
# ===========================================
# Movie date update strategy
MOVIE_DATE_UPDATE_MODE=overwrite
MOVIE_PRESERVE_EXISTING=false
# When to update existing dates (always, missing_only, never)
UPDATE_MODE=always
# File modification time behavior (update, leave_alone)
MTIME_BEHAVIOR=update
# Manual scan priority: use existing NFO dates for speed vs check APIs for accuracy
# false (default) = Check external APIs first, use NFO as fallback (slower but accurate)
# true = Use NFO dates immediately without API checks (faster but may use wrong dates)
MANUAL_SCAN_PRIORITIZE_NFO=false
# ===========================================
# PERFORMANCE & BATCHING
# ===========================================
# Delay before processing batched events (seconds)
BATCH_DELAY=5.0
# Maximum concurrent series processing
MAX_CONCURRENT_SERIES=3
# Sequential processing delay for bulk downloads (seconds)
# When multiple episodes are downloaded at once, process them one by one with this delay
# Set to 0 to disable sequential processing (parallel mode)
SEQUENTIAL_DELAY=20.0
# API timeout in seconds
TIMEOUT_SECONDS=45
# ===========================================
# DEBUGGING
# ===========================================
# Enable verbose logging (true/false)
# NFOGuard specific settings
DEBUG=false
# Enable path mapping debug output (true/false)
PATH_DEBUG=false
# Suppress TVDB API warnings (true/false) - TVDB failures are common and non-critical
SUPPRESS_TVDB_WARNINGS=true
# ===========================================
# WEB INTERFACE AUTHENTICATION
# ===========================================
# Enable web interface authentication (default: false)
WEB_AUTH_ENABLED=false
# Session timeout in seconds (default: 3600 = 1 hour)
WEB_AUTH_SESSION_TIMEOUT=3600
SUPPRESS_TVDB_WARNINGS=true
.env.secrets.example → .env.secrets.template
+3 -19
View File
@@ -4,17 +4,9 @@
# Add .env.secrets to your .gitignore
# ===========================================
# NFOGUARD DATABASE CREDENTIALS (REQUIRED)
# RADARR DATABASE CREDENTIALS
# ===========================================
# NFOGuard PostgreSQL database password (REQUIRED for v2.6+)
DB_PASSWORD=your_secure_nfoguard_password
# Optional: Override default database user (defaults to 'nfoguard')
# DB_USER=nfoguard
# ===========================================
# RADARR DATABASE CREDENTIALS (OPTIONAL)
# ===========================================
# Database password for external Radarr PostgreSQL connection (optional optimization)
# Database password for PostgreSQL connection
RADARR_DB_PASSWORD=your_radarr_db_password
# ===========================================
@@ -38,12 +30,4 @@ JELLYSEERR_API_KEY=your_jellyseerr_api_key
# TVDB API key (RECOMMENDED for v0.6.1+ Emby TV NFO Compatibility)
# Required to convert IMDB IDs to TVDB IDs for proper Emby metadata loading
# Get free API key at: https://thetvdb.com/api-information
TVDB_API_KEY=your_tvdb_api_key
# ===========================================
# WEB INTERFACE AUTHENTICATION (OPTIONAL)
# ===========================================
# Web interface login credentials (only used if WEB_AUTH_ENABLED=true in .env)
# Set WEB_AUTH_ENABLED=true in .env file to enable authentication
WEB_AUTH_USERNAME=admin
WEB_AUTH_PASSWORD=your_secure_web_password
TVDB_API_KEY=your_tvdb_api_key
.env.template
+124
View File
@@ -0,0 +1,124 @@
# ===========================================
# NFOGuard Configuration - CORRECTED VERSION
# ===========================================
# Main configuration file - safe to share for debugging
# Sensitive data (API keys, passwords) are in .env.secrets
# ===========================================
# MEDIA PATHS (REQUIRED) - FIXED
# ===========================================
# Container paths (what NFOGuard sees inside container)
MOVIE_PATHS=/media/Movies/movies,/media/Movies/movies6
TV_PATHS=/media/TV/tv,/media/TV/tv6
# Radarr paths (what Radarr sees on the host system)
RADARR_ROOT_FOLDERS=/mnt/unionfs/Media/Movies/movies,/mnt/unionfs/Media/Movies/movies6
# Sonarr paths (what Sonarr sees on the host system) - FIXED VARIABLE NAME AND PATHS
SONARR_ROOT_FOLDERS=/mnt/unionfs/Media/TV/tv,/mnt/unionfs/Media/TV/tv6
# Download detection paths (for identifying downloads vs existing files)
DOWNLOAD_PATH_INDICATORS=/downloads/,/nzbget/,/completed/,/sabnzbd/,/torrents/
# ===========================================
# DATABASE CONFIGURATION
# ===========================================
# NFOGuard SQLite database location
DB_PATH=/app/data/media_dates.db
# Log file directory
LOG_DIR=/app/data/logs
# ===========================================
# RADARR DATABASE CONNECTION (RECOMMENDED)
# ===========================================
# Direct database access for better performance
RADARR_DB_TYPE=postgresql
RADARR_DB_HOST=192.168.255.50
RADARR_DB_PORT=5432
RADARR_DB_NAME=radarr-main
RADARR_DB_USER=postgres
# ===========================================
# API CONNECTIONS (OPTIONAL)
# ===========================================
# API keys are stored in .env.secrets for security
RADARR_URL=http://radarr:7878
SONARR_URL=http://sonarr:8989
JELLYSEERR_URL=http://jellyseerr:5055
# ===========================================
# RELEASE DATE PROCESSING
# ===========================================
# Priority order for release date fallbacks (digital, physical, theatrical)
RELEASE_DATE_PRIORITY=digital,physical,theatrical
#RELEASE_DATE_PRIORITY=digital,theatrical,physical
ENABLE_SMART_DATE_VALIDATION=true
MAX_RELEASE_DATE_GAP_YEARS=10
# Prefer API release dates over file modification dates for manual imports
PREFER_RELEASE_DATES_OVER_FILE_DATES=true
# Disable file date fallback completely (recommended for clean imports)
ALLOW_FILE_DATE_FALLBACK=false
# TMDB country for regional release date preferences
TMDB_COUNTRY=US
# ===========================================
# NFO FILE MANAGEMENT
# ===========================================
# Create/update .nfo files
MANAGE_NFO=true
# Update file modification times to match import dates
FIX_DIR_MTIMES=true
# Add lockdata tags to prevent metadata overwrites
LOCK_METADATA=true
# Brand name in NFO comments
MANAGER_BRAND=NFOGuard
# ===========================================
# PROCESSING BEHAVIOR
# ===========================================
# Movie date update strategy
MOVIE_DATE_UPDATE_MODE=overwrite
MOVIE_PRESERVE_EXISTING=false
# When to update existing dates (always, missing_only, never)
UPDATE_MODE=always
# File modification time behavior (update, leave_alone)
MTIME_BEHAVIOR=update
# ===========================================
# PERFORMANCE & BATCHING
# ===========================================
# Delay before processing batched events (seconds)
BATCH_DELAY=5.0
# Maximum concurrent series processing
MAX_CONCURRENT_SERIES=3
# API timeout in seconds
TIMEOUT_SECONDS=45
# ===========================================
# DEBUGGING
# ===========================================
# Enable verbose logging (true/false)
DEBUG=false
# Enable path mapping debug output (true/false)
PATH_DEBUG=false
# Suppress TVDB API warnings (true/false) - TVDB failures are common and non-critical
SUPPRESS_TVDB_WARNINGS=true
# ===========================================
# SERVER CONFIGURATION
# ===========================================
# Port for webhook server
PORT=8080
.gitea/workflows/ci-dev.yml
-165
View File
@@ -1,165 +0,0 @@
name: Local Docker Build (Dev)
on:
push:
branches: [ dev ]
pull_request:
branches: [ dev ]
jobs:
build-dev:
runs-on: host
steps:
- name: Pre-cleanup
run: |
docker system prune -f
docker builder prune -f --filter until=24h
- name: Checkout code (DEV)
run: |
echo "Current workspace: $(pwd)"
# Clean workspace and temp directory first
rm -rf * .git* 2>/dev/null || true
rm -rf /tmp/repo 2>/dev/null || true
# Clone the repository since Gitea runner doesn't auto-checkout
echo "Cloning repository..."
git clone --branch dev http://${{ secrets.username }}:${{ secrets.token }}@192.168.253.221:3000/sbcrumb/nfoguard.git /tmp/repo
cp -r /tmp/repo/* .
cp -r /tmp/repo/.* . 2>/dev/null || true
rm -rf /tmp/repo
echo "Repository cloned successfully"
ls -la
echo "Checking Dockerfile:"
head -30 Dockerfile
- name: Check Docker availability (DEV)
run: |
echo "Checking Docker installation..."
which docker || (echo "Docker not found in PATH" && exit 1)
docker --version || (echo "Docker not available" && exit 1)
docker info || (echo "Docker daemon not running" && exit 1)
- name: Configure Docker for local registry (DEV)
run: |
echo "Configuring Docker client for insecure local registry..."
LOCAL_REGISTRY="192.168.253.221:3000"
export DOCKER_CONFIG=/tmp/docker-config-dev
mkdir -p $DOCKER_CONFIG
# Create Docker client config for insecure registry
cat > $DOCKER_CONFIG/config.json << EOF
{
"auths": {},
"HttpHeaders": {
"User-Agent": "Docker-Client/20.10.0 (linux)"
},
"credsStore": "",
"credHelpers": {},
"experimental": "disabled"
}
EOF
echo "Docker client config created for DEV"
echo "Testing registry connectivity..."
curl -s "http://$LOCAL_REGISTRY/v2/" && echo "✅ Registry accessible via HTTP" || echo "❌ Registry not accessible"
- name: Build Docker image with caching (DEV)
run: |
echo "Building DEV Docker image with layer caching..."
LOCAL_REGISTRY="192.168.253.221:3000"
CACHE_IMAGE="$LOCAL_REGISTRY/sbcrumb/nfoguard:dev-buildcache"
# Clear any existing Docker configs to prevent permission issues
unset DOCKER_CONFIG
export HOME=/tmp/docker-home-dev
mkdir -p $HOME
# Enable Docker BuildKit for better caching
export DOCKER_BUILDKIT=1
# Check if DEV cache image exists and pull it
echo "Checking for existing DEV cache image..."
CACHE_ARGS=""
if curl -s "http://$LOCAL_REGISTRY/v2/sbcrumb/nfoguard/manifests/dev-buildcache" > /dev/null 2>&1; then
echo "✅ DEV cache manifest found, pulling image..."
if docker pull "$CACHE_IMAGE" 2>/dev/null; then
echo "✅ DEV cache image pulled successfully"
CACHE_ARGS="--cache-from=$CACHE_IMAGE"
else
echo "⚠️ DEV cache manifest exists but pull failed"
fi
else
echo "ℹ️ No DEV cache image found (first build or cache expired)"
fi
# Build DEV image with cache (if available)
echo "Building DEV image with layer caching..."
VERSION=$(cat VERSION 2>/dev/null || echo "unknown")
echo "Building version: $VERSION"
docker build \
$CACHE_ARGS \
--build-arg GIT_BRANCH=dev \
--build-arg BUILD_SOURCE=gitea \
--tag nfoguard:dev \
--tag nfoguard:${VERSION}-dev-gitea \
--tag nfoguard:dev-${{ github.sha }} \
--tag "$CACHE_IMAGE" \
.
echo "DEV Docker image built and tagged successfully"
- name: Login and Push to Gitea registry (DEV tags)
continue-on-error: true
run: |
echo "Using LOCAL IP ONLY for DEV push!"
export DOCKER_CONFIG=/tmp/docker-config-dev
LOCAL_REGISTRY="192.168.253.221:3000"
echo "Registry: $LOCAL_REGISTRY (DEV TAGS)"
# Try login to local registry
if echo "${{ secrets.token }}" | docker --config $DOCKER_CONFIG login "$LOCAL_REGISTRY" -u "${{ secrets.username }}" --password-stdin 2>/dev/null; then
echo "✅ DEV Login succeeded"
# Tag for local registry
VERSION=$(cat VERSION 2>/dev/null || echo "unknown")
docker tag nfoguard:dev "$LOCAL_REGISTRY/sbcrumb/nfoguard:dev"
docker tag nfoguard:${VERSION}-dev-gitea "$LOCAL_REGISTRY/sbcrumb/nfoguard:${VERSION}-dev-gitea"
docker tag nfoguard:dev "$LOCAL_REGISTRY/sbcrumb/nfoguard:dev-${{ github.sha }}"
# Push DEV images
echo "=== Pushing DEV cache image ==="
timeout 60 docker --config $DOCKER_CONFIG push "$LOCAL_REGISTRY/sbcrumb/nfoguard:dev-buildcache" && echo "✅ DEV cache pushed" || echo "⚠️ DEV cache push failed"
echo "=== Pushing DEV latest tag ==="
if timeout 120 docker --config $DOCKER_CONFIG push "$LOCAL_REGISTRY/sbcrumb/nfoguard:dev"; then
echo "✅ DEV tag pushed successfully"
echo "=== Pushing version-dev-gitea tag ==="
if timeout 60 docker --config $DOCKER_CONFIG push "$LOCAL_REGISTRY/sbcrumb/nfoguard:${VERSION}-dev-gitea"; then
echo "✅ Version-dev-gitea tag pushed successfully: ${VERSION}-dev-gitea"
fi
if timeout 60 docker --config $DOCKER_CONFIG push "$LOCAL_REGISTRY/sbcrumb/nfoguard:dev-${{ github.sha }}"; then
echo "✅ DEV SHA tag pushed successfully"
fi
fi
else
echo "❌ DEV local registry login failed - continuing to Docker Hub"
fi
echo ""
echo "🎉 DEV build completed successfully!"
echo ""
VERSION=$(cat VERSION 2>/dev/null || echo "unknown")
echo "📦 Available DEV images:"
echo " Local only: nfoguard:dev"
echo " Version tag: nfoguard:${VERSION}-dev-gitea"
echo " Local SHA: nfoguard:dev-${{ github.sha }}"
echo ""
echo "🐳 Pull with: docker pull 192.168.253.221:3000/sbcrumb/nfoguard:${VERSION}-dev-gitea"
.gitea/workflows/ci.yml
-240
View File
@@ -1,240 +0,0 @@
name: Local Docker Build (Main)
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
jobs:
build:
runs-on: host
steps:
- name: Pre-cleanup Docker space
run: |
echo "🧹 Cleaning up Docker space before build..."
docker system prune -f
docker builder prune -f --filter until=24h
docker image prune -af --filter until=48h
echo "📊 Docker space after cleanup:"
docker system df
- name: Checkout code
run: |
echo "Current workspace: $(pwd)"
# Clean workspace and temp directory first
rm -rf * .git* 2>/dev/null || true
rm -rf /tmp/repo 2>/dev/null || true
# Clone the repository since Gitea runner doesn't auto-checkout
echo "Cloning repository..."
git clone --branch main http://${{ secrets.username }}:${{ secrets.token }}@192.168.253.221:3000/sbcrumb/nfoguard.git /tmp/repo
cp -r /tmp/repo/* .
cp -r /tmp/repo/.* . 2>/dev/null || true
rm -rf /tmp/repo
echo "Repository cloned successfully"
ls -la
echo "Checking Dockerfile:"
head -30 Dockerfile
- name: Check Docker availability
run: |
echo "Checking Docker installation..."
which docker || (echo "Docker not found in PATH" && exit 1)
docker --version || (echo "Docker not available" && exit 1)
docker info || (echo "Docker daemon not running" && exit 1)
- name: Configure Docker for local registry
run: |
echo "Configuring Docker client for insecure local registry..."
LOCAL_REGISTRY="192.168.253.221:3000"
# We can't use sudo in the container, so we'll configure the client differently
export DOCKER_CONFIG=/tmp/docker-config
mkdir -p $DOCKER_CONFIG
# Create Docker client config for insecure registry
cat > $DOCKER_CONFIG/config.json << EOF
{
"auths": {},
"HttpHeaders": {
"User-Agent": "Docker-Client/20.10.0 (linux)"
},
"credsStore": "",
"credHelpers": {},
"experimental": "disabled"
}
EOF
echo "Docker client config created"
echo "Note: Since we can't modify daemon config in container, we'll use --insecure-registry flag"
echo "Testing registry connectivity..."
curl -s "http://$LOCAL_REGISTRY/v2/" && echo "✅ Registry accessible via HTTP" || echo "❌ Registry not accessible"
- name: Build Docker image with caching
run: |
echo "Building Docker image with layer caching..."
LOCAL_REGISTRY="192.168.253.221:3000"
CACHE_IMAGE="$LOCAL_REGISTRY/sbcrumb/nfoguard:buildcache"
# Clear any existing Docker configs to prevent permission issues
unset DOCKER_CONFIG
export HOME=/tmp/docker-home
mkdir -p $HOME
# Enable Docker BuildKit for better caching
export DOCKER_BUILDKIT=1
# Check if cache image exists and pull it
echo "Checking for existing cache image..."
CACHE_ARGS=""
if curl -s "http://$LOCAL_REGISTRY/v2/sbcrumb/nfoguard/manifests/buildcache" > /dev/null 2>&1; then
echo "✅ Cache manifest found, pulling image..."
if docker pull "$CACHE_IMAGE" 2>/dev/null; then
echo "✅ Cache image pulled successfully"
CACHE_ARGS="--cache-from=$CACHE_IMAGE"
else
echo "⚠️ Cache manifest exists but pull failed"
fi
else
echo "ℹ️ No cache image found (first build or cache expired)"
fi
# Build with cache (if available)
echo "Building with layer caching..."
VERSION=$(cat VERSION 2>/dev/null || echo "unknown")
echo "Building version: $VERSION"
docker build \
$CACHE_ARGS \
--build-arg GIT_BRANCH=main \
--build-arg BUILD_SOURCE=gitea \
--tag nfoguard:latest \
--tag nfoguard:${VERSION}-gitea \
--tag nfoguard:${{ github.sha }} \
--tag "$CACHE_IMAGE" \
.
echo "Docker image built and tagged successfully"
# Show layer info for debugging (with error handling)
echo "=== Image layer history ==="
if docker history nfoguard:latest --format "table {{.CreatedBy}}\t{{.Size}}" 2>/dev/null; then
echo "✅ Image history displayed successfully"
else
echo "⚠️ Could not display image history (permissions issue, but build succeeded)"
fi
- name: Login and Push to Gitea registry (local IP only)
continue-on-error: true # Don't fail the build if local registry fails
run: |
echo "Using LOCAL IP ONLY - no Cloudflare tunnel!"
export DOCKER_CONFIG=/tmp/docker-config
# Force local IP - never use domain
LOCAL_REGISTRY="192.168.253.221:3000"
echo "Registry: $LOCAL_REGISTRY (LOCAL IP ONLY)"
# Check image size
IMAGE_SIZE=$(docker images nfoguard:latest --format "table {{.Size}}" | tail -n1)
echo "Image size: $IMAGE_SIZE"
echo "Testing local registry connectivity..."
curl -s "http://$LOCAL_REGISTRY/v2/" && echo "✅ Local registry accessible via HTTP" || echo "❌ Local registry not accessible"
# Try multiple approaches to handle the HTTP issue
echo "Attempting login to LOCAL registry..."
# Approach 1: Try with explicit HTTP in auth
if echo "${{ secrets.token }}" | docker --config $DOCKER_CONFIG login "$LOCAL_REGISTRY" -u "${{ secrets.username }}" --password-stdin 2>/dev/null; then
echo "✅ Login succeeded with direct method"
# Tag for local registry (cache image already tagged in build step)
VERSION=$(cat VERSION 2>/dev/null || echo "unknown")
docker tag nfoguard:latest "$LOCAL_REGISTRY/sbcrumb/nfoguard:latest"
docker tag nfoguard:${VERSION}-gitea "$LOCAL_REGISTRY/sbcrumb/nfoguard:${VERSION}-gitea"
docker tag nfoguard:latest "$LOCAL_REGISTRY/sbcrumb/nfoguard:${{ github.sha }}"
echo "Pushing to LOCAL registry (gigabit speed!)..."
# Push with reasonable timeout for local network
echo "=== Pushing cache image (for faster future builds) ==="
timeout 60 docker --config $DOCKER_CONFIG push "$LOCAL_REGISTRY/sbcrumb/nfoguard:buildcache" && echo "✅ Cache image pushed" || echo "⚠️ Cache push failed"
echo "=== Pushing latest tag to LOCAL IP ==="
if timeout 120 docker --config $DOCKER_CONFIG push "$LOCAL_REGISTRY/sbcrumb/nfoguard:latest"; then
echo "✅ Latest tag pushed successfully to LOCAL registry"
echo "=== Pushing version-gitea tag ==="
if timeout 60 docker --config $DOCKER_CONFIG push "$LOCAL_REGISTRY/sbcrumb/nfoguard:${VERSION}-gitea"; then
echo "✅ Version-gitea tag pushed successfully: ${VERSION}-gitea"
fi
echo "=== Pushing SHA tag to LOCAL IP ==="
if timeout 60 docker --config $DOCKER_CONFIG push "$LOCAL_REGISTRY/sbcrumb/nfoguard:${{ github.sha }}"; then
echo "✅ SHA tag pushed successfully to LOCAL registry"
else
echo "⚠️ SHA tag push failed but latest succeeded"
fi
else
echo "❌ Push to LOCAL registry failed"
echo "This is likely because Docker daemon needs insecure-registry configuration"
fi
else
echo "❌ Local registry login failed"
echo "This is expected - Docker daemon needs insecure-registry configuration"
echo ""
echo "🔧 To fix, run on the host machine:"
echo "sudo mkdir -p /etc/docker"
echo 'echo '"'"'{"insecure-registries":["'$LOCAL_REGISTRY'"]}'"'"' | sudo tee /etc/docker/daemon.json'
echo "sudo systemctl restart docker"
echo ""
echo "⏭️ Continuing to Docker Hub push..."
fi
echo "🎉 Local registry step completed!"
echo ""
echo "🎉 Local build completed successfully!"
echo ""
VERSION=$(cat VERSION 2>/dev/null || echo "unknown")
echo "📦 Available images:"
echo " Local Docker: nfoguard:latest"
echo " Version tag: nfoguard:${VERSION}-gitea"
echo " Local SHA: nfoguard:${{ github.sha }}"
echo " Local Gitea: 192.168.253.221:3000/sbcrumb/nfoguard:latest"
echo ""
echo "🐳 Pull with: docker pull 192.168.253.221:3000/sbcrumb/nfoguard:${VERSION}-gitea"
- name: Post-cleanup Docker space
if: always() # Run even if build fails
run: |
echo "🧹 Final cleanup to prevent disk space issues..."
docker system prune -f
docker builder prune -af --filter until=24h
echo "📊 Final Docker space usage:"
docker system df
deploy:
needs: build
runs-on: host
if: github.ref == 'refs/heads/main'
steps:
- name: Deploy application
run: |
echo "Deploying application..."
# Add your deployment commands here
# Examples:
# docker-compose pull
# docker-compose up -d
# or
# docker stop nfoguard || true
# docker run -d --name nfoguard -p 8080:8080 nfoguard:latest
echo "Deployment completed"
Dockerfile
+2 -9
View File
@@ -12,12 +12,11 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
GIT_BRANCH=${GIT_BRANCH} \
BUILD_SOURCE=${BUILD_SOURCE}
# Install system dependencies including PostgreSQL client libraries and tini
# Install system dependencies including PostgreSQL client libraries
RUN apt-get update && apt-get install -y \
curl \
libpq-dev \
gcc \
tini \
&& rm -rf /var/lib/apt/lists/*
# Create app user and directory
@@ -32,9 +31,6 @@ RUN pip install --no-cache-dir --upgrade pip && \
# Copy application code
COPY . .
# Copy web starter file to root directory
COPY start_web.py .
# Create git metadata for version detection based on build arg
RUN mkdir -p .git && \
echo "ref: refs/heads/${GIT_BRANCH}" > .git/HEAD
@@ -55,7 +51,7 @@ RUN mkdir -p /app/emby-plugin && \
echo ' echo "No Emby plugins directory found - skipping plugin deployment"' >> /app/deploy-plugin.sh && \
echo ' echo "To enable plugin deployment, bind mount your Emby plugins directory to /emby-plugins"' >> /app/deploy-plugin.sh && \
echo 'fi' >> /app/deploy-plugin.sh && \
echo 'exec python -u main.py' >> /app/deploy-plugin.sh && \
echo 'exec python -u nfoguard.py' >> /app/deploy-plugin.sh && \
chmod +x /app/deploy-plugin.sh
# Set ownership
@@ -71,8 +67,5 @@ HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
# Expose port
EXPOSE 8080
# Use tini as init process to handle signals and zombie processes properly
ENTRYPOINT ["tini", "--"]
# Run the application with plugin deployment
CMD ["/app/deploy-plugin.sh"]
Emby-DLL/NFOGuard.Emby.Plugin.dll
BIN
View File
Binary file not shown.
MAINTAINARR_WEBHOOK.md
-207
View File
@@ -1,207 +0,0 @@
# Maintainarr Webhook Integration
This document describes how to configure NFOGuard to receive webhooks from Maintainarr for automatic database cleanup when media is removed.
## Overview
When Maintainarr removes media from your Plex/Radarr/Sonarr collections, NFOGuard can automatically remove the corresponding entries from its database to keep it clean and up-to-date.
## Webhook Configuration
### 1. NFOGuard Endpoint
The webhook endpoint is available at:
```
http://YOUR_NFOGUARD_HOST:8080/webhook/maintainarr
```
**Important**: Use the core NFOGuard container port (typically 8080), not the web interface port (8081).
### 2. Maintainarr Configuration
In Maintainarr, create a new webhook notification agent with these settings:
#### Basic Settings
- **Name**: `NFOGuard Cleanup`
- **Enabled**: ✅ Checked
- **Agent**: `Webhook`
#### Webhook Configuration
- **Webhook URL**: `http://YOUR_NFOGUARD_HOST:8080/webhook/maintainarr`
- **JSON Payload**: Use the template below
- **Auth Header**: *(Leave empty - no authentication required)*
#### JSON Payload Template
Copy and paste this JSON template into the "Json Payload" field:
```json
{
"notification_type": "{{notification_type}}",
"subject": "{{subject}}",
"message": "{{message}}",
"extra": "{{extra}}"
}
```
**Important Note**: Maintainarr's template variables may not include IMDb IDs directly. The webhook will attempt to extract IMDb IDs from the notification content, but this may require manual configuration or rule setup in Maintainarr to include IMDb IDs in the notification text.
**Alternative Approach**: If Maintainarr doesn't provide IMDb IDs in notifications, you may need to use NFOGuard's manual cleanup tools or configure Maintainarr rules to include IMDb information in the message content.
#### Event Types
Select these notification types:
- ✅ **Media Removed From Collection** - Removes media from NFOGuard database
- ✅ **Media About To Be Handled** - Optional: Log when media is about to be processed
Optional event types (will be logged but not processed):
- ☐ Media Added To Collection
- ☐ Media Handled
- ☐ Rule Handling Failed
- ☐ Collection Handling Failed
## Supported Operations
### Movies
When a movie is removed from Maintainarr:
- NFOGuard checks if the movie exists in its database (by IMDb ID)
- If found, removes the movie record from the `movies` table
- Logs the deletion operation
### TV Series
When a TV series is removed from Maintainarr:
- NFOGuard checks if the series exists in its database (by IMDb ID)
- If found, removes all episode records from the `episodes` table
- Removes the series record from the `series` table
- Logs the deletion operation with episode count
## Webhook Payload
Maintainarr sends webhook payloads using template variables that you configure:
```json
{
"notification_type": "Media Removed",
"subject": "Example Movie (2023)",
"message": "Removed movie Example Movie from collection Action Movies - IMDb: tt1234567",
"extra": "tt1234567"
}
```
### How It Works
1. **Maintainarr** populates the template variables ({{notification_type}}, {{subject}}, {{message}}, {{extra}})
2. **NFOGuard** receives the webhook and parses the content to extract:
- **IMDb ID**: Extracted from `message`, `subject`, or `extra` fields using pattern matching
- **Media Type**: Determined from message content keywords or database lookup
- **Title**: Extracted from `subject` or `message` fields
### Media Identification
NFOGuard looks for IMDb IDs in this format:
- `tt1234567` (preferred)
- `1234567` (will be converted to tt1234567)
The webhook handler uses intelligent parsing to:
- Extract IMDb IDs from any field using regex patterns
- Determine if media is a Movie or Series based on keywords or database lookup
- Extract the media title from subject or message content
## Response Format
NFOGuard responds with JSON indicating the result:
### Success Response
```json
{
"status": "success",
"message": "Processed Media Removed for Example Movie",
"media_type": "Movie",
"imdb_id": "tt1234567",
"removed_count": 1,
"removed_items": ["Movie: Example Movie (tt1234567)"]
}
```
### Ignored Response
```json
{
"status": "ignored",
"reason": "Media tt1234567 not found in database"
}
```
### Error Response
```json
{
"status": "error",
"message": "No IMDb ID found in webhook payload"
}
```
## Logging
All webhook activities are logged with details:
```
INFO: Received Maintainarr webhook: Media Removed
INFO: Processing movie deletion for Example Movie (tt1234567)
SUCCESS: Removed movie Example Movie (tt1234567) from database
INFO: Maintainarr cleanup: Media Removed - Movie 'Example Movie' (tt1234567). Removed from database: Movie: Example Movie (tt1234567)
```
## Troubleshooting
### Common Issues
1. **No IMDb ID Found**:
- Maintainarr template variables may not include IMDb IDs
- Check if the notification message contains IMDb information
- You may need to manually include IMDb IDs in Maintainarr rule configurations
2. **Media Not Found**:
- Check if the media exists in NFOGuard's database
- Verify the IMDb ID matches between Maintainarr and NFOGuard
3. **Connection Issues**:
- Ensure NFOGuard core container is accessible on port 8080
- Check firewall settings and network connectivity
4. **Authentication Errors**:
- No authentication is required for the webhook endpoint
- Ensure you're using the core container port, not web interface port
5. **Test Notifications**:
- Test notifications (like the one you just sent) will be acknowledged but not processed
- Real media removal events will trigger the cleanup process
### Testing the Webhook
You can test the webhook manually using curl:
```bash
curl -X POST http://YOUR_NFOGUARD_HOST:8080/webhook/maintainarr \
-H "Content-Type: application/json" \
-d '{
"notification_type": "Media Removed",
"subject": "Test Movie (2023)",
"message": "Removed movie Test Movie from collection - IMDb: tt1234567",
"extra": "tt1234567"
}'
```
## Security Considerations
- The webhook endpoint does not require authentication
- Consider using firewalls or network restrictions to limit access
- The endpoint only processes deletion requests, not additions
- All operations are logged for audit purposes
## Integration Benefits
- **Automatic Cleanup**: Keeps NFOGuard database synchronized with your media collection
- **Accurate Statistics**: Dashboard stats reflect only currently available media
- **Reduced Manual Maintenance**: No need to manually clean up orphaned entries
- **Audit Trail**: All deletions are logged with full details
## Version Compatibility
- NFOGuard: v2.8.0+
- Maintainarr: All versions with webhook support
- Requires NFOGuard core container (processing container), not web-only container
README.md
+31 -195
View File
@@ -8,59 +8,41 @@
---
> **⚠️ ALPHA SOFTWARE NOTICE ⚠️**
>
> NFOGuard is currently in **Alpha** stage. While functional, it may have bugs or missing features.
>
> **🔌 Emby Plugin Included**: The Emby companion plugin is now bundled directly into the Docker image — no extra steps required.
>
> **💬 Community Feedback**: Join our Discord if you'd like to share feedback, test new features, or discuss improvements with other users:
> **💬 Community Feedback**: Join our Discord if you’d like to share feedback, test new features early, or discuss improvements with other users:
>
> **[Join Discord: https://discord.gg/ZykJRGt72b](https://discord.gg/ZykJRGt72b)**
> **[Join Discord: https://discord.gg/bbD9Pmtr](https://discord.gg/bbD9Pmtr)**
>
> *If the Discord link has expired, please [open an issue](https://github.com/sbcrumb/NFOguard/issues) and we'll provide an updated link.*
---
NFOGuard automatically updates movie and TV show NFO files with proper release dates and metadata when triggered by Radarr/Sonarr webhooks. It preserves existing metadata while adding clean, accurate date information at the bottom of NFO files.
## Features
## ✨ Features
### **Core Media Management**
- **Movie & TV Support** - Works with both Radarr and Sonarr
- **Smart Date Handling** - Prioritizes digital, physical, and theatrical release dates
- **Webhook Integration** - Triggers automatically on import, upgrade, and rename
- **NFO Preservation** - Maintains existing metadata, adds fields cleanly at bottom
- **Metadata Locking** - Prevents overwrites with lockdata tags
- 🎬 **Movie & TV Support** - Works with both Radarr and Sonarr
- 📅 **Smart Date Handling** - Prioritizes digital, physical, and theatrical release dates
- 🔄 **Webhook Integration** - Triggers automatically on import, upgrade, and rename
- 🗄️ **Database Integration** - Direct PostgreSQL access for better performance
- 📝 **NFO Preservation** - Maintains existing metadata, adds fields cleanly at bottom
- 🔒 **Metadata Locking** - Prevents overwrites with lockdata tags
- ⚡ **Batch Processing** - Efficient handling of multiple files
- 🐳 **Docker Ready** - Easy deployment with Docker Compose
### **Performance & Scalability**
- **Async I/O Operations** - High-performance concurrent file processing
- **Batch Processing** - Efficient handling of multiple files simultaneously
- **PostgreSQL Database** - Production-ready database with optimized queries
- **Smart Skip Logic** - Database-first checking eliminates expensive filesystem scans
- **88% Scan Optimization** - Subsequent scans reduced from hours to minutes via smart skip logic
### **Web Interface & Management**
- **Complete Web UI** - Episode and movie management with filtering and search
- **Database Cleanup Tools** - Delete orphaned episodes with confirmation dialogs
- **Real-time Statistics** - Live episode counts and source mapping
- **Manual Scan Control** - Smart, full, and incomplete scan modes
- **Health Monitoring** - System status and performance metrics
### **Configuration & Validation**
- **Comprehensive Config Validation** - Validates all settings before startup
- **Runtime Health Checks** - Monitors system health and dependencies
- **Path Validation** - Ensures media directories are accessible
- **Database Connectivity Tests** - Validates database connections
### **Production Ready**
- **Docker & Kubernetes** - Health checks for orchestration platforms
- **Graceful Shutdown** - Proper signal handling for container management
- **Configuration CLI** - Validation tools for troubleshooting
- **Modular Architecture** - Clean separation of concerns for maintainability
## Quick Start
## 🚀 Quick Start
### 1. Download Configuration Files
```bash
wget https://raw.githubusercontent.com/sbcrumb/nfoguard/main/.env.template
wget https://raw.githubusercontent.com/sbcrumb/nfoguard/main/.env.secrets.template
wget https://raw.githubusercontent.com/sbcrumb/nfoguard/main/docker-compose.example.yml
wget https://raw.githubusercontent.com/sbcrumb/NFOguard/main/.env.template
wget https://raw.githubusercontent.com/sbcrumb/NFOguard/main/.env.secrets.template
wget https://raw.githubusercontent.com/sbcrumb/NFOguard/main/docker-compose.example.yml
```
### 2. Configure Environment
@@ -96,7 +78,7 @@ docker-compose logs -f nfoguard
curl http://localhost:8080/health
```
## Configuration
## ⚙️ Configuration
### Environment Files
@@ -129,7 +111,7 @@ DEBUG=false # Clean production logs
SUPPRESS_TVDB_WARNINGS=true # Hide non-critical API failures
```
## Docker Images
## 🐳 Docker Images
### Production (Stable)
```yaml
@@ -141,13 +123,11 @@ image: sbcrumb/nfoguard:latest
image: sbcrumb/nfoguard:dev
```
### Specific Version
### Specific Version
```yaml
image: sbcrumb/nfoguard:v2.6.7 # Latest with PostgreSQL & optimization
image: sbcrumb/nfoguard:v1.5.5
```
> **🚀 Version 2.6.7** includes major architecture improvements, PostgreSQL database migration, 88% scan optimization, async I/O performance enhancements, comprehensive monitoring, and configuration validation.
## 🔗 Webhook Setup
Configure these webhook URLs in your applications:
@@ -178,88 +158,17 @@ curl -X POST "http://localhost:8080/manual/scan?path=/media/movies"
# Bulk update all movies from Radarr database
curl -X POST "http://localhost:8080/bulk/update"
# Verify NFO files match database data
curl -X POST "http://localhost:8080/database/verify/nfo-sync?media_type=both"
# Fix NFO files that don't match database
curl -X POST "http://localhost:8080/database/fix/nfo-sync?media_type=both"
```
### NFO Verification & Synchronization
NFOGuard includes comprehensive verification tools to ensure NFO files remain synchronized with database dates:
```bash
# Verify all NFO files (movies and episodes)
curl -X POST "http://localhost:8080/database/verify/nfo-sync?media_type=both"
# Verify only movies
curl -X POST "http://localhost:8080/database/verify/nfo-sync?media_type=movies"
# Verify only TV episodes
curl -X POST "http://localhost:8080/database/verify/nfo-sync?media_type=episodes"
# Fix detected issues by regenerating NFO files
curl -X POST "http://localhost:8080/database/fix/nfo-sync?media_type=both"
```
**Verification Checks:**
- **Missing NFO files** - Database entries without corresponding NFO files
- **Empty NFO files** - NFO files that exist but contain no content
- **Date mismatches** - Database dates don't match NFO file dates
- **Source mismatches** - Database source doesn't match NFO source information
**Fix Operations:**
- Regenerates NFO files from database data for any detected issues
- Preserves existing metadata while updating NFOGuard date sections
- Only operates when `MANAGE_NFO=true` is configured
- Creates proper movie.nfo and episode S##E##.nfo files
### API Endpoints
#### **Core Operations**
| Endpoint | Method | Purpose |
|----------|--------|---------|
| `/health` | GET | Health check |
| `/webhook/radarr` | POST | Radarr webhook handler |
| `/webhook/sonarr` | POST | Sonarr webhook handler |
| `/manual/scan` | POST | Manual media scanning |
| `/bulk/update` | POST | Bulk update from Radarr database |
#### **Health & Monitoring**
| Endpoint | Method | Purpose |
|----------|--------|---------|
| `/api/v1/health` | GET | Comprehensive health status |
| `/api/v1/health/ready` | GET | Kubernetes readiness probe |
| `/api/v1/health/live` | GET | Kubernetes liveness probe |
| `/api/v1/status` | GET | Complete system status |
| `/api/v1/status/brief` | GET | Quick system overview |
#### **Metrics & Performance**
| Endpoint | Method | Purpose |
|----------|--------|---------|
| `/api/v1/metrics` | GET | Prometheus metrics (text format) |
| `/api/v1/metrics/json` | GET | Metrics in JSON format |
| `/api/v1/metrics/processing` | GET | Processing-specific metrics |
| `/api/v1/metrics/errors` | GET | Error metrics and recent failures |
| `/api/v1/metrics/system` | GET | System resource metrics |
#### **Database Management**
| Endpoint | Method | Purpose |
|----------|--------|---------|
| `/database/backfill/movie-release-dates` | POST | Backfill missing release dates for existing movies |
| `/database/cleanup/orphaned-episodes` | POST | Delete episodes without video files |
| `/database/cleanup/orphaned-movies` | POST | Delete movies without video files |
| `/database/cleanup/orphaned-series` | POST | Delete TV series without directories |
| `/database/verify/nfo-sync` | POST | Verify that database dates match NFO file contents |
| `/database/fix/nfo-sync` | POST | Fix NFO sync issues by regenerating NFO files from database data |
#### **Configuration & Debugging**
| Endpoint | Method | Purpose |
|----------|--------|---------|
| `/debug/movie/{imdb_id}/priority` | GET | Debug movie priority logic |
| `/debug/tmdb/{imdb_id}` | GET | Debug TMDB lookup |
| `/ping` | GET | Simple connectivity test |
| `/bulk/update` | POST | Bulk movie updates from Radarr DB |
### Manual Scan Parameters
@@ -298,7 +207,7 @@ volumes:
- /home/user/media/tv:/media/TV/tv:rw
```
## Troubleshooting
## 🔧 Troubleshooting
### Check Logs
```bash
@@ -317,79 +226,6 @@ PATH_DEBUG=true
curl http://localhost:8080/health
```
## Monitoring & Observability
NFOGuard provides comprehensive monitoring capabilities for production deployments:
### **Health Checks**
```bash
# Basic health status
curl http://localhost:8080/api/v1/health
# Kubernetes readiness probe
curl http://localhost:8080/api/v1/health/ready
# Kubernetes liveness probe
curl http://localhost:8080/api/v1/health/live
# Quick system overview
curl http://localhost:8080/api/v1/status/brief
```
### **Metrics for Grafana/Prometheus**
```bash
# Prometheus metrics format
curl http://localhost:8080/api/v1/metrics
# JSON metrics for dashboards
curl http://localhost:8080/api/v1/metrics/json
# Processing performance metrics
curl http://localhost:8080/api/v1/metrics/processing
# Error rates and recent failures
curl http://localhost:8080/api/v1/metrics/errors
# System resource usage
curl http://localhost:8080/api/v1/metrics/system
```
### **Configuration Validation**
```bash
# Validate configuration before deployment
python config/validation_cli.py
# Include runtime tests (database, APIs, file access)
python config/validation_cli.py --runtime
# JSON output for automation
python config/validation_cli.py --runtime --json
```
### **Docker Health Checks**
Add to your `docker-compose.yml`:
```yaml
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8080/api/v1/health/live"]
interval: 30s
timeout: 10s
retries: 3
start_period: 40s
```
### **Structured Logging**
NFOGuard uses structured JSON logging with correlation IDs for request tracing:
```bash
# Enable structured logging
STRUCTURED_LOGGING=true
# Log level configuration
LOG_LEVEL=INFO
# Example log output with correlation ID
{"timestamp": "2024-01-01T12:00:00Z", "level": "INFO", "correlation_id": "req_123", "message": "Webhook received", "context": {"webhook_type": "radarr", "media_type": "movie"}}
```
### Common Issues
1. **Permission Errors**: Ensure NFOGuard can write to mounted directories
@@ -397,7 +233,7 @@ LOG_LEVEL=INFO
3. **Webhooks**: Check URLs and ensure port 8080 is accessible
4. **Database**: Verify PostgreSQL credentials in `.env.secrets`
## What NFOGuard Does
## 📊 What NFOGuard Does
### Before
```xml
@@ -433,7 +269,7 @@ LOG_LEVEL=INFO
NFOGuard identifies movies and TV shows using two methods: directory names with IMDb IDs (primary) or NFO files with IMDb IDs (fallback). Your media should follow these conventions:
### **Movies**
### 🎬 **Movies**
**Directory Structure:**
```
@@ -539,7 +375,7 @@ NFOGuard will ignore:
## 🆘 Support
- **Issues**: [GitHub Issues](https://github.com/sbcrumb/nfoguard/issues)
- **Issues**: [GitHub Issues](https://github.com/sbcrumb/NFOguard/issues)
- **Documentation**: See `SETUP.md` for detailed instructions
- **Docker Hub**: [`sbcrumb/nfoguard`](https://hub.docker.com/r/sbcrumb/nfoguard)
SETUP.md
+1 -1
View File
@@ -132,7 +132,7 @@ services:
- ./.env:/app/.env:ro # Main configuration
- ./.env.secrets:/app/.env.secrets:ro # Secrets
environment:
- CORE_API_PORT=8080
- PORT=8080
depends_on:
- radarr-postgres
```
VERSION
+1 -1
View File
@@ -1 +1 @@
2.10.0-skipped-imdb-edit-v6
2.1.5-preserve-long-nfo-episodes-only
api/auth.py
-184
View File
@@ -1,184 +0,0 @@
"""
Simple authentication middleware for NFOGuard web interface
Provides basic HTTP auth and session management for web interface protection
"""
import secrets
import hashlib
from datetime import datetime, timedelta
from typing import Optional, Dict, Any
from fastapi import HTTPException, status, Request, Response
from fastapi.security import HTTPBasic, HTTPBasicCredentials
from starlette.middleware.base import BaseHTTPMiddleware
class AuthSession:
"""Simple session management for web interface"""
def __init__(self, timeout_seconds: int = 3600):
self.sessions: Dict[str, Dict[str, Any]] = {}
self.timeout_seconds = timeout_seconds
def create_session(self, username: str) -> str:
"""Create a new session and return session token"""
session_token = secrets.token_urlsafe(32)
self.sessions[session_token] = {
"username": username,
"created_at": datetime.utcnow(),
"last_activity": datetime.utcnow()
}
return session_token
def validate_session(self, session_token: str) -> bool:
"""Validate session token and update last activity"""
if not session_token or session_token not in self.sessions:
return False
session = self.sessions[session_token]
now = datetime.utcnow()
# Check if session expired
if (now - session["last_activity"]).seconds > self.timeout_seconds:
del self.sessions[session_token]
return False
# Update last activity
session["last_activity"] = now
return True
def get_session_user(self, session_token: str) -> Optional[str]:
"""Get username from valid session"""
if self.validate_session(session_token):
return self.sessions[session_token]["username"]
return None
def delete_session(self, session_token: str) -> None:
"""Delete a session (logout)"""
if session_token in self.sessions:
del self.sessions[session_token]
def cleanup_expired_sessions(self) -> None:
"""Remove expired sessions"""
now = datetime.utcnow()
expired_tokens = []
for token, session in self.sessions.items():
if (now - session["last_activity"]).seconds > self.timeout_seconds:
expired_tokens.append(token)
for token in expired_tokens:
del self.sessions[token]
class SimpleAuthMiddleware(BaseHTTPMiddleware):
"""Simple authentication middleware for web interface routes"""
def __init__(self, app, config, session_manager=None):
super().__init__(app)
self.config = config
self.session_manager = session_manager or AuthSession(config.web_auth_session_timeout)
self.security = HTTPBasic()
# Routes that require authentication (web interface)
self.protected_routes = [
"/", # Main web interface
"/static/", # Static files (CSS, JS)
"/api/movies", # Web API endpoints
"/api/series",
"/api/episodes",
"/api/dashboard"
]
# Routes that are always public (webhooks, health checks, API endpoints)
self.public_routes = [
"/webhook/",
"/health",
"/logo/", # Logo files should always be accessible
"/favicon.ico", # Favicon should always be accessible
"/ping",
"/api/v1/health",
"/api/v1/metrics",
"/database/", # Database management endpoints (API access)
"/manual/", # Manual scan endpoints (API access)
"/debug/", # Debug endpoints (API access)
"/test/", # Test endpoints (API access)
"/bulk/" # Bulk operation endpoints (API access)
]
async def dispatch(self, request: Request, call_next):
"""Process request through authentication middleware"""
# Skip authentication if disabled
if not self.config.web_auth_enabled:
return await call_next(request)
# Check if route requires authentication
path = request.url.path
needs_auth = any(path.startswith(route) for route in self.protected_routes)
is_public = any(path.startswith(route) for route in self.public_routes)
if is_public or not needs_auth:
return await call_next(request)
# Check for existing session
session_token = request.cookies.get("nfoguard_session")
if session_token and self.session_manager.validate_session(session_token):
# Valid session, proceed
return await call_next(request)
# Check for HTTP Basic Auth
auth_header = request.headers.get("authorization")
if auth_header and auth_header.startswith("Basic "):
credentials = self._parse_basic_auth(auth_header)
if credentials and self._validate_credentials(credentials.username, credentials.password):
# Create session for successful login
session_token = self.session_manager.create_session(credentials.username)
response = await call_next(request)
response.set_cookie(
key="nfoguard_session",
value=session_token,
max_age=self.config.web_auth_session_timeout,
httponly=True,
secure=False # Set to True if using HTTPS
)
return response
# Authentication required
return self._auth_required_response()
def _parse_basic_auth(self, auth_header: str) -> Optional[HTTPBasicCredentials]:
"""Parse HTTP Basic Auth header"""
try:
import base64
encoded_credentials = auth_header.split(" ")[1]
decoded_credentials = base64.b64decode(encoded_credentials).decode('utf-8')
username, password = decoded_credentials.split(":", 1)
return HTTPBasicCredentials(username=username, password=password)
except Exception:
return None
def _validate_credentials(self, username: str, password: str) -> bool:
"""Validate username and password"""
return (username == self.config.web_auth_username and
password == self.config.web_auth_password)
def _auth_required_response(self) -> Response:
"""Return 401 response with WWW-Authenticate header"""
return Response(
content="Authentication required",
status_code=status.HTTP_401_UNAUTHORIZED,
headers={"WWW-Authenticate": "Basic realm=\"NFOGuard Web Interface\""}
)
def create_auth_dependencies(config) -> Dict[str, Any]:
"""Create authentication-related dependencies for dependency injection"""
session_manager = AuthSession(config.web_auth_session_timeout)
return {
"session_manager": session_manager,
"auth_enabled": config.web_auth_enabled,
"auth_config": {
"username": config.web_auth_username,
"timeout": config.web_auth_session_timeout
}
}
api/models.py
+1 -132
View File
@@ -31,18 +31,6 @@ class RadarrWebhook(BaseModel):
extra = "allow"
class MaintainarrWebhook(BaseModel):
"""Maintainarr webhook payload model - uses template variables"""
notification_type: Optional[str] = None # e.g., "Media Removed"
subject: Optional[str] = None
message: Optional[str] = None
image: Optional[str] = None
extra: Optional[str] = None
class Config:
extra = "allow"
class HealthResponse(BaseModel):
"""Health check response model"""
status: str
@@ -61,124 +49,5 @@ class TVSeasonRequest(BaseModel):
class TVEpisodeRequest(BaseModel):
"""TV episode processing request model"""
series_path: str
season: int
episode: int
# Web interface models
class MovieUpdateRequest(BaseModel):
"""Request to update movie dateadded"""
dateadded: Optional[str]
source: str
class EpisodeUpdateRequest(BaseModel):
"""Request to update episode dateadded"""
dateadded: Optional[str]
source: str
class BulkUpdateRequest(BaseModel):
"""Request for bulk source updates"""
media_type: str # "movies" or "episodes"
old_source: str
new_source: str
class MovieResponse(BaseModel):
"""Movie data response"""
imdb_id: str
title: str
path: str
released: Optional[str]
dateadded: Optional[str]
source: Optional[str]
has_video_file: bool
last_updated: str
class SeriesResponse(BaseModel):
"""TV series data response"""
imdb_id: str
title: str
path: str
last_updated: str
total_episodes: int
episodes_with_dates: int
episodes_with_video: int
class EpisodeResponse(BaseModel):
"""TV episode data response"""
season: int
episode: int
aired: Optional[str]
dateadded: Optional[str]
source: Optional[str]
has_video_file: bool
last_updated: str
series_path: str
season_name: str
episode_name: str
# Scheduled Scans Models
class CreateScheduledScanRequest(BaseModel):
"""Request model for creating a scheduled scan"""
name: str
description: Optional[str] = None
cron_expression: str
media_type: str # 'tv', 'movies', 'both'
scan_mode: str # 'smart', 'full', 'incomplete'
specific_paths: Optional[str] = None
enabled: bool = True
class UpdateScheduledScanRequest(BaseModel):
"""Request model for updating a scheduled scan"""
name: Optional[str] = None
description: Optional[str] = None
cron_expression: Optional[str] = None
media_type: Optional[str] = None
scan_mode: Optional[str] = None
specific_paths: Optional[str] = None
enabled: Optional[bool] = None
class ScheduledScanResponse(BaseModel):
"""Response model for scheduled scan data"""
id: int
name: str
description: Optional[str]
cron_expression: str
media_type: str
scan_mode: str
specific_paths: Optional[str]
enabled: bool
created_at: str
updated_at: str
last_run_at: Optional[str]
next_run_at: Optional[str]
run_count: int
created_by: Optional[str]
updated_by: Optional[str]
class ScheduleExecutionResponse(BaseModel):
"""Response model for schedule execution data"""
id: int
schedule_id: int
schedule_name: str
started_at: str
completed_at: Optional[str]
status: str
media_type: str
scan_mode: str
items_processed: int
items_skipped: int
items_failed: int
execution_time_seconds: Optional[int]
error_message: Optional[str]
logs: Optional[str]
triggered_by: Optional[str]
episode_name: str
api/monitoring_routes.py
-351
View File
@@ -1,351 +0,0 @@
"""
Monitoring API Routes for NFOGuard
Provides health checks, metrics, and system status endpoints
"""
from fastapi import APIRouter, Response, HTTPException
from typing import Dict, Any, Optional
import time
from monitoring.health import health_checker, HealthStatus
from monitoring.metrics import metrics
try:
from config.validator import get_configuration_summary
except ImportError:
def get_configuration_summary():
return {"status": "Configuration validator not available"}
router = APIRouter(prefix="/api/v1", tags=["monitoring"])
@router.get("/health")
async def get_health_status():
"""
Get comprehensive health status
Returns detailed health information including:
- Overall system health
- Individual component health checks
- Performance metrics
- Error status
"""
try:
health_status = await health_checker.get_full_health_status()
# Set appropriate HTTP status code
if health_status.status == HealthStatus.HEALTHY:
status_code = 200
elif health_status.status == HealthStatus.DEGRADED:
status_code = 200 # Still operational
else: # UNHEALTHY
status_code = 503 # Service unavailable
return Response(
content=health_status.to_dict(),
status_code=status_code,
media_type="application/json"
)
except Exception as e:
# Return unhealthy status if health check itself fails
return Response(
content={
"status": "unhealthy",
"message": f"Health check failed: {e}",
"timestamp": time.time()
},
status_code=500,
media_type="application/json"
)
@router.get("/health/ready")
async def get_readiness_status():
"""
Kubernetes readiness probe endpoint
Returns 200 if service is ready to accept traffic
Returns 503 if service is not ready
"""
try:
readiness = await health_checker.get_readiness_status()
status_code = 200 if readiness["ready"] else 503
return Response(
content=readiness,
status_code=status_code,
media_type="application/json"
)
except Exception as e:
return Response(
content={
"ready": False,
"message": f"Readiness check failed: {e}",
"timestamp": time.time()
},
status_code=503,
media_type="application/json"
)
@router.get("/health/live")
async def get_liveness_status():
"""
Kubernetes liveness probe endpoint
Returns 200 if service is alive and responsive
Returns 500 if service should be restarted
"""
try:
liveness = await health_checker.get_liveness_status()
status_code = 200 if liveness["alive"] else 500
return Response(
content=liveness,
status_code=status_code,
media_type="application/json"
)
except Exception as e:
return Response(
content={
"alive": False,
"message": f"Liveness check failed: {e}",
"timestamp": time.time()
},
status_code=500,
media_type="application/json"
)
@router.get("/metrics")
async def get_prometheus_metrics():
"""
Prometheus-compatible metrics endpoint
Returns metrics in Prometheus text format for scraping
"""
try:
prometheus_metrics = metrics.get_prometheus_metrics()
return Response(
content=prometheus_metrics,
media_type="text/plain; charset=utf-8"
)
except Exception as e:
raise HTTPException(
status_code=500,
detail=f"Failed to generate metrics: {e}"
)
@router.get("/metrics/json")
async def get_metrics_json():
"""
Get all metrics in JSON format
Returns structured metrics data including:
- System metrics (CPU, memory, disk)
- Processing metrics (rates, durations)
- Error metrics (counts, recent errors)
"""
try:
all_metrics = metrics.get_all_metrics()
return all_metrics
except Exception as e:
raise HTTPException(
status_code=500,
detail=f"Failed to get metrics: {e}"
)
@router.get("/status")
async def get_system_status():
"""
Get comprehensive system status
Returns detailed system information including:
- Health status
- Configuration summary
- Performance metrics
- Recent activity
"""
try:
# Get health status
health_status = await health_checker.get_full_health_status()
# Get metrics
all_metrics = metrics.get_all_metrics()
# Get configuration summary
config_summary = get_configuration_summary()
# Combine into comprehensive status
status_response = {
"overall_status": health_status.status.value,
"timestamp": time.time(),
"uptime_seconds": health_status.uptime_seconds,
"version": health_status.version,
"health": health_status.to_dict(),
"metrics": all_metrics,
"configuration": config_summary,
"summary": {
"service_healthy": health_status.status in [HealthStatus.HEALTHY, HealthStatus.DEGRADED],
"total_webhooks_processed": all_metrics["processing"]["total_webhooks"],
"total_nfo_files_created": all_metrics["processing"]["total_nfo_created"],
"total_errors": all_metrics["processing"]["total_errors"],
"current_processing_rate": all_metrics["processing"]["webhooks_received_per_minute"],
"average_processing_time": all_metrics["processing"]["average_processing_time_seconds"]
}
}
return status_response
except Exception as e:
raise HTTPException(
status_code=500,
detail=f"Failed to get system status: {e}"
)
@router.get("/status/brief")
async def get_brief_status():
"""
Get brief system status for quick monitoring
Returns essential status information without detailed metrics
"""
try:
# Get basic health
liveness = await health_checker.get_liveness_status()
readiness = await health_checker.get_readiness_status()
# Get basic metrics
processing_metrics = metrics.get_processing_metrics()
system_metrics = metrics.get_system_metrics()
return {
"status": "healthy" if liveness["alive"] and readiness["ready"] else "unhealthy",
"alive": liveness["alive"],
"ready": readiness["ready"],
"uptime_seconds": liveness["uptime_seconds"],
"webhooks_per_minute": processing_metrics["webhooks_received_per_minute"],
"active_operations": processing_metrics["active_operations"],
"total_errors": processing_metrics["total_errors"],
"cpu_percent": system_metrics.get("cpu_percent", 0),
"memory_percent": system_metrics.get("memory_percent", 0),
"timestamp": time.time()
}
except Exception as e:
raise HTTPException(
status_code=500,
detail=f"Failed to get brief status: {e}"
)
@router.get("/metrics/processing")
async def get_processing_metrics():
"""
Get processing-specific metrics
Returns metrics focused on NFO processing performance
"""
try:
processing_metrics = metrics.get_processing_metrics()
return processing_metrics
except Exception as e:
raise HTTPException(
status_code=500,
detail=f"Failed to get processing metrics: {e}"
)
@router.get("/metrics/errors")
async def get_error_metrics():
"""
Get error-specific metrics
Returns error counts, types, and recent error information
"""
try:
error_metrics = metrics.get_error_metrics()
return error_metrics
except Exception as e:
raise HTTPException(
status_code=500,
detail=f"Failed to get error metrics: {e}"
)
@router.get("/metrics/system")
async def get_system_metrics():
"""
Get system resource metrics
Returns CPU, memory, disk, and process information
"""
try:
system_metrics = metrics.get_system_metrics()
return system_metrics
except Exception as e:
raise HTTPException(
status_code=500,
detail=f"Failed to get system metrics: {e}"
)
@router.post("/metrics/reset")
async def reset_metrics(metric_types: Optional[str] = None):
"""
Reset specific metric types
Parameters:
- metric_types: Comma-separated list of metric types to reset
(counters, histograms, errors, timeseries)
If not specified, resets all metrics
"""
try:
reset_types = None
if metric_types:
reset_types = [t.strip() for t in metric_types.split(",")]
metrics.reset_metrics(reset_types)
return {
"message": "Metrics reset successfully",
"reset_types": reset_types or "all",
"timestamp": time.time()
}
except Exception as e:
raise HTTPException(
status_code=500,
detail=f"Failed to reset metrics: {e}"
)
# Legacy endpoints for backwards compatibility
@router.get("/health-check")
async def legacy_health_check():
"""Legacy health check endpoint (redirects to /health)"""
return await get_brief_status()
@router.get("/ping")
async def ping():
"""Simple ping endpoint for basic connectivity testing"""
return {
"message": "pong",
"timestamp": time.time(),
"service": "nfoguard",
"version": "2.0.0"
}
api/routes.py
+44 -2833
View File
File diff suppressed because it is too large Load Diff
api/web_routes.py
-2657
View File
File diff suppressed because it is too large Load Diff
clients/external_clients.py
-30
View File
@@ -659,36 +659,6 @@ class ExternalClientManager:
return None
return self.tvdb.imdb_to_tvdb_series_id(imdb_id)
def get_episode_air_date(self, imdb_id: str, season: int, episode: int) -> Optional[str]:
"""Get episode air date from external sources"""
_log("DEBUG", f"Looking for air date for {imdb_id} S{season:02d}E{episode:02d}")
# Try TMDB first if available
if self.tmdb.enabled:
# Find TV show by IMDB ID
tv_find_result = self.tmdb._get(f"/find/{imdb_id}", {"external_source": "imdb_id"})
if tv_find_result and tv_find_result.get("tv_results"):
tv_show = tv_find_result["tv_results"][0]
tv_id = tv_show.get("id")
if tv_id:
_log("DEBUG", f"Found TMDB TV ID {tv_id} for {imdb_id}")
episodes = self.tmdb.get_tv_season_episodes(tv_id, season)
if episode in episodes:
air_date = episodes[episode]
_log("INFO", f"Found TMDB air date for {imdb_id} S{season:02d}E{episode:02d}: {air_date}")
return _parse_date_to_iso(air_date)
# Try OMDb as fallback
if self.omdb.enabled:
episodes = self.omdb.get_tv_season_episodes(imdb_id, season)
if episode in episodes:
air_date = episodes[episode]
_log("INFO", f"Found OMDb air date for {imdb_id} S{season:02d}E{episode:02d}: {air_date}")
return _parse_date_to_iso(air_date)
_log("WARNING", f"No air date found for {imdb_id} S{season:02d}E{episode:02d}")
return None
if __name__ == "__main__":
clients/radarr_db_client.py
+1 -48
View File
@@ -164,54 +164,7 @@ class RadarrDbClient:
_log("ERROR", f"Database query error for IMDb {imdb_id}: {e}")
return None
def get_all_movies(self) -> List[Dict[str, Any]]:
"""
Get all movies from Radarr database (only movies with files)
Returns:
List of dictionaries with movie info
"""
query = """
SELECT
m."Id" as id,
m."Path" as path,
m."Added" as added,
mm."ImdbId" as imdb_id,
mm."Title" as title,
mm."Year" as year,
mm."DigitalRelease" as digital_release,
mm."PhysicalRelease" as physical_release,
mm."InCinemas" as in_cinemas,
(SELECT COUNT(*) FROM "MovieFiles" mf WHERE mf."MovieId" = m."Id") as file_count
FROM "Movies" m
JOIN "MovieMetadata" mm ON m."MovieMetadataId" = mm."Id"
WHERE (SELECT COUNT(*) FROM "MovieFiles" mf WHERE mf."MovieId" = m."Id") > 0
ORDER BY mm."Title"
"""
if self.db_type == "sqlite":
# SQLite uses ? placeholders but this query has none
pass
try:
with self._get_connection() as conn:
if self.db_type == "postgresql":
cursor = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
else:
cursor = conn.cursor()
cursor.execute(query)
rows = cursor.fetchall()
if rows:
return [dict(row) if self.db_type == "sqlite" else row for row in rows]
return []
except Exception as e:
_log("ERROR", f"Database query error getting all movies: {e}")
return []
def get_earliest_import_date(self, movie_id: int) -> Tuple[Optional[str], str]:
"""
Get earliest import date from History table, accounting for upgrade scenarios
clients/sonarr_client.py
+7 -15
View File
@@ -150,10 +150,6 @@ class SonarrClient:
_log("WARNING", f"No series found with IMDb ID via direct lookup: {imdb_id}")
return None
def get_series_by_id(self, series_id: int) -> Optional[Dict[str, Any]]:
"""Get series information by Sonarr series ID"""
return self._get(f"/series/{series_id}")
def episodes_for_series(self, series_id: int) -> List[Dict[str, Any]]:
"""Get all episodes for a series"""
return self._get("/episode", {"seriesId": series_id}) or []
@@ -226,7 +222,7 @@ class SonarrClient:
import_date = earliest_import["date"]
_log("INFO", f"Found import date: {import_date} for episode {episode_id}")
# Check chronological order of events
# Check if this looks like an upgrade by comparing to renames
if rename_events:
earliest_rename = min(rename_events, key=lambda x: x["date"])
rename_date = earliest_rename["date"]
@@ -234,16 +230,12 @@ class SonarrClient:
try:
import_dt = datetime.fromisoformat(import_date.replace("Z", "+00:00"))
rename_dt = datetime.fromisoformat(rename_date.replace("Z", "+00:00"))
days_diff = (import_dt - rename_dt).days
# If import happened BEFORE rename, it's valid original import
if import_dt <= rename_dt:
_log("INFO", f"Import {import_date} happened before/during rename {rename_date} - using import date")
return import_date
# If rename happened BEFORE import - always use aired date fallback
else:
_log("WARNING", f"Rename {rename_date} happened before import {import_date} - using aired date fallback")
return None # Trigger aired date fallback
# If import is significantly after rename, prefer rename date
if days_diff > 30:
_log("WARNING", f"Import {import_date} is {days_diff} days after rename {rename_date} - using rename date")
return rename_date
except Exception as e:
_log("DEBUG", f"Error comparing dates: {e}")
@@ -256,7 +248,7 @@ class SonarrClient:
_log("WARNING", f"No import events, using grab date: {earliest_grab['date']} for episode {episode_id}")
return earliest_grab["date"]
_log("WARNING", f"No reliable import events found for episode {episode_id} - should use air date instead")
_log("WARNING", f"No reliable import events found for episode {episode_id}")
return None
clients/sonarr_db_client.py
-582
View File
@@ -1,582 +0,0 @@
#!/usr/bin/env python3
"""
Direct Sonarr Database Client for NFOGuard
Provides high-performance access to Sonarr's SQLite/PostgreSQL database
"""
import os
import sqlite3
import psycopg2
import psycopg2.extras
from datetime import datetime, timezone
from pathlib import Path
from typing import Dict, Any, List, Optional, Tuple, Union
from urllib.parse import urlparse
from core.logging import _log
class SonarrDbClient:
"""Direct database client for Sonarr's SQLite or PostgreSQL database"""
def __init__(self,
db_type: str = "sqlite",
db_path: Optional[str] = None,
db_host: Optional[str] = None,
db_port: Optional[int] = None,
db_name: Optional[str] = None,
db_user: Optional[str] = None,
db_password: Optional[str] = None):
"""
Initialize Sonarr database client
Args:
db_type: "sqlite" or "postgresql"
db_path: Path to SQLite database file
db_host: PostgreSQL host
db_port: PostgreSQL port
db_name: PostgreSQL database name
db_user: PostgreSQL username
db_password: PostgreSQL password
"""
self.db_type = db_type.lower()
self.db_path = db_path
self.db_host = db_host
self.db_port = db_port or 5432
self.db_name = db_name
self.db_user = db_user
self.db_password = db_password
self._test_connection()
@classmethod
def from_env(cls) -> Optional['SonarrDbClient']:
"""Create client from environment variables"""
db_type = os.environ.get("SONARR_DB_TYPE", "").lower()
if not db_type:
return None
if db_type == "sqlite":
db_path = os.environ.get("SONARR_DB_PATH")
if not db_path or not Path(db_path).exists():
_log("WARNING", f"SONARR_DB_PATH not found or invalid: {db_path}")
return None
return cls(db_type="sqlite", db_path=db_path)
elif db_type == "postgresql":
# Support both individual vars and connection string
db_url = os.environ.get("SONARR_DB_URL")
if db_url:
parsed = urlparse(db_url)
return cls(
db_type="postgresql",
db_host=parsed.hostname,
db_port=parsed.port or 5432,
db_name=parsed.path.lstrip('/'),
db_user=parsed.username,
db_password=parsed.password
)
else:
return cls(
db_type="postgresql",
db_host=os.environ.get("SONARR_DB_HOST"),
db_port=int(os.environ.get("SONARR_DB_PORT", "5432")),
db_name=os.environ.get("SONARR_DB_NAME"),
db_user=os.environ.get("SONARR_DB_USER"),
db_password=os.environ.get("SONARR_DB_PASSWORD")
)
else:
_log("ERROR", f"Unsupported database type: {db_type}")
return None
def _test_connection(self) -> None:
"""Test database connection on initialization"""
try:
conn = self._get_connection()
if conn:
conn.close()
_log("INFO", f"Connected to Sonarr {self.db_type} database successfully")
else:
raise Exception("Failed to create connection")
except Exception as e:
_log("ERROR", f"Failed to connect to Sonarr database: {e}")
raise
def _get_connection(self) -> Union[sqlite3.Connection, psycopg2.extensions.connection]:
"""Get database connection"""
if self.db_type == "sqlite":
conn = sqlite3.connect(self.db_path)
conn.row_factory = sqlite3.Row
return conn
elif self.db_type == "postgresql":
conn = psycopg2.connect(
host=self.db_host,
port=self.db_port,
database=self.db_name,
user=self.db_user,
password=self.db_password
)
return conn
else:
raise ValueError(f"Unsupported database type: {self.db_type}")
def get_series_by_imdb(self, imdb_id: str) -> Optional[Dict[str, Any]]:
"""
Find series by IMDb ID using database query
Returns:
Dictionary with series info including id, imdb_id, title, path
"""
imdb_id = imdb_id if imdb_id.startswith("tt") else f"tt{imdb_id}"
query = """
SELECT
"Id" as id,
"ImdbId" as imdb_id,
"TvdbId" as tvdb_id,
"Title" as title,
"Path" as path,
"Added" as added
FROM "Series"
WHERE "ImdbId" = %s
"""
if self.db_type == "sqlite":
query = query.replace("%s", "?")
try:
with self._get_connection() as conn:
if self.db_type == "postgresql":
cursor = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
else:
cursor = conn.cursor()
cursor.execute(query, (imdb_id,))
row = cursor.fetchone()
if row:
return dict(row) if self.db_type == "sqlite" else row
except Exception as e:
_log("ERROR", f"Database query error for IMDb {imdb_id}: {e}")
return None
def get_all_episodes_for_series(self, series_id: int) -> List[Dict[str, Any]]:
"""
Get all episodes for a series
Args:
series_id: Sonarr series ID
Returns:
List of episode dictionaries with season, episode, air_date
"""
query = """
SELECT
"Id" as id,
"SeasonNumber" as season,
"EpisodeNumber" as episode,
"Title" as title,
"AirDate" as air_date,
"EpisodeFileId" as episode_file_id
FROM "Episodes"
WHERE "SeriesId" = %s
ORDER BY "SeasonNumber", "EpisodeNumber"
"""
if self.db_type == "sqlite":
query = query.replace("%s", "?")
try:
with self._get_connection() as conn:
if self.db_type == "postgresql":
cursor = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
else:
cursor = conn.cursor()
cursor.execute(query, (series_id,))
rows = cursor.fetchall()
if self.db_type == "sqlite":
return [dict(row) for row in rows]
else:
return rows
except Exception as e:
_log("ERROR", f"Database query error for series {series_id}: {e}")
return []
def get_episode_import_date(self, episode_id: int) -> Tuple[Optional[str], str]:
"""
Get earliest import date for an episode from History table
Args:
episode_id: Sonarr episode ID
Returns:
(date_iso, source_description)
"""
# Query for earliest import event (EventType 3)
import_query = """
SELECT
"Date" as event_date,
"EventType" as event_type
FROM "History"
WHERE "EpisodeId" = %s
AND "EventType" = 3
ORDER BY "Date" ASC
LIMIT 1
"""
if self.db_type == "sqlite":
import_query = import_query.replace("%s", "?")
try:
with self._get_connection() as conn:
if self.db_type == "postgresql":
cursor = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
else:
cursor = conn.cursor()
# Try import events first
cursor.execute(import_query, (episode_id,))
row = cursor.fetchone()
if row:
event_date = row['event_date'] if self.db_type == "postgresql" else row[0]
if isinstance(event_date, str):
dt = datetime.fromisoformat(event_date.replace("Z", "+00:00"))
else:
dt = event_date.replace(tzinfo=timezone.utc)
date_iso = dt.astimezone(timezone.utc).isoformat(timespec="seconds")
return date_iso, "sonarr:db.history.import"
except Exception as e:
_log("ERROR", f"Database query error for episode {episode_id}: {e}")
return None, "sonarr:db.no_import_found"
def get_episode_file_date(self, series_id: int, season: int, episode: int) -> Optional[str]:
"""
Get episode file DateAdded as fallback
Args:
series_id: Sonarr series ID
season: Season number
episode: Episode number
Returns:
ISO date string or None
"""
# First get the episode ID to find the file
episode_query = """
SELECT "EpisodeFileId"
FROM "Episodes"
WHERE "SeriesId" = %s
AND "SeasonNumber" = %s
AND "EpisodeNumber" = %s
"""
file_query = """
SELECT "DateAdded"
FROM "EpisodeFiles"
WHERE "Id" = %s
"""
if self.db_type == "sqlite":
episode_query = episode_query.replace("%s", "?")
file_query = file_query.replace("%s", "?")
try:
with self._get_connection() as conn:
if self.db_type == "postgresql":
cursor = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
else:
cursor = conn.cursor()
# Get episode file ID
cursor.execute(episode_query, (series_id, season, episode))
row = cursor.fetchone()
if row:
file_id = row['EpisodeFileId'] if self.db_type == "postgresql" else row[0]
if file_id:
# Get file date
cursor.execute(file_query, (file_id,))
file_row = cursor.fetchone()
if file_row:
date_value = file_row['DateAdded'] if self.db_type == "postgresql" else file_row[0]
if isinstance(date_value, str):
dt = datetime.fromisoformat(date_value.replace("Z", "+00:00"))
else:
dt = date_value.replace(tzinfo=timezone.utc)
return dt.astimezone(timezone.utc).isoformat(timespec="seconds")
except Exception as e:
_log("ERROR", f"Database query error for episode file: {e}")
return None
def bulk_import_dates_for_series(self, series_id: int) -> Dict[Tuple[int, int], Tuple[Optional[str], str]]:
"""
Get import dates for all episodes in a series in a single query
Args:
series_id: Sonarr series ID
Returns:
Dictionary mapping (season, episode) -> (date_iso, source)
"""
query = """
SELECT
e."SeasonNumber" as season,
e."EpisodeNumber" as episode,
e."Id" as episode_id,
MIN(h."Date") as earliest_import
FROM "Episodes" e
LEFT JOIN "History" h ON e."Id" = h."EpisodeId" AND h."EventType" = 3
WHERE e."SeriesId" = %s
GROUP BY e."SeasonNumber", e."EpisodeNumber", e."Id"
ORDER BY e."SeasonNumber", e."EpisodeNumber"
"""
if self.db_type == "sqlite":
query = query.replace("%s", "?")
results = {}
try:
with self._get_connection() as conn:
if self.db_type == "postgresql":
cursor = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
else:
cursor = conn.cursor()
cursor.execute(query, (series_id,))
rows = cursor.fetchall()
for row in rows:
if self.db_type == "postgresql":
season, episode, episode_id, earliest_import = row['season'], row['episode'], row['episode_id'], row['earliest_import']
else:
season, episode, episode_id, earliest_import = row[0], row[1], row[2], row[3]
if earliest_import:
if isinstance(earliest_import, str):
dt = datetime.fromisoformat(earliest_import.replace("Z", "+00:00"))
else:
dt = earliest_import.replace(tzinfo=timezone.utc)
date_iso = dt.astimezone(timezone.utc).isoformat(timespec="seconds")
results[(season, episode)] = (date_iso, "sonarr:db.bulk.import")
else:
results[(season, episode)] = (None, "sonarr:db.bulk.no_import")
except Exception as e:
_log("ERROR", f"Bulk query error for series {series_id}: {e}")
return results
def get_database_stats(self) -> Dict[str, Any]:
"""Get basic statistics about the Sonarr database"""
stats = {}
queries = {
"total_series": 'SELECT COUNT(*) FROM "Series"',
"total_episodes": 'SELECT COUNT(*) FROM "Episodes"',
"total_episode_files": 'SELECT COUNT(*) FROM "EpisodeFiles"',
"total_history_events": 'SELECT COUNT(*) FROM "History"',
"import_events": 'SELECT COUNT(*) FROM "History" WHERE "EventType" = 3',
"grab_events": 'SELECT COUNT(*) FROM "History" WHERE "EventType" = 1'
}
try:
with self._get_connection() as conn:
cursor = conn.cursor()
for stat_name, query in queries.items():
cursor.execute(query)
result = cursor.fetchone()
stats[stat_name] = result[0] if result else 0
except Exception as e:
_log("ERROR", f"Stats query error: {e}")
stats["error"] = str(e)
return stats
def health_check(self) -> Dict[str, Any]:
"""
Comprehensive health check for the Sonarr database connection
Returns:
Dictionary with health status, connection info, and basic functionality tests
"""
health = {
"status": "healthy",
"database_type": self.db_type,
"connection": "ok",
"readable": False,
"tables_exist": False,
"sample_data": False,
"issues": [],
"tested_at": datetime.now(timezone.utc).isoformat(timespec="seconds")
}
try:
with self._get_connection() as conn:
cursor = conn.cursor()
# Test 1: Basic read
try:
cursor.execute('SELECT 1')
result = cursor.fetchone()
if result and result[0] == 1:
health["readable"] = True
health["connection"] = "readable"
else:
health["issues"].append("Basic SELECT query failed")
except Exception as e:
health["issues"].append(f"Read test failed: {e}")
health["status"] = "degraded"
# Test 2: Check required tables
required_tables = ["Series", "Episodes", "EpisodeFiles", "History"]
existing_tables = []
try:
if self.db_type == "postgresql":
cursor.execute("""
SELECT table_name
FROM information_schema.tables
WHERE table_schema = 'public'
AND table_name IN ('Series', 'Episodes', 'EpisodeFiles', 'History')
""")
else: # SQLite
cursor.execute("""
SELECT name
FROM sqlite_master
WHERE type='table'
AND name IN ('Series', 'Episodes', 'EpisodeFiles', 'History')
""")
rows = cursor.fetchall()
existing_tables = [row[0] for row in rows]
if len(existing_tables) == len(required_tables):
health["tables_exist"] = True
else:
missing = set(required_tables) - set(existing_tables)
health["issues"].append(f"Missing tables: {list(missing)}")
health["status"] = "degraded"
health["existing_tables"] = existing_tables
except Exception as e:
health["issues"].append(f"Table check failed: {e}")
health["status"] = "degraded"
# Test 3: Check for sample data
if health["tables_exist"]:
try:
cursor.execute('SELECT COUNT(*) FROM "Series"')
series_count = cursor.fetchone()[0]
cursor.execute('SELECT COUNT(*) FROM "Episodes"')
episode_count = cursor.fetchone()[0]
cursor.execute('SELECT COUNT(*) FROM "History"')
history_count = cursor.fetchone()[0]
if series_count > 0 and episode_count > 0:
health["sample_data"] = True
health["series_count"] = series_count
health["episode_count"] = episode_count
health["history_count"] = history_count
else:
health["issues"].append(f"Low data counts - Series: {series_count}, Episodes: {episode_count}")
except Exception as e:
health["issues"].append(f"Sample data check failed: {e}")
# Test 4: Test a real query
if health["sample_data"]:
try:
cursor.execute("""
SELECT COUNT(*)
FROM "Series"
WHERE "ImdbId" IS NOT NULL
""")
imdb_series = cursor.fetchone()[0]
health["series_with_imdb"] = imdb_series
if imdb_series > 0:
health["functional"] = True
else:
health["issues"].append("No series with IMDb IDs found")
except Exception as e:
health["issues"].append(f"Functional test failed: {e}")
health["status"] = "degraded"
except Exception as e:
health["status"] = "error"
health["connection"] = "failed"
health["issues"].append(f"Connection failed: {e}")
_log("ERROR", f"Database health check failed: {e}")
# Overall status
if health["issues"]:
if health["status"] == "healthy":
health["status"] = "degraded"
# Add connection details (safe info only)
health["connection_info"] = {
"type": self.db_type,
"host": self.db_host if self.db_type == "postgresql" else None,
"port": self.db_port if self.db_type == "postgresql" else None,
"database": self.db_name if self.db_type == "postgresql" else None,
"path": self.db_path if self.db_type == "sqlite" else None
}
return health
if __name__ == "__main__":
# Test the database client
print("Testing SonarrDbClient...")
# Test with environment variables
client = SonarrDbClient.from_env()
if client:
print("✅ Connected to Sonarr database")
# Test stats
stats = client.get_database_stats()
print(f"Database stats: {stats}")
# Test series lookup
test_series = client.get_series_by_imdb("tt1628033") # Top Gear from your data
if test_series:
print(f"Found test series: {test_series}")
# Test episodes
series_id = test_series['id']
episodes = client.get_all_episodes_for_series(series_id)
print(f"Found {len(episodes)} episodes")
# Test bulk import dates
if episodes:
import_dates = client.bulk_import_dates_for_series(series_id)
print(f"Import dates for {len(import_dates)} episodes")
else:
print("Test series not found")
else:
print("❌ Could not connect to database - check environment variables")
config/runtime_validator.py
-547
View File
@@ -1,547 +0,0 @@
"""
Runtime Configuration Validator for NFOGuard
Provides validation of configuration at runtime with health checks
"""
import asyncio
import time
from pathlib import Path
from typing import Dict, List, Any, Optional, Tuple
from dataclasses import dataclass, field
from contextlib import asynccontextmanager
import aiohttp
import sqlite3
import psycopg2
from config.validator import ValidationIssue, ValidationResult, ValidationSeverity
from utils.exceptions import ConfigurationError, NetworkRetryableError, DatabaseError
@dataclass
class HealthCheckResult:
"""Result of a runtime health check"""
component: str
is_healthy: bool
response_time_ms: Optional[float] = None
message: str = ""
details: Dict[str, Any] = field(default_factory=dict)
def to_dict(self) -> Dict[str, Any]:
return {
"component": self.component,
"is_healthy": self.is_healthy,
"response_time_ms": self.response_time_ms,
"message": self.message,
"details": self.details,
"timestamp": time.time()
}
class RuntimeValidator:
"""Runtime configuration and system health validator"""
def __init__(self, config):
self.config = config
self._health_cache = {}
self._cache_ttl = 60 # Cache health results for 60 seconds
async def validate_runtime_config(self) -> ValidationResult:
"""Perform runtime validation of configuration"""
result = ValidationResult(is_valid=True)
# Validate filesystem access
await self._validate_filesystem_access(result)
# Validate database connectivity
await self._validate_database_connectivity(result)
# Validate external API connectivity
await self._validate_api_connectivity(result)
# Validate permissions
await self._validate_permissions(result)
# Validate resource availability
await self._validate_resources(result)
return result
async def _validate_filesystem_access(self, result: ValidationResult) -> None:
"""Validate filesystem access for media paths"""
all_paths = []
# Collect all media paths
all_paths.extend(self.config.tv_paths)
all_paths.extend(self.config.movie_paths)
for path in all_paths:
try:
# Check if path exists and is accessible
if not path.exists():
result.add_issue(ValidationIssue(
setting="media_paths",
severity=ValidationSeverity.ERROR,
message=f"Media path does not exist: {path}",
current_value=str(path)
))
continue
# Check if path is readable
test_file = None
try:
# Try to list directory contents
list(path.iterdir())
except PermissionError:
result.add_issue(ValidationIssue(
setting="media_paths",
severity=ValidationSeverity.ERROR,
message=f"No read permission for media path: {path}",
current_value=str(path)
))
continue
except OSError as e:
result.add_issue(ValidationIssue(
setting="media_paths",
severity=ValidationSeverity.WARNING,
message=f"Error accessing media path: {e}",
current_value=str(path)
))
continue
# Check write permissions (needed for NFO files)
try:
test_file = path / ".nfoguard_write_test"
test_file.write_text("test")
test_file.unlink()
except PermissionError:
result.add_issue(ValidationIssue(
setting="media_paths",
severity=ValidationSeverity.ERROR,
message=f"No write permission for media path: {path}",
current_value=str(path),
details={"required_for": "NFO file creation"}
))
except OSError as e:
result.add_issue(ValidationIssue(
setting="media_paths",
severity=ValidationSeverity.WARNING,
message=f"Write test failed for media path: {e}",
current_value=str(path)
))
finally:
# Cleanup test file if it exists
if test_file and test_file.exists():
try:
test_file.unlink()
except:
pass
except Exception as e:
result.add_issue(ValidationIssue(
setting="media_paths",
severity=ValidationSeverity.ERROR,
message=f"Unexpected error accessing path: {e}",
current_value=str(path)
))
# Validate database directory
db_path = Path(self.config.db_path)
db_dir = db_path.parent
if not db_dir.exists():
try:
db_dir.mkdir(parents=True, exist_ok=True)
except PermissionError:
result.add_issue(ValidationIssue(
setting="DB_PATH",
severity=ValidationSeverity.ERROR,
message=f"Cannot create database directory: {db_dir}",
current_value=str(db_path)
))
except OSError as e:
result.add_issue(ValidationIssue(
setting="DB_PATH",
severity=ValidationSeverity.ERROR,
message=f"Error creating database directory: {e}",
current_value=str(db_path)
))
# Test database file access
if not db_path.exists():
try:
# Try to create the database
with sqlite3.connect(str(db_path)) as conn:
conn.execute("SELECT 1")
except PermissionError:
result.add_issue(ValidationIssue(
setting="DB_PATH",
severity=ValidationSeverity.ERROR,
message=f"Cannot create database file: {db_path}",
current_value=str(db_path)
))
except sqlite3.Error as e:
result.add_issue(ValidationIssue(
setting="DB_PATH",
severity=ValidationSeverity.ERROR,
message=f"Database error: {e}",
current_value=str(db_path)
))
async def _validate_database_connectivity(self, result: ValidationResult) -> None:
"""Validate database connectivity"""
# Test Radarr database if configured
if hasattr(self.config, 'db_type') or 'RADARR_DB_TYPE' in os.environ:
import os
db_type = getattr(self.config, 'db_type', os.environ.get('RADARR_DB_TYPE', '')).lower()
if db_type == 'postgresql':
await self._test_postgresql_connection(result)
elif db_type == 'sqlite':
await self._test_sqlite_connection(result)
# Test local SQLite database
await self._test_local_database(result)
async def _test_postgresql_connection(self, result: ValidationResult) -> None:
"""Test PostgreSQL connection"""
import os
try:
host = os.environ.get('RADARR_DB_HOST')
port = int(os.environ.get('RADARR_DB_PORT', 5432))
database = os.environ.get('RADARR_DB_NAME')
user = os.environ.get('RADARR_DB_USER')
password = os.environ.get('RADARR_DB_PASSWORD', '')
start_time = time.time()
# Use asyncio to run blocking DB call
def test_connection():
conn = psycopg2.connect(
host=host, port=port, database=database,
user=user, password=password,
connect_timeout=10
)
with conn:
with conn.cursor() as cur:
cur.execute("SELECT 1")
return conn
conn = await asyncio.get_event_loop().run_in_executor(None, test_connection)
conn.close()
response_time = (time.time() - start_time) * 1000
if response_time > 5000: # 5 seconds
result.add_issue(ValidationIssue(
setting="RADARR_DB",
severity=ValidationSeverity.WARNING,
message=f"Slow database connection ({response_time:.0f}ms)",
details={"response_time_ms": response_time}
))
except psycopg2.OperationalError as e:
result.add_issue(ValidationIssue(
setting="RADARR_DB",
severity=ValidationSeverity.ERROR,
message=f"Cannot connect to PostgreSQL database: {e}",
details={"error_type": "connection_failed"}
))
except Exception as e:
result.add_issue(ValidationIssue(
setting="RADARR_DB",
severity=ValidationSeverity.ERROR,
message=f"Database connection error: {e}",
details={"error_type": "unexpected_error"}
))
async def _test_local_database(self, result: ValidationResult) -> None:
"""Test local SQLite database"""
try:
db_path = Path(self.config.db_path)
start_time = time.time()
def test_db():
with sqlite3.connect(str(db_path), timeout=10) as conn:
conn.execute("SELECT 1")
return True
await asyncio.get_event_loop().run_in_executor(None, test_db)
response_time = (time.time() - start_time) * 1000
if response_time > 1000: # 1 second is slow for SQLite
result.add_issue(ValidationIssue(
setting="DB_PATH",
severity=ValidationSeverity.WARNING,
message=f"Slow local database access ({response_time:.0f}ms)",
current_value=str(db_path),
details={"response_time_ms": response_time}
))
except sqlite3.OperationalError as e:
result.add_issue(ValidationIssue(
setting="DB_PATH",
severity=ValidationSeverity.ERROR,
message=f"Local database error: {e}",
current_value=str(self.config.db_path)
))
async def _validate_api_connectivity(self, result: ValidationResult) -> None:
"""Validate external API connectivity"""
apis = []
if hasattr(self.config, 'radarr_url') and self.config.radarr_url:
apis.append(("Radarr", self.config.radarr_url))
if hasattr(self.config, 'sonarr_url') and self.config.sonarr_url:
apis.append(("Sonarr", self.config.sonarr_url))
for api_name, base_url in apis:
await self._test_api_connectivity(result, api_name, base_url)
async def _test_api_connectivity(self, result: ValidationResult, api_name: str, base_url: str) -> None:
"""Test connectivity to a specific API"""
try:
timeout = aiohttp.ClientTimeout(total=10)
async with aiohttp.ClientSession(timeout=timeout) as session:
start_time = time.time()
# Test basic connectivity
test_url = f"{base_url.rstrip('/')}/api/v1/health"
async with session.get(test_url) as response:
response_time = (time.time() - start_time) * 1000
if response.status == 200:
if response_time > 5000: # 5 seconds
result.add_issue(ValidationIssue(
setting=f"{api_name.upper()}_URL",
severity=ValidationSeverity.WARNING,
message=f"Slow {api_name} API response ({response_time:.0f}ms)",
current_value=base_url,
details={"response_time_ms": response_time}
))
else:
result.add_issue(ValidationIssue(
setting=f"{api_name.upper()}_URL",
severity=ValidationSeverity.WARNING,
message=f"{api_name} API returned HTTP {response.status}",
current_value=base_url,
details={"status_code": response.status}
))
except asyncio.TimeoutError:
result.add_issue(ValidationIssue(
setting=f"{api_name.upper()}_URL",
severity=ValidationSeverity.WARNING,
message=f"{api_name} API connection timeout",
current_value=base_url,
details={"error_type": "timeout"}
))
except Exception as e:
result.add_issue(ValidationIssue(
setting=f"{api_name.upper()}_URL",
severity=ValidationSeverity.WARNING,
message=f"{api_name} API connection error: {e}",
current_value=base_url,
details={"error_type": "connection_error"}
))
async def _validate_permissions(self, result: ValidationResult) -> None:
"""Validate file system permissions"""
# This is partially covered in filesystem validation
# Additional permission checks can be added here
pass
async def _validate_resources(self, result: ValidationResult) -> None:
"""Validate system resources"""
import shutil
try:
# Check disk space for database directory
db_path = Path(self.config.db_path)
db_dir = db_path.parent if not db_path.is_dir() else db_path
if db_dir.exists():
free_space = shutil.disk_usage(str(db_dir)).free
free_space_mb = free_space / (1024 * 1024)
if free_space_mb < 100: # Less than 100MB
result.add_issue(ValidationIssue(
setting="DB_PATH",
severity=ValidationSeverity.ERROR,
message=f"Low disk space in database directory ({free_space_mb:.1f}MB free)",
current_value=str(db_path),
details={"free_space_mb": free_space_mb}
))
elif free_space_mb < 1000: # Less than 1GB
result.add_issue(ValidationIssue(
setting="DB_PATH",
severity=ValidationSeverity.WARNING,
message=f"Low disk space in database directory ({free_space_mb:.1f}MB free)",
current_value=str(db_path),
details={"free_space_mb": free_space_mb}
))
except Exception as e:
result.add_issue(ValidationIssue(
setting="system_resources",
severity=ValidationSeverity.WARNING,
message=f"Could not check disk space: {e}",
details={"error_type": "resource_check_failed"}
))
async def get_system_health(self) -> Dict[str, HealthCheckResult]:
"""Get comprehensive system health status"""
health_checks = {}
# Database health
health_checks["database"] = await self._check_database_health()
# Filesystem health
health_checks["filesystem"] = await self._check_filesystem_health()
# API health
health_checks["external_apis"] = await self._check_api_health()
return health_checks
async def _check_database_health(self) -> HealthCheckResult:
"""Check database health"""
try:
start_time = time.time()
def test_db():
with sqlite3.connect(str(self.config.db_path), timeout=5) as conn:
conn.execute("SELECT COUNT(*) FROM sqlite_master")
return True
await asyncio.get_event_loop().run_in_executor(None, test_db)
response_time = (time.time() - start_time) * 1000
return HealthCheckResult(
component="database",
is_healthy=True,
response_time_ms=response_time,
message="Database accessible",
details={"db_path": str(self.config.db_path)}
)
except Exception as e:
return HealthCheckResult(
component="database",
is_healthy=False,
message=f"Database error: {e}",
details={"db_path": str(self.config.db_path), "error": str(e)}
)
async def _check_filesystem_health(self) -> HealthCheckResult:
"""Check filesystem health"""
try:
accessible_paths = 0
total_paths = len(self.config.tv_paths) + len(self.config.movie_paths)
for path in list(self.config.tv_paths) + list(self.config.movie_paths):
if path.exists() and path.is_dir():
try:
# Quick access test
next(path.iterdir(), None)
accessible_paths += 1
except:
pass
is_healthy = accessible_paths == total_paths
health_percentage = (accessible_paths / total_paths * 100) if total_paths > 0 else 0
return HealthCheckResult(
component="filesystem",
is_healthy=is_healthy,
message=f"{accessible_paths}/{total_paths} media paths accessible ({health_percentage:.1f}%)",
details={
"accessible_paths": accessible_paths,
"total_paths": total_paths,
"health_percentage": health_percentage
}
)
except Exception as e:
return HealthCheckResult(
component="filesystem",
is_healthy=False,
message=f"Filesystem check error: {e}",
details={"error": str(e)}
)
async def _check_api_health(self) -> HealthCheckResult:
"""Check external API health"""
apis_tested = 0
apis_healthy = 0
api_details = {}
try:
# Test configured APIs
if hasattr(self.config, 'radarr_url') and self.config.radarr_url:
apis_tested += 1
healthy = await self._test_single_api("Radarr", self.config.radarr_url)
if healthy:
apis_healthy += 1
api_details["radarr"] = {"healthy": healthy}
if hasattr(self.config, 'sonarr_url') and self.config.sonarr_url:
apis_tested += 1
healthy = await self._test_single_api("Sonarr", self.config.sonarr_url)
if healthy:
apis_healthy += 1
api_details["sonarr"] = {"healthy": healthy}
if apis_tested == 0:
return HealthCheckResult(
component="external_apis",
is_healthy=True,
message="No external APIs configured",
details={"apis_configured": 0}
)
is_healthy = apis_healthy == apis_tested
health_percentage = (apis_healthy / apis_tested * 100) if apis_tested > 0 else 0
return HealthCheckResult(
component="external_apis",
is_healthy=is_healthy,
message=f"{apis_healthy}/{apis_tested} APIs healthy ({health_percentage:.1f}%)",
details={
"healthy_apis": apis_healthy,
"total_apis": apis_tested,
"health_percentage": health_percentage,
"api_status": api_details
}
)
except Exception as e:
return HealthCheckResult(
component="external_apis",
is_healthy=False,
message=f"API health check error: {e}",
details={"error": str(e)}
)
async def _test_single_api(self, api_name: str, base_url: str) -> bool:
"""Test a single API for health"""
try:
timeout = aiohttp.ClientTimeout(total=5)
async with aiohttp.ClientSession(timeout=timeout) as session:
test_url = f"{base_url.rstrip('/')}/api/v1/health"
async with session.get(test_url) as response:
return response.status == 200
except:
return False
# Import needed for database validation
import os
config/settings.py
+23 -290
View File
@@ -1,17 +1,10 @@
"""
NFOGuard Configuration Module
Handles all configuration loading and validation with comprehensive error reporting
Handles all configuration loading and validation
"""
import os
import sys
import logging
from pathlib import Path
from typing import List, Optional, Dict, Any
from utils.exceptions import ConfigurationError
logger = logging.getLogger(__name__)
from typing import List
def _bool_env(name: str, default: bool) -> bool:
@@ -23,33 +16,20 @@ def _bool_env(name: str, default: bool) -> bool:
class NFOGuardConfig:
"""Configuration class for NFOGuard with integrated validation"""
"""Configuration class for NFOGuard"""
def __init__(self, validate_on_init: bool = True, strict_validation: bool = False):
"""
Initialize NFOGuard configuration
def __init__(self):
# Paths - No hardcoded defaults, must be configured via environment
tv_paths_env = os.environ.get("TV_PATHS", "")
movie_paths_env = os.environ.get("MOVIE_PATHS", "")
Args:
validate_on_init: Run validation during initialization
strict_validation: Treat warnings as errors
"""
self.strict_validation = strict_validation
self._validation_issues = []
# Initialize configuration
self._load_configuration()
# Run validation if requested
if validate_on_init:
self._validate_configuration()
def _load_configuration(self) -> None:
"""Load all configuration from environment variables"""
# Server configuration
self._load_server_settings()
# Core paths - Required
self._load_paths()
if not tv_paths_env:
raise ValueError("TV_PATHS environment variable is required but not set")
if not movie_paths_env:
raise ValueError("MOVIE_PATHS environment variable is required but not set")
self.tv_paths = [Path(p.strip()) for p in tv_paths_env.split(",") if p.strip()]
self.movie_paths = [Path(p.strip()) for p in movie_paths_env.split(",") if p.strip()]
# Core settings
self.manage_nfo = _bool_env("MANAGE_NFO", True)
@@ -58,275 +38,28 @@ class NFOGuardConfig:
self.debug = _bool_env("DEBUG", False)
self.manager_brand = os.environ.get("MANAGER_BRAND", "NFOGuard")
# Batching and performance
self.batch_delay = self._get_float_env("BATCH_DELAY", 5.0, 0.1, 300.0)
self.max_concurrent = self._get_int_env("MAX_CONCURRENT_SERIES", 3, 1, 10)
self.sequential_delay = self._get_float_env("SEQUENTIAL_DELAY", 20.0, 0.0, 60.0) # Delay between sequential episodes (default 20s)
# Batching
self.batch_delay = float(os.environ.get("BATCH_DELAY", "5.0"))
self.max_concurrent = int(os.environ.get("MAX_CONCURRENT_SERIES", "3"))
# Database
self.db_type = os.environ.get("DB_TYPE", "sqlite").lower()
self.db_path = Path(os.environ.get("DB_PATH", "/app/data/media_dates.db"))
# PostgreSQL database settings
if self.db_type == "postgresql":
self.db_host = os.environ.get("DB_HOST", "localhost")
self.db_port = self._get_int_env("DB_PORT", 5432, 1, 65535)
self.db_name = os.environ.get("DB_NAME", "nfoguard")
self.db_user = os.environ.get("DB_USER", "nfoguard")
self.db_password = os.environ.get("DB_PASSWORD", "")
# External connections
self._load_external_connections()
# Movie processing
self._load_movie_settings()
# TV processing
self._load_tv_settings()
# Web interface authentication
self._load_auth_settings()
def _load_paths(self) -> None:
"""Load and validate path configuration"""
tv_paths_env = os.environ.get("TV_PATHS", "")
movie_paths_env = os.environ.get("MOVIE_PATHS", "")
if not tv_paths_env:
raise ConfigurationError(
setting="TV_PATHS",
reason="TV_PATHS environment variable is required but not set"
)
if not movie_paths_env:
raise ConfigurationError(
setting="MOVIE_PATHS",
reason="MOVIE_PATHS environment variable is required but not set"
)
# Parse paths
self.tv_paths = [Path(p.strip()) for p in tv_paths_env.split(",") if p.strip()]
self.movie_paths = [Path(p.strip()) for p in movie_paths_env.split(",") if p.strip()]
if not self.tv_paths:
raise ConfigurationError(
setting="TV_PATHS",
reason="No valid TV paths found after parsing",
current_value=tv_paths_env
)
if not self.movie_paths:
raise ConfigurationError(
setting="MOVIE_PATHS",
reason="No valid movie paths found after parsing",
current_value=movie_paths_env
)
def _load_server_settings(self) -> None:
"""Load server configuration"""
# Core API settings (webhooks, processing, database management)
self.core_api_host = os.environ.get("CORE_API_HOST", "0.0.0.0")
self.core_api_port = self._get_int_env("CORE_API_PORT", 8080, 1024, 65535)
# Web API settings (dashboard, web interface) - for reference/connection
self.web_api_host = os.environ.get("WEB_API_HOST", "0.0.0.0")
self.web_api_port = self._get_int_env("WEB_API_PORT", 8081, 1024, 65535)
def _load_external_connections(self) -> None:
"""Load external API and database connection settings"""
# API URLs
self.radarr_url = os.environ.get("RADARR_URL", "")
self.sonarr_url = os.environ.get("SONARR_URL", "")
self.jellyseerr_url = os.environ.get("JELLYSEERR_URL", "")
# Radarr database settings
self.radarr_db_type = os.environ.get("RADARR_DB_TYPE", "").lower()
self.radarr_db_host = os.environ.get("RADARR_DB_HOST", "")
self.radarr_db_port = self._get_int_env("RADARR_DB_PORT", 5432, 1, 65535)
self.radarr_db_name = os.environ.get("RADARR_DB_NAME", "")
self.radarr_db_user = os.environ.get("RADARR_DB_USER", "")
# Timeout settings
self.timeout_seconds = self._get_int_env("TIMEOUT_SECONDS", 45, 10, 300)
def _load_movie_settings(self) -> None:
"""Load movie processing settings"""
self.movie_priority = os.environ.get("MOVIE_PRIORITY", "import_then_digital").lower()
self.prefer_release_dates_over_file_dates = _bool_env("PREFER_RELEASE_DATES_OVER_FILE_DATES", True)
self.allow_file_date_fallback = _bool_env("ALLOW_FILE_DATE_FALLBACK", False)
# Manual scan behavior
self.manual_scan_prioritize_nfo = _bool_env("MANUAL_SCAN_PRIORITIZE_NFO", False)
# Release date settings
release_priority_env = os.environ.get("RELEASE_DATE_PRIORITY", "digital,physical,theatrical")
self.release_date_priority = [p.strip() for p in release_priority_env.split(",") if p.strip()]
self.release_date_priority = [p.strip() for p in os.environ.get("RELEASE_DATE_PRIORITY", "digital,physical,theatrical").split(",")]
self.enable_smart_date_validation = _bool_env("ENABLE_SMART_DATE_VALIDATION", True)
self.max_release_date_gap_years = self._get_int_env("MAX_RELEASE_DATE_GAP_YEARS", 10, 1, 50)
self.max_release_date_gap_years = int(os.environ.get("MAX_RELEASE_DATE_GAP_YEARS", "10"))
self.movie_poll_mode = os.environ.get("MOVIE_POLL_MODE", "always").lower()
self.movie_update_mode = os.environ.get("MOVIE_DATE_UPDATE_MODE", "backfill_only").lower()
def _load_tv_settings(self) -> None:
"""Load TV processing settings"""
# TV processing
self.tv_season_dir_format = os.environ.get("TV_SEASON_DIR_FORMAT", "Season {season:02d}")
self.tv_season_dir_pattern = os.environ.get("TV_SEASON_DIR_PATTERN", "season ").lower()
self.tv_webhook_processing_mode = os.environ.get("TV_WEBHOOK_PROCESSING_MODE", "targeted").lower()
def get_season_dir_name(self, season: int) -> str:
"""Get the directory name for a specific season, handling Season 0 as 'Specials'"""
if season == 0:
return "Specials"
return self.tv_season_dir_format.format(season=season)
def _load_auth_settings(self) -> None:
"""Load web interface authentication settings"""
self.web_auth_enabled = _bool_env("WEB_AUTH_ENABLED", False)
self.web_auth_username = os.environ.get("WEB_AUTH_USERNAME", "admin")
self.web_auth_password = os.environ.get("WEB_AUTH_PASSWORD", "")
self.web_auth_session_timeout = self._get_int_env("WEB_AUTH_SESSION_TIMEOUT", 3600, 300, 86400) # 1 hour default, 5min-24h range
def _get_int_env(self, name: str, default: int, min_val: int, max_val: int) -> int:
"""Get integer environment variable with validation"""
value_str = os.environ.get(name)
if not value_str:
return default
try:
value = int(value_str)
if value < min_val or value > max_val:
raise ConfigurationError(
setting=name,
reason=f"Value must be between {min_val} and {max_val}",
current_value=value_str
)
return value
except ValueError:
raise ConfigurationError(
setting=name,
reason=f"Invalid integer value",
current_value=value_str
)
def _get_float_env(self, name: str, default: float, min_val: float, max_val: float) -> float:
"""Get float environment variable with validation"""
value_str = os.environ.get(name)
if not value_str:
return default
try:
value = float(value_str)
if value < min_val or value > max_val:
raise ConfigurationError(
setting=name,
reason=f"Value must be between {min_val} and {max_val}",
current_value=value_str
)
return value
except ValueError:
raise ConfigurationError(
setting=name,
reason=f"Invalid float value",
current_value=value_str
)
def _validate_configuration(self) -> None:
"""Validate configuration using the validator"""
try:
# Import here to avoid circular imports
from config.validator import validate_configuration_and_raise
validate_configuration_and_raise()
except ImportError:
# Fallback to basic validation if validator not available
logger.warning("Configuration validator not available, using basic validation")
self._basic_validation()
except ConfigurationError:
if self.strict_validation:
raise
else:
# Log warning but continue
logger.warning("Configuration validation found issues", exc_info=True)
def _basic_validation(self) -> None:
"""Basic fallback validation"""
# Validate that paths exist (basic check)
for path_list, path_type in [(self.tv_paths, "TV"), (self.movie_paths, "Movie")]:
for path in path_list:
if not path.is_absolute():
logger.warning(f"{path_type} path should be absolute: {path}")
def get_configuration_summary(self) -> Dict[str, Any]:
"""Get a summary of current configuration"""
return {
"tv_paths": [str(p) for p in self.tv_paths],
"movie_paths": [str(p) for p in self.movie_paths],
"database": {
"type": self.db_type,
"path": str(self.db_path) if self.db_type == "sqlite" else None,
"host": getattr(self, 'db_host', None) if self.db_type == "postgresql" else None,
"port": getattr(self, 'db_port', None) if self.db_type == "postgresql" else None,
"name": getattr(self, 'db_name', None) if self.db_type == "postgresql" else None
},
"external_apis": {
"radarr": bool(self.radarr_url),
"sonarr": bool(self.sonarr_url),
"jellyseerr": bool(self.jellyseerr_url)
},
"radarr_database": {
"type": getattr(self, 'radarr_db_type', None),
"configured": bool(getattr(self, 'radarr_db_type', None) and getattr(self, 'radarr_db_host', None))
},
"performance": {
"batch_delay": self.batch_delay,
"max_concurrent": self.max_concurrent,
"timeout_seconds": self.timeout_seconds
},
"features": {
"manage_nfo": self.manage_nfo,
"fix_dir_mtimes": self.fix_dir_mtimes,
"lock_metadata": self.lock_metadata,
"debug": self.debug,
"manual_scan_prioritize_nfo": self.manual_scan_prioritize_nfo
}
}
def validate_runtime_access(self) -> Dict[str, bool]:
"""Quick runtime validation of critical paths"""
results = {
"tv_paths_accessible": True,
"movie_paths_accessible": True,
"database_writable": True
}
# Test TV paths
for path in self.tv_paths:
if not path.exists() or not path.is_dir():
results["tv_paths_accessible"] = False
break
# Test movie paths
for path in self.movie_paths:
if not path.exists() or not path.is_dir():
results["movie_paths_accessible"] = False
break
# Test database directory
db_dir = self.db_path.parent
try:
if not db_dir.exists():
db_dir.mkdir(parents=True, exist_ok=True)
# Test write access
test_file = db_dir / ".nfoguard_write_test"
test_file.write_text("test")
test_file.unlink()
except (PermissionError, OSError):
results["database_writable"] = False
return results
# Global config instance - Initialize with validation disabled by default for backwards compatibility
# Applications can enable validation by creating their own instance with validate_on_init=True
config = NFOGuardConfig(validate_on_init=False)
# Global config instance
config = NFOGuardConfig()
config/validation_cli.py
-293
View File
@@ -1,293 +0,0 @@
#!/usr/bin/env python3
"""
Configuration Validation CLI for NFOGuard
Provides command-line validation and reporting of configuration issues
"""
import sys
import json
import argparse
from pathlib import Path
from typing import Dict, Any, List
from datetime import datetime
from config.validator import validate_configuration, ValidationSeverity
from config.runtime_validator import RuntimeValidator
from config.settings import NFOGuardConfig
class ValidationReporter:
"""Formats and displays validation results"""
def __init__(self, verbose: bool = False, json_output: bool = False):
self.verbose = verbose
self.json_output = json_output
# Color codes for terminal output
self.colors = {
'error': '\033[91m', # Red
'warning': '\033[93m', # Yellow
'info': '\033[94m', # Blue
'success': '\033[92m', # Green
'reset': '\033[0m', # Reset
'bold': '\033[1m' # Bold
}
def report_validation_results(self, result, runtime_result=None) -> int:
"""
Report validation results
Returns:
Exit code (0 for success, 1 for warnings, 2 for errors)
"""
if self.json_output:
return self._report_json(result, runtime_result)
else:
return self._report_human_readable(result, runtime_result)
def _report_json(self, result, runtime_result=None) -> int:
"""Report results in JSON format"""
output = {
"timestamp": datetime.now().isoformat(),
"validation": result.to_dict()
}
if runtime_result:
output["runtime_validation"] = runtime_result.to_dict()
print(json.dumps(output, indent=2))
if result.errors_count > 0:
return 2
elif result.warnings_count > 0:
return 1
return 0
def _report_human_readable(self, result, runtime_result=None) -> int:
"""Report results in human-readable format"""
print(f"{self.colors['bold']}NFOGuard Configuration Validation Report{self.colors['reset']}")
print("=" * 50)
# Overall status
if result.is_valid:
status_color = self.colors['success']
status_text = "✓ VALID"
else:
status_color = self.colors['error']
status_text = "✗ INVALID"
print(f"Status: {status_color}{status_text}{self.colors['reset']}")
print(f"Errors: {result.errors_count}")
print(f"Warnings: {result.warnings_count}")
print(f"Total Issues: {len(result.issues)}")
print()
# Report issues by severity
if result.issues:
self._report_issues_by_severity(result.issues)
# Report runtime validation if available
if runtime_result:
print(f"\n{self.colors['bold']}Runtime Validation{self.colors['reset']}")
print("-" * 20)
if runtime_result.issues:
self._report_issues_by_severity(runtime_result.issues, "Runtime")
else:
print(f"{self.colors['success']}✓ All runtime checks passed{self.colors['reset']}")
# Summary and recommendations
self._report_summary_and_recommendations(result)
# Return appropriate exit code
if result.errors_count > 0 or (runtime_result and runtime_result.errors_count > 0):
return 2
elif result.warnings_count > 0 or (runtime_result and runtime_result.warnings_count > 0):
return 1
return 0
def _report_issues_by_severity(self, issues: List, context: str = "Configuration") -> None:
"""Report issues grouped by severity"""
errors = [issue for issue in issues if issue.severity == ValidationSeverity.ERROR]
warnings = [issue for issue in issues if issue.severity == ValidationSeverity.WARNING]
info_issues = [issue for issue in issues if issue.severity == ValidationSeverity.INFO]
if errors:
print(f"{self.colors['error']}{self.colors['bold']}ERRORS ({len(errors)}):{self.colors['reset']}")
for issue in errors:
self._format_issue(issue)
print()
if warnings:
print(f"{self.colors['warning']}{self.colors['bold']}WARNINGS ({len(warnings)}):{self.colors['reset']}")
for issue in warnings:
self._format_issue(issue)
print()
if info_issues and self.verbose:
print(f"{self.colors['info']}{self.colors['bold']}INFO ({len(info_issues)}):{self.colors['reset']}")
for issue in info_issues:
self._format_issue(issue)
print()
def _format_issue(self, issue) -> None:
"""Format a single validation issue"""
severity_colors = {
ValidationSeverity.ERROR: self.colors['error'],
ValidationSeverity.WARNING: self.colors['warning'],
ValidationSeverity.INFO: self.colors['info']
}
color = severity_colors.get(issue.severity, '')
print(f" {color}• {issue.setting}:{self.colors['reset']} {issue.message}")
if issue.current_value is not None:
print(f" Current: {issue.current_value}")
if issue.suggested_value is not None:
print(f" Suggested: {issue.suggested_value}")
if self.verbose and issue.details:
print(f" Details: {issue.details}")
def _report_summary_and_recommendations(self, result) -> None:
"""Report summary and general recommendations"""
print(f"\n{self.colors['bold']}Summary{self.colors['reset']}")
print("-" * 10)
if result.is_valid:
print(f"{self.colors['success']}✓ Your configuration is valid and ready to use!{self.colors['reset']}")
else:
print(f"{self.colors['error']}✗ Your configuration has issues that need to be resolved.{self.colors['reset']}")
# Provide specific recommendations based on issue types
recommendations = self._generate_recommendations(result)
if recommendations:
print(f"\n{self.colors['bold']}Recommendations{self.colors['reset']}")
print("-" * 15)
for rec in recommendations:
print(f" {self.colors['info']}• {rec}{self.colors['reset']}")
def _generate_recommendations(self, result) -> List[str]:
"""Generate recommendations based on validation results"""
recommendations = []
# Check for common patterns
path_issues = [issue for issue in result.issues if 'path' in issue.setting.lower()]
if path_issues:
recommendations.append("Verify all file paths are correct for your environment")
recommendations.append("Ensure paths are absolute when using Docker")
db_issues = [issue for issue in result.issues if 'db' in issue.setting.lower()]
if db_issues:
recommendations.append("Check database connection settings and credentials")
url_issues = [issue for issue in result.issues if 'url' in issue.setting.lower()]
if url_issues:
recommendations.append("Verify API URLs are reachable and include the correct port")
perf_issues = [issue for issue in result.issues if any(perf in issue.setting.lower()
for perf in ['concurrent', 'delay', 'timeout'])]
if perf_issues:
recommendations.append("Consider adjusting performance settings based on your system resources")
# General recommendations
if result.errors_count > 0:
recommendations.append("Fix all ERROR-level issues before starting NFOGuard")
if result.warnings_count > 0:
recommendations.append("Review WARNING-level issues to optimize performance and reliability")
return recommendations
async def run_validation(args) -> int:
"""Run configuration validation"""
reporter = ValidationReporter(verbose=args.verbose, json_output=args.json)
try:
# Run static validation
print("Running configuration validation..." if not args.json else "", file=sys.stderr)
result = validate_configuration()
runtime_result = None
# Run runtime validation if requested
if args.runtime:
print("Running runtime validation..." if not args.json else "", file=sys.stderr)
try:
config = NFOGuardConfig()
runtime_validator = RuntimeValidator(config)
runtime_result = await runtime_validator.validate_runtime_config()
except Exception as e:
if not args.json:
print(f"Runtime validation failed: {e}", file=sys.stderr)
# Continue with static validation results
# Report results
return reporter.report_validation_results(result, runtime_result)
except Exception as e:
if args.json:
error_output = {
"timestamp": datetime.now().isoformat(),
"error": {
"message": str(e),
"type": type(e).__name__
}
}
print(json.dumps(error_output, indent=2))
else:
print(f"Validation failed: {e}", file=sys.stderr)
return 2
def main():
"""Main CLI entry point"""
parser = argparse.ArgumentParser(
description="Validate NFOGuard configuration",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
%(prog)s # Basic validation
%(prog)s --runtime # Include runtime checks
%(prog)s --verbose # Show detailed information
%(prog)s --json # Output JSON format
%(prog)s --runtime --json # Runtime validation with JSON output
"""
)
parser.add_argument(
"--runtime",
action="store_true",
help="Perform runtime validation (tests actual connectivity and permissions)"
)
parser.add_argument(
"--verbose", "-v",
action="store_true",
help="Show verbose output including info-level messages"
)
parser.add_argument(
"--json",
action="store_true",
help="Output results in JSON format"
)
args = parser.parse_args()
# Run validation
import asyncio
try:
exit_code = asyncio.run(run_validation(args))
sys.exit(exit_code)
except KeyboardInterrupt:
print("Validation interrupted", file=sys.stderr)
sys.exit(130)
except Exception as e:
print(f"Unexpected error: {e}", file=sys.stderr)
sys.exit(2)
if __name__ == "__main__":
main()
config/validator.py
-498
View File
@@ -1,498 +0,0 @@
"""
Configuration Validator for NFOGuard
Provides comprehensive validation of all configuration settings with detailed error reporting
"""
import os
import re
from pathlib import Path
from typing import Dict, List, Any, Optional, Union, Type, Callable
from dataclasses import dataclass, field
from enum import Enum
from utils.exceptions import ConfigurationError
from utils.validation import validate_url_format
class ValidationSeverity(Enum):
"""Severity levels for validation issues"""
ERROR = "error" # Configuration is invalid, will cause failures
WARNING = "warning" # Configuration may cause issues but is workable
INFO = "info" # Configuration could be improved
@dataclass
class ValidationIssue:
"""Represents a configuration validation issue"""
setting: str
severity: ValidationSeverity
message: str
current_value: Any = None
suggested_value: Any = None
details: Dict[str, Any] = field(default_factory=dict)
def to_dict(self) -> Dict[str, Any]:
"""Convert to dictionary for structured logging"""
return {
"setting": self.setting,
"severity": self.severity.value,
"message": self.message,
"current_value": str(self.current_value) if self.current_value is not None else None,
"suggested_value": str(self.suggested_value) if self.suggested_value is not None else None,
"details": self.details
}
@dataclass
class ValidationResult:
"""Results of configuration validation"""
is_valid: bool
issues: List[ValidationIssue] = field(default_factory=list)
warnings_count: int = 0
errors_count: int = 0
def add_issue(self, issue: ValidationIssue) -> None:
"""Add a validation issue"""
self.issues.append(issue)
if issue.severity == ValidationSeverity.ERROR:
self.errors_count += 1
self.is_valid = False
elif issue.severity == ValidationSeverity.WARNING:
self.warnings_count += 1
def get_errors(self) -> List[ValidationIssue]:
"""Get only error-level issues"""
return [issue for issue in self.issues if issue.severity == ValidationSeverity.ERROR]
def get_warnings(self) -> List[ValidationIssue]:
"""Get only warning-level issues"""
return [issue for issue in self.issues if issue.severity == ValidationSeverity.WARNING]
def to_dict(self) -> Dict[str, Any]:
"""Convert to dictionary for structured logging"""
return {
"is_valid": self.is_valid,
"errors_count": self.errors_count,
"warnings_count": self.warnings_count,
"issues": [issue.to_dict() for issue in self.issues]
}
class ConfigValidator:
"""Comprehensive configuration validator for NFOGuard"""
def __init__(self):
self.result = ValidationResult(is_valid=True)
# Define validation rules
self._path_settings = {
"TV_PATHS", "MOVIE_PATHS", "RADARR_ROOT_FOLDERS",
"SONARR_ROOT_FOLDERS", "DB_PATH", "LOG_DIR"
}
self._url_settings = {
"RADARR_URL", "SONARR_URL", "JELLYSEERR_URL"
}
self._required_settings = {
"TV_PATHS", "MOVIE_PATHS"
}
self._numeric_settings = {
"BATCH_DELAY": (float, 0.1, 300.0),
"MAX_CONCURRENT_SERIES": (int, 1, 10),
"TIMEOUT_SECONDS": (int, 10, 300),
"PORT": (int, 1024, 65535),
"RADARR_DB_PORT": (int, 1, 65535),
"MAX_RELEASE_DATE_GAP_YEARS": (int, 1, 50)
}
self._boolean_settings = {
"MANAGE_NFO", "FIX_DIR_MTIMES", "LOCK_METADATA", "DEBUG",
"PREFER_RELEASE_DATES_OVER_FILE_DATES", "ALLOW_FILE_DATE_FALLBACK",
"ENABLE_SMART_DATE_VALIDATION", "PATH_DEBUG", "SUPPRESS_TVDB_WARNINGS"
}
self._choice_settings = {
"MOVIE_PRIORITY": ["import_then_digital", "digital_first", "file_date_only"],
"MOVIE_POLL_MODE": ["always", "missing_only", "never"],
"MOVIE_DATE_UPDATE_MODE": ["overwrite", "backfill_only", "preserve_existing"],
"TV_WEBHOOK_PROCESSING_MODE": ["targeted", "full_scan", "hybrid"],
"UPDATE_MODE": ["always", "missing_only", "never"],
"MTIME_BEHAVIOR": ["update", "leave_alone"],
"RADARR_DB_TYPE": ["postgresql", "sqlite"]
}
def validate_all(self) -> ValidationResult:
"""Validate all configuration settings"""
self.result = ValidationResult(is_valid=True)
# Validate required settings
self._validate_required_settings()
# Validate paths
self._validate_paths()
# Validate URLs
self._validate_urls()
# Validate numeric settings
self._validate_numeric_settings()
# Validate boolean settings
self._validate_boolean_settings()
# Validate choice settings
self._validate_choice_settings()
# Validate database configuration
self._validate_database_config()
# Validate release date configuration
self._validate_release_date_config()
# Validate performance settings
self._validate_performance_settings()
# Validate cross-setting dependencies
self._validate_dependencies()
return self.result
def _validate_required_settings(self) -> None:
"""Validate required environment variables are set"""
for setting in self._required_settings:
value = os.environ.get(setting)
if not value:
self.result.add_issue(ValidationIssue(
setting=setting,
severity=ValidationSeverity.ERROR,
message=f"Required setting {setting} is not set",
current_value=value
))
elif not value.strip():
self.result.add_issue(ValidationIssue(
setting=setting,
severity=ValidationSeverity.ERROR,
message=f"Required setting {setting} is empty",
current_value=value
))
def _validate_paths(self) -> None:
"""Validate all path-related settings"""
for setting in self._path_settings:
value = os.environ.get(setting)
if not value:
if setting in self._required_settings:
continue # Already handled in required validation
else:
# Optional path settings
continue
if setting in {"TV_PATHS", "MOVIE_PATHS", "RADARR_ROOT_FOLDERS", "SONARR_ROOT_FOLDERS"}:
# Multi-path settings
paths = [p.strip() for p in value.split(",") if p.strip()]
if not paths:
self.result.add_issue(ValidationIssue(
setting=setting,
severity=ValidationSeverity.ERROR,
message=f"No valid paths found in {setting}",
current_value=value
))
continue
for path_str in paths:
self._validate_single_path(setting, path_str)
else:
# Single path settings
self._validate_single_path(setting, value)
def _validate_single_path(self, setting: str, path_str: str) -> None:
"""Validate a single path"""
try:
path = Path(path_str)
# Check if path is absolute (recommended for Docker)
if not path.is_absolute():
self.result.add_issue(ValidationIssue(
setting=setting,
severity=ValidationSeverity.WARNING,
message=f"Path should be absolute for reliable Docker operation",
current_value=path_str,
suggested_value=f"Use absolute path like /media/..."
))
# For media paths, check existence if not in container
if setting in {"TV_PATHS", "MOVIE_PATHS"} and not self._is_likely_container_path(path_str):
if not path.exists():
self.result.add_issue(ValidationIssue(
setting=setting,
severity=ValidationSeverity.WARNING,
message=f"Path does not exist (may be valid in container)",
current_value=path_str,
details={"path_type": "media"}
))
elif not path.is_dir():
self.result.add_issue(ValidationIssue(
setting=setting,
severity=ValidationSeverity.ERROR,
message=f"Path exists but is not a directory",
current_value=path_str
))
# Check for database directory
if setting == "DB_PATH":
parent_dir = path.parent
if not self._is_likely_container_path(str(parent_dir)) and not parent_dir.exists():
self.result.add_issue(ValidationIssue(
setting=setting,
severity=ValidationSeverity.WARNING,
message=f"Database directory does not exist: {parent_dir}",
current_value=path_str
))
except (OSError, ValueError) as e:
self.result.add_issue(ValidationIssue(
setting=setting,
severity=ValidationSeverity.ERROR,
message=f"Invalid path format: {e}",
current_value=path_str
))
def _is_likely_container_path(self, path: str) -> bool:
"""Check if path looks like a container path"""
container_indicators = ["/app/", "/media/", "/config/", "/data/"]
return any(indicator in path for indicator in container_indicators)
def _validate_urls(self) -> None:
"""Validate URL settings"""
for setting in self._url_settings:
value = os.environ.get(setting)
if value and not validate_url_format(value):
self.result.add_issue(ValidationIssue(
setting=setting,
severity=ValidationSeverity.ERROR,
message=f"Invalid URL format",
current_value=value,
suggested_value="Use format: http://hostname:port or https://hostname:port"
))
def _validate_numeric_settings(self) -> None:
"""Validate numeric settings"""
for setting, (type_class, min_val, max_val) in self._numeric_settings.items():
value = os.environ.get(setting)
if not value:
continue
try:
numeric_value = type_class(value)
if numeric_value < min_val or numeric_value > max_val:
self.result.add_issue(ValidationIssue(
setting=setting,
severity=ValidationSeverity.ERROR,
message=f"Value must be between {min_val} and {max_val}",
current_value=value,
suggested_value=f"Use value between {min_val}-{max_val}"
))
except (ValueError, TypeError):
self.result.add_issue(ValidationIssue(
setting=setting,
severity=ValidationSeverity.ERROR,
message=f"Invalid {type_class.__name__} value",
current_value=value,
suggested_value=f"Use a valid {type_class.__name__} between {min_val}-{max_val}"
))
def _validate_boolean_settings(self) -> None:
"""Validate boolean settings"""
valid_true = {"1", "true", "yes", "y", "on"}
valid_false = {"0", "false", "no", "n", "off"}
valid_values = valid_true | valid_false
for setting in self._boolean_settings:
value = os.environ.get(setting)
if value and value.lower() not in valid_values:
self.result.add_issue(ValidationIssue(
setting=setting,
severity=ValidationSeverity.ERROR,
message=f"Invalid boolean value",
current_value=value,
suggested_value="Use: true/false, 1/0, yes/no, y/n, on/off"
))
def _validate_choice_settings(self) -> None:
"""Validate settings with predefined choices"""
for setting, valid_choices in self._choice_settings.items():
value = os.environ.get(setting)
if value and value.lower() not in [choice.lower() for choice in valid_choices]:
self.result.add_issue(ValidationIssue(
setting=setting,
severity=ValidationSeverity.ERROR,
message=f"Invalid choice",
current_value=value,
suggested_value=f"Use one of: {', '.join(valid_choices)}"
))
def _validate_database_config(self) -> None:
"""Validate database configuration"""
db_type = os.environ.get("RADARR_DB_TYPE", "").lower()
if db_type == "postgresql":
# Check required PostgreSQL settings
required_pg_settings = ["RADARR_DB_HOST", "RADARR_DB_NAME", "RADARR_DB_USER"]
for setting in required_pg_settings:
if not os.environ.get(setting):
self.result.add_issue(ValidationIssue(
setting=setting,
severity=ValidationSeverity.ERROR,
message=f"Required for PostgreSQL database connection",
current_value=os.environ.get(setting)
))
def _validate_release_date_config(self) -> None:
"""Validate release date processing configuration"""
priority = os.environ.get("RELEASE_DATE_PRIORITY", "")
if priority:
priorities = [p.strip().lower() for p in priority.split(",")]
valid_priorities = {"digital", "physical", "theatrical"}
invalid_priorities = [p for p in priorities if p not in valid_priorities]
if invalid_priorities:
self.result.add_issue(ValidationIssue(
setting="RELEASE_DATE_PRIORITY",
severity=ValidationSeverity.ERROR,
message=f"Invalid release date priorities: {', '.join(invalid_priorities)}",
current_value=priority,
suggested_value="Use: digital, physical, theatrical"
))
if len(set(priorities)) != len(priorities):
self.result.add_issue(ValidationIssue(
setting="RELEASE_DATE_PRIORITY",
severity=ValidationSeverity.WARNING,
message="Duplicate priorities found",
current_value=priority
))
def _validate_performance_settings(self) -> None:
"""Validate performance-related settings"""
batch_delay = os.environ.get("BATCH_DELAY")
max_concurrent = os.environ.get("MAX_CONCURRENT_SERIES")
# Performance recommendations
if batch_delay:
try:
delay = float(batch_delay)
if delay < 1.0:
self.result.add_issue(ValidationIssue(
setting="BATCH_DELAY",
severity=ValidationSeverity.WARNING,
message="Very low batch delay may increase system load",
current_value=batch_delay,
suggested_value="Consider using 1.0 or higher for better stability"
))
except ValueError:
pass # Already caught in numeric validation
if max_concurrent:
try:
concurrent = int(max_concurrent)
if concurrent > 5:
self.result.add_issue(ValidationIssue(
setting="MAX_CONCURRENT_SERIES",
severity=ValidationSeverity.WARNING,
message="High concurrency may overload system resources",
current_value=max_concurrent,
suggested_value="Consider using 3-5 for optimal balance"
))
except ValueError:
pass # Already caught in numeric validation
def _validate_dependencies(self) -> None:
"""Validate cross-setting dependencies"""
# If database is configured, recommend using database over API
db_type = os.environ.get("RADARR_DB_TYPE")
radarr_url = os.environ.get("RADARR_URL")
if db_type and radarr_url:
self.result.add_issue(ValidationIssue(
setting="RADARR_DB_TYPE",
severity=ValidationSeverity.INFO,
message="Database connection preferred over API for better performance",
details={"recommendation": "Database access is faster and more reliable"}
))
# Check path mapping consistency
tv_paths = os.environ.get("TV_PATHS", "").split(",")
sonarr_paths = os.environ.get("SONARR_ROOT_FOLDERS", "").split(",")
if len([p for p in tv_paths if p.strip()]) != len([p for p in sonarr_paths if p.strip()]):
self.result.add_issue(ValidationIssue(
setting="TV_PATHS",
severity=ValidationSeverity.WARNING,
message="TV_PATHS and SONARR_ROOT_FOLDERS should have matching number of paths",
details={
"tv_paths_count": len([p for p in tv_paths if p.strip()]),
"sonarr_paths_count": len([p for p in sonarr_paths if p.strip()])
}
))
def validate_configuration() -> ValidationResult:
"""
Validate the complete NFOGuard configuration
Returns:
ValidationResult with all validation issues found
"""
validator = ConfigValidator()
return validator.validate_all()
def validate_configuration_and_raise() -> None:
"""
Validate configuration and raise ConfigurationError if invalid
Raises:
ConfigurationError: If configuration validation fails
"""
result = validate_configuration()
if not result.is_valid:
error_messages = []
for error in result.get_errors():
error_messages.append(f"{error.setting}: {error.message}")
raise ConfigurationError(
setting="configuration",
reason=f"Configuration validation failed with {result.errors_count} errors",
current_value={
"errors": error_messages,
"warnings_count": result.warnings_count,
"validation_details": result.to_dict()
}
)
def get_configuration_summary() -> Dict[str, Any]:
"""
Get a summary of current configuration status
Returns:
Dictionary with configuration summary
"""
result = validate_configuration()
return {
"is_valid": result.is_valid,
"errors_count": result.errors_count,
"warnings_count": result.warnings_count,
"total_issues": len(result.issues),
"critical_issues": [
issue.to_dict() for issue in result.issues
if issue.severity == ValidationSeverity.ERROR
],
"recommendations": [
issue.to_dict() for issue in result.issues
if issue.severity == ValidationSeverity.WARNING
]
}
core/database.py
+149 -1082
View File
File diff suppressed because it is too large Load Diff
core/database_populator.py
-476
View File
@@ -1,476 +0,0 @@
#!/usr/bin/env python3
"""
Database Populator for NFOGuard
Bulk populates the NFOGuard database from Radarr/Sonarr
Phase 4: Replace NFO-based initial population with direct DB/API queries
"""
import time
import hashlib
from typing import Dict, List, Optional, Tuple
from datetime import datetime
from pathlib import Path
from core.database import NFOGuardDatabase
from clients.radarr_client import RadarrClient
from clients.sonarr_client import SonarrClient
from utils.logging import _log
from utils.imdb_utils import parse_imdb_from_path
class DatabasePopulator:
"""Populates NFOGuard database from Radarr/Sonarr sources"""
def __init__(self, db: NFOGuardDatabase, radarr_client: RadarrClient, sonarr_client: SonarrClient):
self.db = db
self.radarr = radarr_client
self.sonarr = sonarr_client
def populate_movies(self) -> Dict[str, any]:
"""
Populate movies from Radarr database/API
Returns:
Dictionary with statistics: {
'total': int,
'added': int,
'updated': int,
'skipped': int,
'errors': int,
'duration': float
}
"""
_log("INFO", "Starting movie population from Radarr")
start_time = time.time()
stats = {
'total': 0,
'added': 0,
'updated': 0,
'skipped': 0,
'errors': 0,
'duration': 0.0,
'skipped_items': [] # Track what was skipped and why
}
try:
# Get all movies from Radarr database
if not hasattr(self.radarr, 'db_client') or not self.radarr.db_client:
_log("ERROR", "Radarr database client not available - cannot populate movies")
stats['errors'] += 1
return stats
movies = self.radarr.db_client.get_all_movies()
if not movies:
_log("WARNING", "No movies found in Radarr database")
return stats
stats['total'] = len(movies)
_log("INFO", f"Found {stats['total']} movies in Radarr")
# Process each movie
for movie in movies:
try:
# Get movie path first (we'll need it for IMDb extraction)
path = movie.get('path', '')
# Try to get IMDb ID from Radarr database
imdb_id = movie.get('imdb_id')
# If not in database, try extracting from directory/filename
if not imdb_id and path:
imdb_id = parse_imdb_from_path(Path(path))
if imdb_id:
_log("DEBUG", f"Extracted IMDb ID {imdb_id} from path for: {movie.get('title')}")
if not imdb_id:
# Generate placeholder IMDb ID using hash of path
path_hash = hashlib.md5(path.encode()).hexdigest()[:12]
imdb_id = f"missing-{path_hash}"
skip_reason = 'No IMDb ID found'
skip_info = {
'title': movie.get('title', 'Unknown'),
'year': movie.get('year'),
'imdb_id': imdb_id,
'path': path,
'reason': skip_reason
}
stats['skipped_items'].append(skip_info)
_log("DEBUG", f"Movie without IMDb ID: {movie.get('title')} (path: {path}), using placeholder {imdb_id}")
# Mark as skipped in database with placeholder IMDb ID
self.db.mark_movie_skipped(
imdb_id=imdb_id,
title=movie.get('title', 'Unknown'),
year=movie.get('year', 0),
path=path,
reason=skip_reason
)
stats['skipped'] += 1
continue
# Check if movie already exists in database
existing = self.db.get_movie_dates(imdb_id)
if existing and existing.get('dateadded'):
# Already in database - update file path and video status if needed
existing_path = existing.get('path')
if not existing_path or existing_path == 'unknown' or existing_path != path:
_log("INFO", f"Movie {imdb_id} exists but updating file info: {path}")
self.db.update_movie_file_info(imdb_id, path, has_video_file=True)
# Add to processing history
try:
self.db.add_processing_history(
imdb_id=imdb_id,
media_type='movie',
event_type='file_info_update',
details={'path': path}
)
except Exception as e:
_log("WARNING", f"Failed to add processing history for {imdb_id}: {e}")
stats['updated'] += 1
else:
_log("DEBUG", f"Movie {imdb_id} already in database with correct path, skipping")
continue
# Get release date
released = None
if movie.get('digital_release'):
released = movie.get('digital_release')
source_type = 'radarr:digital'
elif movie.get('physical_release'):
released = movie.get('physical_release')
source_type = 'radarr:physical'
elif movie.get('in_cinemas'):
released = movie.get('in_cinemas')
source_type = 'radarr:theatrical'
else:
source_type = 'radarr:unknown'
# Get import date from Radarr history using Radarr's internal movie ID
radarr_movie_id = movie.get('id')
if radarr_movie_id:
# get_movie_import_date returns tuple (date, source)
import_date, import_source = self.radarr.get_movie_import_date(radarr_movie_id)
if import_date:
dateadded = import_date
source = import_source
elif released:
# Use release date as fallback
dateadded = released
source = f'{source_type}_fallback'
else:
skip_reason = 'No import date in Radarr history and no release dates available'
skip_info = {
'title': movie.get('title', 'Unknown'),
'year': movie.get('year'),
'imdb_id': imdb_id,
'reason': skip_reason
}
stats['skipped_items'].append(skip_info)
_log("DEBUG", f"No date available for movie {imdb_id}, skipping")
# Mark as skipped in database for troubleshooting
self.db.mark_movie_skipped(
imdb_id=imdb_id,
title=movie.get('title', 'Unknown'),
year=movie.get('year', 0),
path=path or 'unknown',
reason=skip_reason
)
stats['skipped'] += 1
continue
elif released:
# No Radarr ID, use release date
dateadded = released
source = f'{source_type}_fallback'
else:
skip_reason = 'No Radarr movie ID and no release dates available'
skip_info = {
'title': movie.get('title', 'Unknown'),
'year': movie.get('year'),
'imdb_id': imdb_id,
'reason': skip_reason
}
stats['skipped_items'].append(skip_info)
_log("DEBUG", f"No date available for movie {imdb_id}, skipping")
# Mark as skipped in database for troubleshooting
self.db.mark_movie_skipped(
imdb_id=imdb_id,
title=movie.get('title', 'Unknown'),
year=movie.get('year', 0),
path=path or 'unknown',
reason=skip_reason
)
stats['skipped'] += 1
continue
# Insert into database with title and year
title = movie.get('title')
year = movie.get('year')
self.db.upsert_movie_dates(
imdb_id, released, dateadded, source,
has_video_file=True, title=title, year=year
)
# Add to processing history
try:
self.db.add_processing_history(
imdb_id=imdb_id,
media_type='movie',
event_type='database_population',
details={'source': source, 'title': title}
)
except Exception as e:
_log("WARNING", f"Failed to add processing history for {imdb_id}: {e}")
stats['added'] += 1
_log("DEBUG", f"Added movie {imdb_id}: {title} ({year}) (source: {source})")
except Exception as e:
_log("ERROR", f"Error processing movie {movie.get('title', 'unknown')}: {e}")
stats['errors'] += 1
continue
except Exception as e:
_log("ERROR", f"Error during movie population: {e}")
stats['errors'] += 1
stats['duration'] = time.time() - start_time
_log("INFO", f"Movie population complete: {stats['added']} added, {stats['skipped']} skipped, {stats['errors']} errors in {stats['duration']:.2f}s")
# Log details of skipped items
if stats['skipped_items']:
_log("INFO", f"Skipped items details ({len(stats['skipped_items'])} total):")
for item in stats['skipped_items']:
_log("INFO", f" - {item['title']} ({item.get('year', 'N/A')}) [{item.get('imdb_id', 'No IMDb')}]: {item['reason']}")
return stats
def populate_tv_episodes(self) -> Dict[str, any]:
"""
Populate TV episodes from Sonarr API
Returns:
Dictionary with statistics: {
'total_series': int,
'total_episodes': int,
'added': int,
'updated': int,
'skipped': int,
'errors': int,
'duration': float
}
"""
_log("INFO", "Starting TV episode population from Sonarr")
start_time = time.time()
stats = {
'total_series': 0,
'total_episodes': 0,
'added': 0,
'updated': 0,
'skipped': 0,
'errors': 0,
'duration': 0.0,
'skipped_items': [] # Track what was skipped and why
}
try:
# Get all series from Sonarr
all_series = self.sonarr.get_all_series()
if not all_series:
_log("WARNING", "No series found in Sonarr")
return stats
stats['total_series'] = len(all_series)
_log("INFO", f"Found {stats['total_series']} series in Sonarr")
# Process each series
for series in all_series:
try:
imdb_id = series.get('imdbId')
series_id = series.get('id')
series_path = series.get('path', '')
series_title = series.get('title', 'Unknown')
if not imdb_id:
# Generate placeholder IMDb ID using hash of path
path_hash = hashlib.md5(series_path.encode()).hexdigest()[:12]
imdb_id = f"missing-{path_hash}"
_log("DEBUG", f"Series without IMDb ID: {series_title} (path: {series_path}), using placeholder {imdb_id}")
# Update series record
self.db.upsert_series(imdb_id, series_path)
# Try high-performance database bulk query first
sonarr_db = getattr(self.sonarr, 'db_client', None)
bulk_import_dates = {}
if sonarr_db:
try:
_log("DEBUG", f"Using DB bulk query for {series_title}")
bulk_import_dates = sonarr_db.bulk_import_dates_for_series(series_id)
_log("DEBUG", f"✅ Got {len(bulk_import_dates)} import dates from DB for {series_title}")
except Exception as e:
_log("WARNING", f"DB bulk query failed for {series_title}, falling back to API: {e}")
# Get all episodes for this series
episodes = self.sonarr.episodes_for_series(series_id)
if not episodes:
continue
_log("DEBUG", f"Processing {len(episodes)} episodes for {series_title}")
# Process each episode
for episode in episodes:
try:
season_num = episode.get('seasonNumber', 0)
episode_num = episode.get('episodeNumber', 0)
episode_title = episode.get('title', 'Unknown')
if season_num < 0 or episode_num <= 0:
continue
stats['total_episodes'] += 1
# Check if episode already exists
existing = self.db.get_episode_date(imdb_id, season_num, episode_num)
if existing and existing.get('dateadded'):
# Already in database - update file path and video status if needed
existing_path = existing.get('path')
episode_path = episode.get('path', 'unknown')
if not existing_path or existing_path == 'unknown' or existing_path != episode_path:
_log("INFO", f"Episode {imdb_id} S{season_num:02d}E{episode_num:02d} exists but updating file info: {episode_path}")
self.db.update_episode_file_info(imdb_id, season_num, episode_num, episode_path, has_video_file=True)
# Add to processing history
try:
self.db.add_processing_history(
imdb_id=imdb_id,
media_type='episode',
event_type='file_info_update',
details={'season': season_num, 'episode': episode_num, 'path': episode_path}
)
except Exception as e:
_log("WARNING", f"Failed to add processing history for {imdb_id} S{season_num:02d}E{episode_num:02d}: {e}")
stats['updated'] += 1
continue
# Only process episodes that have video files
has_file = episode.get('hasFile', False)
if not has_file:
# No video file - skip silently (intentionally filtered)
continue
# Get air date
aired = episode.get('airDate')
# Get import date
dateadded = None
source = None
# Try bulk DB result first
if (season_num, episode_num) in bulk_import_dates:
dateadded, source = bulk_import_dates[(season_num, episode_num)]
# Fall back to API query
else:
episode_id = episode.get('id')
if episode_id:
import_date = self.sonarr.get_episode_import_history(episode_id)
if import_date:
dateadded = import_date
source = 'sonarr:api.import_history'
# Fallback to air date if no import date
if not dateadded and aired:
dateadded = aired
source = 'sonarr:aired_fallback'
elif not dateadded:
# No date available
skip_reason = 'No import date from Sonarr history and no air date available'
skip_info = {
'title': series_title,
'episode_title': episode_title,
'season': season_num,
'episode': episode_num,
'reason': skip_reason
}
stats['skipped_items'].append(skip_info)
# Mark as skipped in database for troubleshooting
self.db.mark_episode_skipped(
imdb_id=imdb_id,
season=season_num,
episode=episode_num,
reason=skip_reason
)
stats['skipped'] += 1
continue
# Insert into database
self.db.upsert_episode_date(imdb_id, season_num, episode_num, aired, dateadded, source, has_file)
# Add to processing history
try:
self.db.add_processing_history(
imdb_id=imdb_id,
media_type='episode',
event_type='database_population',
details={'season': season_num, 'episode': episode_num, 'source': source, 'title': episode_title}
)
except Exception as e:
_log("WARNING", f"Failed to add processing history for {imdb_id} S{season_num:02d}E{episode_num:02d}: {e}")
stats['added'] += 1
except Exception as e:
_log("ERROR", f"Error processing episode S{season_num:02d}E{episode_num:02d} of {series_title}: {e}")
stats['errors'] += 1
continue
except Exception as e:
_log("ERROR", f"Error processing series {series.get('title', 'unknown')}: {e}")
stats['errors'] += 1
continue
except Exception as e:
_log("ERROR", f"Error during TV episode population: {e}")
stats['errors'] += 1
stats['duration'] = time.time() - start_time
_log("INFO", f"TV episode population complete: {stats['added']} added, {stats['skipped']} skipped, {stats['errors']} errors in {stats['duration']:.2f}s")
# Log details of skipped items
if stats['skipped_items']:
_log("INFO", f"Skipped episodes details ({len(stats['skipped_items'])} total):")
for item in stats['skipped_items'][:20]: # Only log first 20 to avoid spam
_log("INFO", f" - {item['title']} S{str(item['season']).zfill(2)}E{str(item['episode']).zfill(2)} ({item.get('episode_title', 'Unknown')}): {item['reason']}")
if len(stats['skipped_items']) > 20:
_log("INFO", f" ... and {len(stats['skipped_items']) - 20} more (see web interface for full list)")
return stats
def populate_all(self) -> Dict[str, any]:
"""
Populate both movies and TV episodes
Returns:
Combined statistics dictionary
"""
_log("INFO", "Starting full database population")
start_time = time.time()
movie_stats = self.populate_movies()
tv_stats = self.populate_tv_episodes()
combined_stats = {
'movies': movie_stats,
'tv': tv_stats,
'total_duration': time.time() - start_time
}
_log("INFO", f"Full database population complete in {combined_stats['total_duration']:.2f}s")
return combined_stats
core/episode_nfo_manager.py
+276
View File
@@ -0,0 +1,276 @@
#!/usr/bin/env python3
"""
Episode NFO Manager - Handles TV episode NFO creation with video filename matching
Core principle: NFO filenames should match video filenames
"""
import xml.etree.ElementTree as ET
from pathlib import Path
from typing import Optional, Dict, Any, List, Tuple
import re
from .logging import _log
class EpisodeNFOManager:
"""Manages episode NFO files with video filename matching"""
def __init__(self, manager_brand: str = "NFOGuard"):
self.manager_brand = manager_brand
def find_video_files_for_season(self, season_dir: Path) -> Dict[Tuple[int, int], List[Path]]:
"""Find all video files in season directory, grouped by (season, episode)"""
if not season_dir.exists():
_log("DEBUG", f"Season directory does not exist: {season_dir}")
return {}
video_extensions = [".mkv", ".mp4", ".avi", ".mov", ".m4v"]
episodes = {}
_log("DEBUG", f"Scanning video files in: {season_dir}")
for video_file in season_dir.iterdir():
_log("DEBUG", f"Checking file: {video_file.name} (is_file: {video_file.is_file()}, suffix: {video_file.suffix.lower()})")
if (video_file.is_file() and
video_file.suffix.lower() in video_extensions):
episode_info = self._parse_episode_from_filename(video_file.name)
_log("DEBUG", f"Episode parsing for '{video_file.name}': {episode_info}")
if episode_info:
season_num, episode_num = episode_info
key = (season_num, episode_num)
if key not in episodes:
episodes[key] = []
episodes[key].append(video_file)
_log("DEBUG", f"Added video file: S{season_num:02d}E{episode_num:02d} → {video_file.name}")
_log("DEBUG", f"Total video files found: {len(episodes)} episodes")
return episodes
def _parse_episode_from_filename(self, filename: str) -> Optional[Tuple[int, int]]:
"""Extract season and episode numbers from filename"""
# Try S##E## format first (most common)
match = re.search(r'[Ss](\d{1,2})[Ee](\d{1,2})', filename)
if match:
return int(match.group(1)), int(match.group(2))
# Try ##x## format
match = re.search(r'(\d{1,2})x(\d{1,2})', filename)
if match:
return int(match.group(1)), int(match.group(2))
return None
def find_nfo_for_episode(self, season_dir: Path, season_num: int, episode_num: int) -> Optional[Path]:
"""Find existing NFO file for episode (prefer video-matching filename)"""
if not season_dir.exists():
return None
# First, look for NFO files that match video filenames
video_files = self.find_video_files_for_season(season_dir)
key = (season_num, episode_num)
if key in video_files:
for video_file in video_files[key]:
potential_nfo = season_dir / f"{video_file.stem}.nfo"
if potential_nfo.exists():
_log("DEBUG", f"Found video-matching NFO: {potential_nfo.name}")
return potential_nfo
# Fallback: look for short name NFO
short_nfo = season_dir / f"S{season_num:02d}E{episode_num:02d}.nfo"
if short_nfo.exists():
_log("DEBUG", f"Found short-name NFO: {short_nfo.name}")
return short_nfo
# Last resort: search all NFO files for matching season/episode data
for nfo_file in season_dir.glob("*.nfo"):
if self._nfo_matches_episode(nfo_file, season_num, episode_num):
_log("DEBUG", f"Found matching NFO by content: {nfo_file.name}")
return nfo_file
return None
def _nfo_matches_episode(self, nfo_path: Path, season_num: int, episode_num: int) -> bool:
"""Check if NFO file contains the specified season/episode"""
try:
tree = ET.parse(nfo_path)
root = tree.getroot()
if root.tag == "episodedetails":
season_elem = root.find("season")
episode_elem = root.find("episode")
if (season_elem is not None and episode_elem is not None and
season_elem.text and episode_elem.text):
try:
file_season = int(season_elem.text)
file_episode = int(episode_elem.text)
return file_season == season_num and file_episode == episode_num
except ValueError:
pass
except (ET.ParseError, Exception):
pass
return False
def get_target_nfo_path(self, season_dir: Path, season_num: int, episode_num: int) -> Path:
"""Get the target NFO path (prefer video filename, fallback to short name)"""
video_files = self.find_video_files_for_season(season_dir)
key = (season_num, episode_num)
if key in video_files and video_files[key]:
# Use the first video file found (handle multiple files gracefully)
video_file = video_files[key][0]
target_nfo = season_dir / f"{video_file.stem}.nfo"
_log("DEBUG", f"Target NFO will match video: {target_nfo.name}")
return target_nfo
else:
# Fallback to short name if no video file found
target_nfo = season_dir / f"S{season_num:02d}E{episode_num:02d}.nfo"
_log("WARNING", f"No video file found for S{season_num:02d}E{episode_num:02d}, using short name: {target_nfo.name}")
return target_nfo
def migrate_nfo_to_video_filename(self, season_dir: Path, season_num: int, episode_num: int) -> bool:
"""If short-name NFO exists, rename it to match video filename"""
existing_nfo = self.find_nfo_for_episode(season_dir, season_num, episode_num)
target_nfo = self.get_target_nfo_path(season_dir, season_num, episode_num)
# If we already have the right filename, nothing to do
if existing_nfo and existing_nfo == target_nfo:
return True
# If we have an NFO but it doesn't match target, rename it
if existing_nfo and existing_nfo != target_nfo:
try:
_log("INFO", f"Migrating NFO filename: {existing_nfo.name} -> {target_nfo.name}")
existing_nfo.rename(target_nfo)
return True
except Exception as e:
_log("ERROR", f"Failed to rename NFO: {e}")
return False
return False
def create_episode_nfo(self, season_dir: Path, season_num: int, episode_num: int,
aired: Optional[str], dateadded: Optional[str], source: str,
title: Optional[str] = None, plot: Optional[str] = None) -> bool:
"""Create or update episode NFO with video filename matching"""
# Get the target NFO path (matching video filename)
nfo_path = self.get_target_nfo_path(season_dir, season_num, episode_num)
# Migrate existing NFO if needed
self.migrate_nfo_to_video_filename(season_dir, season_num, episode_num)
try:
# Load existing NFO if it exists
episode_elem = None
if nfo_path.exists():
try:
tree = ET.parse(nfo_path)
episode_elem = tree.getroot()
if episode_elem.tag != "episodedetails":
raise ValueError("Root element is not <episodedetails>")
# Remove NFOGuard-managed fields (we'll re-add them)
for tag in ["season", "episode", "aired", "premiered", "dateadded", "lockdata"]:
existing = episode_elem.find(tag)
if existing is not None:
episode_elem.remove(existing)
_log("DEBUG", f"Loaded existing NFO content for {nfo_path.name}")
except (ET.ParseError, ValueError) as e:
_log("WARNING", f"Corrupted NFO file {nfo_path.name}: {e}. Creating new one.")
episode_elem = None
# Create new structure if needed
if episode_elem is None:
episode_elem = ET.Element("episodedetails")
# Add title if provided and not already present
if title and not episode_elem.find("title"):
title_elem = ET.SubElement(episode_elem, "title")
title_elem.text = title
# Add plot if provided and not already present
if plot and not episode_elem.find("plot"):
plot_elem = ET.SubElement(episode_elem, "plot")
plot_elem.text = plot
# Add NFOGuard fields at the end
season_elem = ET.SubElement(episode_elem, "season")
season_elem.text = str(season_num)
episode_num_elem = ET.SubElement(episode_elem, "episode")
episode_num_elem.text = str(episode_num)
if aired:
aired_elem = ET.SubElement(episode_elem, "aired")
aired_elem.text = aired[:10] if len(aired) >= 10 else aired
# Also add premiered for compatibility
premiered_elem = ET.SubElement(episode_elem, "premiered")
premiered_elem.text = aired[:10] if len(aired) >= 10 else aired
if dateadded:
dateadded_elem = ET.SubElement(episode_elem, "dateadded")
dateadded_elem.text = dateadded
# Add lockdata
lockdata_elem = ET.SubElement(episode_elem, "lockdata")
lockdata_elem.text = "true"
# Add comment with source
comment = ET.Comment(f" Created by {self.manager_brand} - Source: {source} ")
episode_elem.append(comment)
# Write the NFO file
tree = ET.ElementTree(episode_elem)
ET.indent(tree, space=" ", level=0)
tree.write(nfo_path, encoding='utf-8', xml_declaration=True)
_log("INFO", f"✅ Created/updated episode NFO: {nfo_path.name}")
_log("INFO", f" S{season_num:02d}E{episode_num:02d}, Aired: {aired}, DateAdded: {dateadded}, Source: {source}")
return True
except Exception as e:
_log("ERROR", f"Failed to create episode NFO {nfo_path}: {e}")
return False
def extract_nfoguard_data(self, nfo_path: Path) -> Optional[Dict[str, str]]:
"""Extract NFOGuard-managed data from existing NFO"""
if not nfo_path.exists():
return None
try:
tree = ET.parse(nfo_path)
root = tree.getroot()
if root.tag != "episodedetails":
return None
# Look for NFOGuard fields
dateadded_elem = root.find("dateadded")
aired_elem = root.find("aired")
lockdata_elem = root.find("lockdata")
# Only consider it NFOGuard-managed if it has dateadded and lockdata
if (dateadded_elem is not None and dateadded_elem.text and
lockdata_elem is not None and lockdata_elem.text == "true"):
result = {
"dateadded": dateadded_elem.text.strip(),
"source": "existing_nfo"
}
if aired_elem is not None and aired_elem.text:
result["aired"] = aired_elem.text.strip()
_log("DEBUG", f"Found NFOGuard data in {nfo_path.name}: {result}")
return result
except (ET.ParseError, Exception) as e:
_log("WARNING", f"Error parsing NFO {nfo_path.name}: {e}")
return None
core/nfo_manager.py
+700
View File
@@ -0,0 +1,700 @@
#!/usr/bin/env python3
"""
NFO Manager for creating and managing metadata files
Handles NFO creation for movies, TV shows, seasons, and episodes
"""
import os
import xml.etree.ElementTree as ET
from pathlib import Path
from datetime import datetime
from typing import Optional, Dict, Any, Tuple
import re
class NFOManager:
"""Manages NFO file creation and updates"""
def __init__(self, manager_brand: str = "NFOGuard", debug: bool = False):
self.manager_brand = manager_brand
self.debug = debug
def parse_imdb_from_path(self, path: Path) -> Optional[str]:
"""Extract IMDb ID from directory path or filename"""
# Look for various IMDb patterns in both directory and file names
path_str = str(path).lower()
# Try [imdb-ttXXXXXXX] format first (most explicit)
match = re.search(r'\[imdb-?(tt\d+)\]', path_str)
if match:
return match.group(1)
# Try standalone [ttXXXXXXX] format in brackets
match = re.search(r'\[(tt\d+)\]', path_str)
if match:
return match.group(1)
# Try {imdb-ttXXXXXXX} format with curly braces
match = re.search(r'\{imdb-?(tt\d+)\}', path_str)
if match:
return match.group(1)
# Try (imdb-ttXXXXXXX) format with parentheses
match = re.search(r'\(imdb-?(tt\d+)\)', path_str)
if match:
return match.group(1)
# Try ttXXXXXXX at end of filename/dirname (common pattern)
match = re.search(r'[-_\s](tt\d+)$', path_str)
if match:
return match.group(1)
return None
def parse_imdb_from_nfo(self, nfo_path: Path) -> Optional[str]:
"""Extract IMDb ID from NFO file content"""
if not nfo_path.exists():
return None
try:
root = self._parse_nfo_with_tolerance(nfo_path)
# Check for <uniqueid type="imdb">ttXXXXXX</uniqueid>
imdb_uniqueid = root.find('.//uniqueid[@type="imdb"]')
if imdb_uniqueid is not None and imdb_uniqueid.text:
imdb_id = imdb_uniqueid.text.strip()
if imdb_id.startswith('tt'):
return imdb_id
# Check for legacy <imdbid>ttXXXXXX</imdbid>
imdbid_elem = root.find('.//imdbid')
if imdbid_elem is not None and imdbid_elem.text:
imdb_id = imdbid_elem.text.strip()
if imdb_id.startswith('tt'):
return imdb_id
# Check for legacy <imdb>ttXXXXXX</imdb>
imdb_elem = root.find('.//imdb')
if imdb_elem is not None and imdb_elem.text:
imdb_id = imdb_elem.text.strip()
if imdb_id.startswith('tt'):
return imdb_id
# Last resort: Check for TMDB ID as fallback identifier
# This handles movies that only have TMDB IDs in NFO files
tmdb_uniqueid = root.find('.//uniqueid[@type="tmdb"]')
if tmdb_uniqueid is not None and tmdb_uniqueid.text:
tmdb_id = tmdb_uniqueid.text.strip()
if tmdb_id.isdigit():
print(f"⚠️ Found TMDB ID {tmdb_id} but no IMDb ID - using TMDB ID as fallback")
# Return TMDB ID with prefix to distinguish from IMDb IDs
return f"tmdb-{tmdb_id}"
except (ET.ParseError, Exception):
# Skip corrupted or non-XML files
pass
return None
def find_movie_imdb_id(self, movie_dir: Path) -> Optional[str]:
"""Find IMDb ID from directory name, filenames, or NFO file"""
print(f"🔍 Searching for IMDb ID in: {movie_dir.name}")
# First try directory name
imdb_id = self.parse_imdb_from_path(movie_dir)
if imdb_id:
print(f"✅ Found IMDb ID in directory name: {imdb_id}")
return imdb_id
# Try all files in the directory for IMDb ID patterns
for file_path in movie_dir.iterdir():
if file_path.is_file():
imdb_id = self.parse_imdb_from_path(file_path)
if imdb_id:
print(f"✅ Found IMDb ID in filename: {imdb_id} from {file_path.name}")
return imdb_id
# Finally, try NFO file content (including TMDB fallback)
nfo_path = movie_dir / "movie.nfo"
imdb_id = self.parse_imdb_from_nfo(nfo_path)
if imdb_id:
if imdb_id.startswith("tmdb-"):
print(f"✅ Found TMDB ID in NFO file: {imdb_id} from {nfo_path} (fallback mode)")
else:
print(f"✅ Found IMDb ID in NFO file: {imdb_id} from {nfo_path}")
return imdb_id
print(f"❌ No IMDb or TMDB ID found in directory, filenames, or NFO for: {movie_dir.name}")
return None
def extract_nfoguard_dates_from_nfo(self, nfo_path: Path) -> Optional[Dict[str, str]]:
"""Extract NFOGuard-managed dates from existing NFO file"""
if not nfo_path.exists():
return None
try:
tree = ET.parse(nfo_path)
root = tree.getroot()
# Look for NFOGuard fields
dateadded_elem = root.find('.//dateadded')
premiered_elem = root.find('.//premiered')
lockdata_elem = root.find('.//lockdata')
# Only consider it NFOGuard-managed if it has dateadded and lockdata
if (dateadded_elem is not None and dateadded_elem.text and
lockdata_elem is not None and lockdata_elem.text == "true"):
result = {
"dateadded": dateadded_elem.text.strip(),
"source": "nfo_file_existing"
}
if premiered_elem is not None and premiered_elem.text:
result["released"] = premiered_elem.text.strip()
print(f"✅ Found NFOGuard data in NFO: dateadded={result['dateadded']}, released={result.get('released', 'None')}")
return result
except (ET.ParseError, Exception) as e:
print(f"⚠️ Error parsing NFO for NFOGuard data: {e}")
pass
return None
def extract_nfoguard_dates_from_episode_nfo(self, season_path: Path, season_num: int, episode_num: int) -> Optional[Dict[str, str]]:
"""Extract NFOGuard-managed dates from existing episode NFO file"""
nfo_filename = f"s{season_num:02d}e{episode_num:02d}.nfo"
nfo_path = season_path / nfo_filename
if not nfo_path.exists():
return None
try:
tree = ET.parse(nfo_path)
root = tree.getroot()
# Look for NFOGuard fields in episode NFO
dateadded_elem = root.find('.//dateadded')
aired_elem = root.find('.//aired')
lockdata_elem = root.find('.//lockdata')
# Only consider it NFOGuard-managed if it has dateadded and lockdata
if (dateadded_elem is not None and dateadded_elem.text and
lockdata_elem is not None and lockdata_elem.text == "true"):
result = {
"dateadded": dateadded_elem.text.strip(),
"source": "episode_nfo_existing"
}
if aired_elem is not None and aired_elem.text:
result["aired"] = aired_elem.text.strip()
print(f"✅ Found NFOGuard data in episode NFO S{season_num:02d}E{episode_num:02d}: dateadded={result['dateadded']}, aired={result.get('aired', 'None')}")
return result
except (ET.ParseError, Exception) as e:
print(f"⚠️ Error parsing episode NFO for NFOGuard data: {e}")
pass
return None
def _parse_nfo_with_tolerance(self, nfo_path: Path):
"""Parse NFO file with tolerance for URLs appended after XML"""
try:
# First try normal parsing
tree = ET.parse(nfo_path)
return tree.getroot()
except ET.ParseError as e:
# If parsing fails, try to extract just the XML part
try:
with open(nfo_path, 'r', encoding='utf-8') as f:
content = f.read()
# Find the last </movie> tag and truncate after it
last_movie_end = content.rfind('</movie>')
if last_movie_end != -1:
xml_content = content[:last_movie_end + 8] # +8 for </movie>
# Try parsing the truncated content
root = ET.fromstring(xml_content)
print(f"✅ Successfully parsed NFO after removing trailing content: {nfo_path.name}")
return root
else:
# Re-raise original error if we can't find </movie>
raise e
except Exception:
# Re-raise original error if our fix attempt fails
raise e
def create_movie_nfo(self, movie_dir: Path, imdb_id: str, dateadded: str,
released: Optional[str] = None, source: str = "unknown",
lock_metadata: bool = True) -> None:
"""Create or update movie.nfo file preserving existing content"""
nfo_path = movie_dir / "movie.nfo"
print(f"🔍 create_movie_nfo called: imdb_id={imdb_id}, dateadded={dateadded}, released={released}, source={source}")
try:
# Try to load existing NFO file
if nfo_path.exists():
try:
# Try to parse the XML, handling URLs appended after </movie>
movie = self._parse_nfo_with_tolerance(nfo_path)
# Ensure root element is <movie>
if movie.tag != "movie":
raise ValueError("Root element is not <movie>")
# Remove existing NFOGuard-managed elements to avoid duplicates
# These will be re-added at the very bottom
nfoguard_fields = ["dateadded", "lockdata", "premiered", "year"]
for tag in nfoguard_fields:
existing = movie.find(tag)
if existing is not None:
# Store the value before removing (for premiered/year)
if tag == "premiered" and not released:
print(f"🔍 Preserving existing premiered date: {existing.text} (released was None)")
released = existing.text # Preserve existing premiered date
elif tag == "premiered":
print(f"🔍 NOT preserving premiered date: existing={existing.text}, released={released}")
movie.remove(existing)
# Remove ALL existing uniqueid with type="imdb" regardless of attributes
# We'll add a clean one at the bottom
for uniqueid in movie.findall("uniqueid[@type='imdb']"):
movie.remove(uniqueid)
except (ET.ParseError, ValueError) as e:
print(f"⚠️ Corrupted NFO detected: {nfo_path} - {str(e)[:100]}...")
print(f" Creating new clean NFO file to replace corrupted one")
movie = ET.Element("movie")
else:
# Create new NFO structure
movie = ET.Element("movie")
# Now append ALL NFOGuard fields at the VERY END of the file
# Create elements and explicitly append them to ensure they're at the bottom
# Add NFOGuard comment marker as the first of our additions
nfoguard_comment = ET.Comment(f" NFOGuard - Source: {source} ")
movie.append(nfoguard_comment)
# Add IMDb uniqueid at the end (after all existing content)
uniqueid = ET.Element("uniqueid", type="imdb", default="true")
uniqueid.text = imdb_id
movie.append(uniqueid)
# Add premiered date at the bottom if we have it
if released:
premiered_elem = ET.Element("premiered")
premiered_elem.text = released[:10] if len(released) >= 10 else released
movie.append(premiered_elem)
# Extract year from premiered date for consistency
try:
year_value = released[:4] if len(released) >= 4 else None
if year_value and year_value.isdigit():
year_elem = ET.Element("year")
year_elem.text = year_value
movie.append(year_elem)
except:
pass # Skip year if we can't extract it
# Add dateadded at the end - THIS IS CRITICAL FOR EMBY PLUGIN
if dateadded:
dateadded_elem = ET.Element("dateadded")
dateadded_elem.text = dateadded
movie.append(dateadded_elem)
print(f"✅ Added dateadded to NFO: {dateadded}")
# Add lockdata at the very end
if lock_metadata:
lockdata = ET.Element("lockdata")
lockdata.text = "true"
movie.append(lockdata)
# Write file with proper formatting
tree = ET.ElementTree(movie)
ET.indent(tree, space=" ", level=0)
# Write directly to file (comment is already embedded in XML)
with open(nfo_path, 'w', encoding='utf-8') as f:
f.write('<?xml version="1.0" encoding="utf-8"?>\n')
tree.write(f, encoding='unicode', xml_declaration=False)
print(f"✅ Successfully created/updated movie NFO: {nfo_path}")
print(f" IMDb ID: {imdb_id}, Date Added: {dateadded}, Source: {source}")
except Exception as e:
print(f"❌ Error creating/updating movie NFO {nfo_path}: {e}")
def create_tvshow_nfo(self, series_dir: Path, imdb_id: str, tvdb_id: Optional[str] = None) -> None:
"""Create or update tvshow.nfo file preserving existing content"""
nfo_path = series_dir / "tvshow.nfo"
try:
# Try to load existing NFO file
if nfo_path.exists():
try:
tree = ET.parse(nfo_path)
tvshow = tree.getroot()
# Ensure root element is <tvshow>
if tvshow.tag != "tvshow":
raise ValueError("Root element is not <tvshow>")
# Remove existing NFOGuard-managed elements to avoid duplicates
# These will be re-added at the bottom
for tag in ["lockdata"]:
existing = tvshow.find(tag)
if existing is not None:
tvshow.remove(existing)
# Remove ALL existing uniqueid with type="imdb" regardless of attributes
for uniqueid in tvshow.findall("uniqueid[@type='imdb']"):
tvshow.remove(uniqueid)
except (ET.ParseError, ValueError) as e:
print(f"⚠️ Corrupted TV show NFO detected: {nfo_path} - {str(e)[:100]}...")
print(f" Creating new clean tvshow.nfo file to replace corrupted one")
tvshow = ET.Element("tvshow")
else:
# Create new NFO structure
tvshow = ET.Element("tvshow")
# Add NFOGuard fields at the bottom
# Add IMDb uniqueid at the end
imdb_uniqueid = ET.SubElement(tvshow, "uniqueid", type="imdb", default="true")
imdb_uniqueid.text = imdb_id
# Add TVDB ID if available (preserve existing or add new)
if tvdb_id and not tvshow.find("uniqueid[@type='tvdb']"):
tvdb_uniqueid = ET.SubElement(tvshow, "uniqueid", type="tvdb")
tvdb_uniqueid.text = tvdb_id
# Add lockdata at the very end
lockdata = ET.SubElement(tvshow, "lockdata")
lockdata.text = "true"
# Add NFOGuard comment at the beginning
comment_text = f" Created by {self.manager_brand} "
# Write file with proper formatting
tree = ET.ElementTree(tvshow)
ET.indent(tree, space=" ", level=0)
# Write to string first to add comment
xml_str = ET.tostring(tvshow, encoding='unicode')
# Add XML declaration and comment
full_xml = f'<?xml version="1.0" encoding="utf-8"?>\n<!--{comment_text}-->\n{xml_str}'
# Write to file
with open(nfo_path, 'w', encoding='utf-8') as f:
f.write(full_xml)
print(f"✅ Successfully created/updated TV show NFO: {nfo_path}")
print(f" IMDb ID: {imdb_id}" + (f", TVDB ID: {tvdb_id}" if tvdb_id else ""))
except Exception as e:
print(f"❌ Error creating/updating tvshow NFO {nfo_path}: {e}")
def create_season_nfo(self, season_dir: Path, season_number: int) -> None:
"""Create or update season.nfo file preserving existing content"""
nfo_path = season_dir / "season.nfo"
try:
season_dir.mkdir(exist_ok=True)
# Try to load existing NFO file
if nfo_path.exists():
try:
tree = ET.parse(nfo_path)
season = tree.getroot()
# Ensure root element is <season>
if season.tag != "season":
raise ValueError("Root element is not <season>")
# Remove existing NFOGuard-managed elements
for tag in ["seasonnumber", "lockdata"]:
existing = season.find(tag)
if existing is not None:
season.remove(existing)
except (ET.ParseError, ValueError) as e:
print(f"⚠️ Corrupted season NFO detected: {nfo_path} - {str(e)[:100]}...")
print(f" Creating new clean season.nfo file to replace corrupted one")
season = ET.Element("season")
else:
# Create new NFO structure
season = ET.Element("season")
# Add NFOGuard fields at the bottom
seasonnumber = ET.SubElement(season, "seasonnumber")
seasonnumber.text = str(season_number)
# Add lockdata at the end
lockdata = ET.SubElement(season, "lockdata")
lockdata.text = "true"
# Add NFOGuard comment at the beginning
comment_text = f" Created by {self.manager_brand} "
# Write file with proper formatting
tree = ET.ElementTree(season)
ET.indent(tree, space=" ", level=0)
# Write to string first to add comment
xml_str = ET.tostring(season, encoding='unicode')
# Add XML declaration and comment
full_xml = f'<?xml version="1.0" encoding="utf-8"?>\n<!--{comment_text}-->\n{xml_str}'
# Write to file
with open(nfo_path, 'w', encoding='utf-8') as f:
f.write(full_xml)
print(f"✅ Successfully created/updated season NFO: {nfo_path}")
print(f" Season: {season_number}")
except Exception as e:
print(f"❌ Error creating/updating season NFO {nfo_path}: {e}")
def find_existing_episode_nfo(self, season_dir: Path, season_num: int, episode_num: int) -> Optional[Path]:
"""Find any existing episode NFO file that matches season/episode"""
if not season_dir.exists():
return None
# Look for NFO files in the season directory
for nfo_file in season_dir.glob("*.nfo"):
# Check if this NFO contains the right season/episode
try:
tree = ET.parse(nfo_file)
root = tree.getroot()
if root.tag == "episodedetails":
# Check for season/episode elements
season_elem = root.find("season")
episode_elem = root.find("episode")
if (season_elem is not None and episode_elem is not None and
season_elem.text and episode_elem.text):
try:
file_season = int(season_elem.text)
file_episode = int(episode_elem.text)
if file_season == season_num and file_episode == episode_num:
print(f"🔍 Found existing episode NFO: {nfo_file.name}")
return nfo_file
except ValueError:
continue
except (ET.ParseError, Exception):
# Skip corrupted or non-XML files
continue
return None
def _get_target_episode_nfo_name(self, season_dir: Path, season_num: int, episode_num: int) -> str:
"""Get target NFO filename - prefer existing NFO, then matching video file, fallback to short name"""
if not season_dir.exists():
return f"S{season_num:02d}E{episode_num:02d}.nfo"
# First check if an NFO already exists for this episode
existing_nfo = self.find_existing_episode_nfo(season_dir, season_num, episode_num)
if existing_nfo:
print(f"📂 Existing NFO found, will preserve filename: {existing_nfo.name}")
return existing_nfo.name
# Look for video files with matching season/episode
video_extensions = [".mkv", ".mp4", ".avi", ".mov", ".m4v", ".wmv", ".flv", ".webm"]
for video_file in season_dir.iterdir():
if (video_file.is_file() and
video_file.suffix.lower() in video_extensions):
# Parse episode info from video filename
episode_info = self._parse_episode_from_filename(video_file.name)
if episode_info and episode_info == (season_num, episode_num):
# Found matching video file - use its name for NFO
target_nfo_name = f"{video_file.stem}.nfo"
print(f"🎯 Target NFO will match video: {target_nfo_name}")
return target_nfo_name
# Fallback to short name if no matching video found
short_name = f"S{season_num:02d}E{episode_num:02d}.nfo"
print(f"⚠️ No matching video file found, using short name: {short_name}")
return short_name
def _parse_episode_from_filename(self, filename: str) -> Optional[Tuple[int, int]]:
"""Parse season and episode numbers from filename"""
# Try S##E## format first
match = re.search(r'[Ss](\d{1,2})[Ee](\d{1,2})', filename)
if match:
return int(match.group(1)), int(match.group(2))
# Try ##x## format
match = re.search(r'(\d{1,2})x(\d{1,2})', filename)
if match:
return int(match.group(1)), int(match.group(2))
return None
def create_episode_nfo(self, season_dir: Path, season_num: int, episode_num: int,
aired: Optional[str], dateadded: Optional[str], source: str,
lock_metadata: bool = True, enhanced_metadata: Optional[Dict[str, Any]] = None) -> None:
"""Create or update episode NFO file preserving existing content"""
# Get target NFO filename (prefer long name matching video file)
target_nfo_name = self._get_target_episode_nfo_name(season_dir, season_num, episode_num)
nfo_path = season_dir / target_nfo_name
try:
# Check for existing NFO file at target location
source_nfo_path = nfo_path if nfo_path.exists() else None
if source_nfo_path:
try:
tree = ET.parse(source_nfo_path)
episode = tree.getroot()
# Ensure root element is <episodedetails>
if episode.tag != "episodedetails":
raise ValueError("Root element is not <episodedetails>")
print(f"📝 Updating existing NFO: {nfo_path.name}")
# Preserve existing content fields and store NFOGuard-managed fields for re-adding
preserved_values = {}
# Extract and preserve NFOGuard-managed fields (we'll re-add these at the bottom)
nfoguard_fields = ["aired", "dateadded", "lockdata", "season", "episode"]
for tag in nfoguard_fields:
existing = episode.find(tag)
if existing is not None:
# Store the value before removing
if tag == "aired" and not aired:
aired = existing.text # Preserve existing aired date
preserved_values[tag] = existing.text
episode.remove(existing)
# Important: DO NOT remove content fields like title, plot, runtime, premiered, etc.
# These should be preserved from the long-named NFO files
# Debug: Show what fields are preserved after removing NFOGuard fields (if DEBUG enabled)
if self.debug:
preserved_fields = [elem.tag for elem in episode]
if preserved_fields:
print(f" 🔍 Content preserved after cleanup: {', '.join(preserved_fields)}")
else:
print(f" ⚠️ No content fields found after cleanup!")
except (ET.ParseError, ValueError) as e:
print(f"⚠️ Corrupted episode NFO detected: {nfo_path} - {str(e)[:100]}...")
print(f" Creating new clean episode NFO file to replace corrupted one")
episode = ET.Element("episodedetails")
else:
# Create new NFO structure
episode = ET.Element("episodedetails")
# Add enhanced metadata only if not already present (preserve existing from long-named NFO)
if enhanced_metadata:
if enhanced_metadata.get("title") and not episode.find("title"):
title_elem = ET.SubElement(episode, "title")
title_elem.text = enhanced_metadata["title"]
if enhanced_metadata.get("overview") and not episode.find("plot"):
plot_elem = ET.SubElement(episode, "plot")
plot_elem.text = enhanced_metadata["overview"]
if enhanced_metadata.get("runtime") and not episode.find("runtime"):
runtime_elem = ET.SubElement(episode, "runtime")
runtime_elem.text = str(enhanced_metadata["runtime"])
# Add NFOGuard fields at the bottom
# Basic episode info at the end
season_elem = ET.SubElement(episode, "season")
season_elem.text = str(season_num)
episode_elem = ET.SubElement(episode, "episode")
episode_elem.text = str(episode_num)
# Dates at the end
if aired:
aired_elem = ET.SubElement(episode, "aired")
aired_elem.text = aired[:10] if len(aired) >= 10 else aired
if dateadded:
dateadded_elem = ET.SubElement(episode, "dateadded")
dateadded_elem.text = dateadded
# Add lockdata at the very end
if lock_metadata:
lockdata = ET.SubElement(episode, "lockdata")
lockdata.text = "true"
# Add NFOGuard comment at the beginning
comment_text = f" Created by {self.manager_brand} - Source: {source} "
# Write file with proper formatting
tree = ET.ElementTree(episode)
ET.indent(tree, space=" ", level=0)
# Write to string first to add comment
xml_str = ET.tostring(episode, encoding='unicode')
# Add XML declaration and comment
full_xml = f'<?xml version="1.0" encoding="utf-8"?>\n<!--{comment_text}-->\n{xml_str}'
# Write to file
with open(nfo_path, 'w', encoding='utf-8') as f:
f.write(full_xml)
print(f"✅ Successfully created/updated episode NFO: {nfo_path}")
print(f" S{season_num:02d}E{episode_num:02d}, Aired: {aired}, Date Added: {dateadded}")
# NFO file created/updated successfully
pass
except Exception as e:
print(f"❌ Error creating/updating episode NFO {nfo_path}: {e}")
def set_file_mtime(self, file_path: Path, iso_timestamp: str) -> None:
"""Set file modification time to match import date"""
try:
# Parse ISO timestamp
if iso_timestamp.endswith('Z'):
dt = datetime.fromisoformat(iso_timestamp.replace('Z', '+00:00'))
elif '+' in iso_timestamp or 'T' in iso_timestamp:
dt = datetime.fromisoformat(iso_timestamp)
else:
# Assume it's already a simple date
dt = datetime.fromisoformat(iso_timestamp + 'T00:00:00')
# Convert to timestamp
timestamp = dt.timestamp()
# Set both access and modification times
os.utime(file_path, (timestamp, timestamp))
print(f"✅ Updated file timestamp: {file_path.name} -> {iso_timestamp}")
except Exception as e:
print(f"❌ Error setting mtime for {file_path}: {e}")
def update_movie_files_mtime(self, movie_dir: Path, iso_timestamp: str) -> None:
"""Update modification times for all video files in movie directory"""
video_exts = (".mkv", ".mp4", ".avi", ".mov", ".m4v")
updated_files = []
for file_path in movie_dir.iterdir():
if file_path.is_file() and file_path.suffix.lower() in video_exts:
self.set_file_mtime(file_path, iso_timestamp)
updated_files.append(file_path.name)
if updated_files:
print(f"✅ Updated {len(updated_files)} video file timestamps in {movie_dir.name}")
else:
print(f"⚠️ No video files found to update in {movie_dir.name}")
debug_movie.py
-63
View File
@@ -1,63 +0,0 @@
#!/usr/bin/env python3
"""
Debug script to check specific movie data in NFOGuard database
"""
import os
import sys
from pathlib import Path
# Add the project root to the path
sys.path.insert(0, str(Path(__file__).parent))
from core.database import NFOGuardDatabase
from config.settings import config
def debug_movie(imdb_id: str):
"""Debug a specific movie's data"""
print(f"🔍 DEBUG MOVIE: {imdb_id}")
print("=" * 50)
# Initialize database
db = NFOGuardDatabase(config=config)
# Get movie data
movie = db.get_movie_dates(imdb_id)
if not movie:
print(f"❌ Movie {imdb_id} not found in database")
return
print("📊 RAW MOVIE DATA:")
for key, value in movie.items():
print(f" {key}: {repr(value)}")
print("\n🎬 FORMATTED MOVIE DATA:")
print(f" Title/Path: {movie.get('path', 'Unknown')}")
print(f" Released: {movie.get('released', 'None')}")
print(f" Date Added: {movie.get('dateadded', 'None')}")
print(f" Source: {movie.get('source', 'None')}")
print(f" Has Video: {movie.get('has_video_file', False)}")
print(f" Last Updated: {movie.get('last_updated', 'None')}")
# Check if released date is valid
released = movie.get('released')
if released and released.strip():
try:
from datetime import datetime
test_date = f"{released}T00:00:00"
parsed = datetime.fromisoformat(test_date.replace('Z', '+00:00'))
print(f"\n✅ Released date is valid: {parsed}")
except Exception as e:
print(f"\n❌ Released date is INVALID: {e}")
else:
print(f"\n⚠️ Released date is empty or None")
if __name__ == "__main__":
if len(sys.argv) != 2:
print("Usage: python debug_movie.py <imdb_id>")
sys.exit(1)
imdb_id = sys.argv[1]
if not imdb_id.startswith('tt'):
imdb_id = f'tt{imdb_id}'
debug_movie(imdb_id)
debug_tv.py
-202
View File
@@ -1,202 +0,0 @@
#!/usr/bin/env python3
"""
Debug script to check specific TV series/episode data in NFOGuard database
"""
import os
import sys
from pathlib import Path
# Add the project root to the path
sys.path.insert(0, str(Path(__file__).parent))
from core.database import NFOGuardDatabase
from config.settings import config
def debug_series(imdb_id: str):
"""Debug a specific TV series' data"""
print(f"📺 DEBUG TV SERIES: {imdb_id}")
print("=" * 60)
# Initialize database
db = NFOGuardDatabase(config=config)
# Get series episodes
episodes = db.get_series_episodes(imdb_id)
if not episodes:
print(f"❌ TV series {imdb_id} not found in database")
return
print(f"📊 SERIES OVERVIEW:")
print(f" IMDb ID: {imdb_id}")
print(f" Total Episodes: {len(episodes)}")
# Count episodes by status
with_dates = sum(1 for ep in episodes if ep.get('dateadded'))
without_dates = len(episodes) - with_dates
with_video = sum(1 for ep in episodes if ep.get('has_video_file'))
print(f" Episodes with dates: {with_dates}")
print(f" Episodes without dates: {without_dates}")
print(f" Episodes with video files: {with_video}")
# Group by season
seasons = {}
for ep in episodes:
season = ep.get('season', 'Unknown')
if season not in seasons:
seasons[season] = []
seasons[season].append(ep)
print(f" Seasons: {len(seasons)} ({', '.join(f'S{s}' if isinstance(s, int) else str(s) for s in sorted(seasons.keys()))})")
# Show sources breakdown
sources = {}
for ep in episodes:
source = ep.get('source', 'None')
sources[source] = sources.get(source, 0) + 1
print(f"\n📈 SOURCES BREAKDOWN:")
for source, count in sorted(sources.items(), key=lambda x: x[1], reverse=True):
print(f" {source}: {count} episodes")
# Show recent episodes (last 10 by date added)
episodes_with_dates = [ep for ep in episodes if ep.get('dateadded')]
recent_episodes = sorted(episodes_with_dates, key=lambda x: x.get('last_updated', ''), reverse=True)[:10]
if recent_episodes:
print(f"\n🕒 RECENT EPISODES (by last_updated):")
for ep in recent_episodes:
season = ep.get('season', '?')
episode = ep.get('episode', '?')
dateadded = ep.get('dateadded', 'None')
source = ep.get('source', 'None')
video = "✅" if ep.get('has_video_file') else "❌"
print(f" S{season:02d}E{episode:02d}: {dateadded} | {source} | Video: {video}")
def debug_episode(imdb_id: str, season: int, episode: int):
"""Debug a specific episode's data"""
print(f"📺 DEBUG TV EPISODE: {imdb_id} S{season:02d}E{episode:02d}")
print("=" * 60)
# Initialize database
db = NFOGuardDatabase(config=config)
# Get specific episode
episode_data = db.get_episode_date(imdb_id, season, episode)
if not episode_data:
print(f"❌ Episode S{season:02d}E{episode:02d} for series {imdb_id} not found in database")
return
print("📊 RAW EPISODE DATA:")
for key, value in episode_data.items():
print(f" {key}: {repr(value)}")
print("\n📺 FORMATTED EPISODE DATA:")
print(f" Series IMDb: {episode_data.get('imdb_id', 'Unknown')}")
print(f" Season/Episode: S{episode_data.get('season', '?'):02d}E{episode_data.get('episode', '?'):02d}")
print(f" Title: {episode_data.get('title', 'Unknown')}")
print(f" Air Date: {episode_data.get('air_date', 'None')}")
print(f" Date Added: {episode_data.get('dateadded', 'None')}")
print(f" Source: {episode_data.get('source', 'None')}")
print(f" Has Video: {episode_data.get('has_video_file', False)}")
print(f" Video Path: {episode_data.get('video_path', 'None')}")
print(f" Last Updated: {episode_data.get('last_updated', 'None')}")
# Check if air date is valid
air_date = episode_data.get('air_date')
if air_date and air_date.strip():
try:
from datetime import datetime
test_date = f"{air_date}T00:00:00"
parsed = datetime.fromisoformat(test_date.replace('Z', '+00:00'))
print(f"\n✅ Air date is valid: {parsed}")
except Exception as e:
print(f"\n❌ Air date is INVALID: {e}")
else:
print(f"\n⚠️ Air date is empty or None")
# Check if dateadded is valid
dateadded = episode_data.get('dateadded')
if dateadded and dateadded.strip():
try:
from datetime import datetime
if isinstance(dateadded, str):
test_date = f"{dateadded}T00:00:00" if 'T' not in dateadded else dateadded
parsed = datetime.fromisoformat(test_date.replace('Z', '+00:00'))
else:
parsed = dateadded
print(f"✅ Date added is valid: {parsed}")
except Exception as e:
print(f"❌ Date added is INVALID: {e}")
else:
print(f"⚠️ Date added is empty or None")
def debug_season(imdb_id: str, season: int):
"""Debug all episodes in a specific season"""
print(f"📺 DEBUG TV SEASON: {imdb_id} Season {season}")
print("=" * 60)
# Initialize database
db = NFOGuardDatabase(config=config)
# Get series episodes
all_episodes = db.get_series_episodes(imdb_id)
season_episodes = [ep for ep in all_episodes if ep.get('season') == season]
if not season_episodes:
print(f"❌ No episodes found for season {season} of series {imdb_id}")
return
print(f"📊 SEASON {season} OVERVIEW:")
print(f" Total Episodes: {len(season_episodes)}")
# Sort by episode number
season_episodes.sort(key=lambda x: x.get('episode', 0))
with_dates = sum(1 for ep in season_episodes if ep.get('dateadded'))
without_dates = len(season_episodes) - with_dates
with_video = sum(1 for ep in season_episodes if ep.get('has_video_file'))
print(f" Episodes with dates: {with_dates}")
print(f" Episodes without dates: {without_dates}")
print(f" Episodes with video files: {with_video}")
print(f"\n📋 EPISODE LIST:")
for ep in season_episodes:
episode_num = ep.get('episode', '?')
title = ep.get('title', 'Unknown')[:30] + ('...' if len(ep.get('title', '')) > 30 else '')
dateadded = ep.get('dateadded', 'None')
source = ep.get('source', 'None')
video = "✅" if ep.get('has_video_file') else "❌"
air_date = ep.get('air_date', 'None')
print(f" E{episode_num:02d}: {title:<33} | Added: {dateadded} | Air: {air_date} | Video: {video}")
if __name__ == "__main__":
if len(sys.argv) < 2:
print("Usage:")
print(" python debug_tv.py <imdb_id> # Debug entire series")
print(" python debug_tv.py <imdb_id> <season> # Debug specific season")
print(" python debug_tv.py <imdb_id> <season> <episode> # Debug specific episode")
print("\nExamples:")
print(" python debug_tv.py tt0121955 # Debug South Park")
print(" python debug_tv.py tt0121955 27 # Debug South Park Season 27")
print(" python debug_tv.py tt0121955 27 6 # Debug South Park S27E06")
sys.exit(1)
imdb_id = sys.argv[1]
if not imdb_id.startswith('tt'):
imdb_id = f'tt{imdb_id}'
if len(sys.argv) == 4:
# Debug specific episode
season = int(sys.argv[2])
episode = int(sys.argv[3])
debug_episode(imdb_id, season, episode)
elif len(sys.argv) == 3:
# Debug specific season
season = int(sys.argv[2])
debug_season(imdb_id, season)
else:
# Debug entire series
debug_series(imdb_id)
docker-compose.legacy-single.yml → docker-compose.example
+6 -74
View File
@@ -1,14 +1,3 @@
# NFOGuard Legacy Single-Container Configuration
#
# DEPRECATED: This is the legacy single-container setup where web interface
# and core processing run in the same container, which can cause performance
# issues during intensive scans.
#
# RECOMMENDED: Use docker-compose.example.yml for the new 3-container
# architecture with better performance isolation.
#
# This file is maintained for backward compatibility and migration purposes.
version: '3.8'
services:
@@ -17,24 +6,16 @@ services:
image: sbcrumb/nfoguard:latest
# Alternative: Use specific version
# image: sbcrumb/nfoguard:v2.6.5
# image: sbcrumb/nfoguard:v1.5.5
# Alternative: Use development version
# image: sbcrumb/nfoguard:dev
container_name: nfoguard
# Database dependency
depends_on:
- nfoguard-postgres
# Restart policy
restart: unless-stopped
# Graceful shutdown configuration
stop_grace_period: 30s
stop_signal: SIGTERM
# Environment files
env_file:
- .env
@@ -102,54 +83,6 @@ services:
max-size: "10m"
max-file: "3"
# PostgreSQL Database (Required for v2.6+)
nfoguard-postgres:
image: postgres:15-alpine
container_name: nfoguard-db
# Restart policy
restart: unless-stopped
# Environment variables for PostgreSQL
environment:
- POSTGRES_DB=nfoguard
- POSTGRES_USER=nfoguard
- POSTGRES_PASSWORD=${DB_PASSWORD} # Set in .env.secrets
- POSTGRES_INITDB_ARGS=--encoding=UTF-8 --lc-collate=C --lc-ctype=C
# PostgreSQL data persistence
volumes:
- ./postgres-data:/var/lib/postgresql/data
# PostgreSQL port (optional - only needed for external access)
# ports:
# - "5432:5432"
# Health check for database
healthcheck:
test: ["CMD-SHELL", "pg_isready -U nfoguard -d nfoguard"]
interval: 30s
timeout: 10s
retries: 3
start_period: 60s
# Resource limits for database
deploy:
resources:
limits:
memory: 256M
cpus: '0.3'
reservations:
memory: 128M
cpus: '0.1'
# Logging configuration
logging:
driver: "json-file"
options:
max-size: "10m"
max-file: "3"
# Optional: Custom network for media services
# networks:
# media-network:
@@ -160,13 +93,12 @@ services:
# ===========================================
# 1. Copy this file to docker-compose.yml
# 2. Update volume mounts to match your actual media paths
# 3. Copy .env.example to .env and configure all settings
# 4. Copy .env.secrets.example to .env.secrets and add credentials (especially DB_PASSWORD)
# 5. Create directories: mkdir -p ./data ./postgres-data
# 3. Copy .env.template to .env and configure
# 4. Copy .env.secrets.template to .env.secrets and add credentials
# 5. Create data directory: mkdir -p ./data
# 6. Run: docker-compose up -d
# 7. Check logs: docker-compose logs -f
# 8. Access web interface: http://localhost:8080
# 9. Access health check: curl http://localhost:8080/health
# 7. Check logs: docker-compose logs -f nfoguard
# 8. Access health check: curl http://localhost:8080/health
# ===========================================
# VOLUME MAPPING EXAMPLES
docker-compose.example.yml
-144
View File
@@ -1,144 +0,0 @@
# NFOGuard Production Docker Compose - 3-Container Architecture
#
# RECOMMENDED SETUP: Separated core processing and web interface for optimal performance
#
# This is the default configuration providing:
# - Performance isolation between web and processing
# - Webhook responsiveness during scans
# - Independent scaling and updates
# - Professional web interface with branding
#
# For legacy single-container setup, see: docker-compose.legacy-single.yml
services:
# PostgreSQL Database
nfoguard-db:
image: postgres:15-alpine
container_name: nfoguard-db
restart: unless-stopped
env_file:
- .env
- .env.secrets
environment:
- POSTGRES_DB=${DB_NAME}
- POSTGRES_USER=${DB_USER}
- POSTGRES_PASSWORD=${DB_PASSWORD}
- TZ=${TZ}
volumes:
- postgres_data:/var/lib/postgresql/data
ports:
- "${DB_EXTERNAL_PORT:-5432}:5432" # Optional external access
healthcheck:
test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-nfoguard} -d ${DB_NAME:-nfoguard}"]
interval: 10s
timeout: 5s
retries: 5
networks:
- nfoguard-network
# NFOGuard Core (Processing Engine)
nfoguard:
image: sbcrumb/nfoguard:latest
container_name: nfoguard-core
restart: unless-stopped
env_file:
- .env
- .env.secrets
environment:
- TZ=${TZ}
- WEB_EXTERNAL_PORT=${WEB_EXTERNAL_PORT}
volumes:
# Media paths (adjust to your setup)
- /mnt/unionfs/Media/TV:/media/TV:ro
- /mnt/unionfs/Media/Movies:/media/Movies:ro
# Data persistence
- nfoguard_data:/app/data
# Logs
- nfoguard_logs:/app/data/logs
ports:
- "${CORE_API_PORT:-8080}:8080" # Core API (webhooks, processing)
depends_on:
nfoguard-db:
condition: service_healthy
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
interval: 30s
timeout: 10s
retries: 3
start_period: 30s
networks:
- nfoguard-network
# NFOGuard Web Interface
nfoguard-web:
image: sbcrumb/nfoguard:latest # Same image as core!
container_name: nfoguard-web
restart: unless-stopped
command: ["python", "start_web.py"] # Different entry point
env_file:
- .env
- .env.secrets
environment:
- TZ=${TZ:-America/New_York}
ports:
- "${WEB_API_PORT:-8081}:8081" # Web Interface
depends_on:
nfoguard-db:
condition: service_healthy
nfoguard:
condition: service_healthy
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8081/health"]
interval: 30s
timeout: 10s
retries: 3
start_period: 10s
networks:
- nfoguard-network
volumes:
postgres_data:
driver: local
nfoguard_data:
driver: local
nfoguard_logs:
driver: local
networks:
nfoguard-network:
driver: bridge
# Configuration Notes:
# 1. Core Processing (nfoguard): Handles webhooks, scanning, NFO management
# 2. Web Interface (nfoguard-web): Lightweight dashboard and management
# 3. Database (nfoguard-db): Shared PostgreSQL database
#
# Port Configuration:
# - Core API: ${CORE_API_PORT:-8080} (webhooks, processing)
# - Web Interface: ${WEB_API_PORT:-8081} (dashboard)
# - Database: ${DB_EXTERNAL_PORT:-5432} (optional external access)
#
# Performance Benefits:
# - Web interface operations don't impact core processing
# - Webhooks remain responsive during long scans
# - Independent scaling and resource allocation
# - Separated concerns for maintenance and updates
# NFOGuard Core (Processing Engine)
nfoguard:
# ... other settings ...
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8080/health/simple"]
interval: 30s
timeout: 15s # Increased from 10s
retries: 3
start_period: 60s # Increased from 40s
# NFOGuard Web Interface
nfoguard-web:
# ... other settings ...
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8081/health"]
interval: 30s
timeout: 15s # Increased from 10s
retries: 3
start_period: 30s # Increased from 10s
logo/NFOGuardLogo.png
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 926 KiB

logo/NFOguardLogoPlain.png
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 926 KiB

main.py
+19 -98
View File
@@ -1,13 +1,11 @@
#!/usr/bin/env python3
"""
NFOGuard Core - Automated NFO file management and processing engine
Core processing container with webhooks, scanning, and database management
Web interface separated to nfoguard-web container
NFOGuard - Automated NFO file management for Radarr and Sonarr
Modular architecture with webhook processing and intelligent date handling
"""
import os
import sys
import signal
import asyncio
from pathlib import Path
from datetime import datetime, timezone
@@ -16,13 +14,11 @@ from fastapi import FastAPI
# Import configuration first
from config.settings import config
# Authentication removed - handled by separate web container
from utils.logging import _log
# Import core components
from core.database import NFOGuardDatabase
# from core.nfo_manager import NFOManager # Phase 3: Removed - no longer needed
from core.nfo_manager import NFOManager
from core.path_mapper import PathMapper
# Import clients
@@ -38,8 +34,6 @@ from webhooks.webhook_batcher import WebhookBatcher
# Import API routes
from api.routes import register_routes
# Global shutdown event for graceful shutdown coordination
shutdown_event = asyncio.Event()
def get_version() -> str:
"""Get application version"""
@@ -92,72 +86,34 @@ def initialize_components():
start_time = datetime.now(timezone.utc)
# Initialize core components
db = NFOGuardDatabase(config=config)
# nfo_manager = NFOManager(config.manager_brand, config.debug) # Phase 3: Removed
db = NFOGuardDatabase(config.db_path)
nfo_manager = NFOManager(config.manager_brand, config.debug)
path_mapper = PathMapper(config)
# Initialize processors (nfo_manager=None for backward compatibility)
tv_processor = TVProcessor(db, None, path_mapper)
movie_processor = MovieProcessor(db, None, path_mapper)
# Initialize webhook batcher (no longer needs nfo_manager - Phase 3)
batcher = WebhookBatcher(nfo_manager=None)
# Initialize processors
tv_processor = TVProcessor(db, nfo_manager, path_mapper)
movie_processor = MovieProcessor(db, nfo_manager, path_mapper)
# Initialize webhook batcher
batcher = WebhookBatcher()
batcher.set_processors(tv_processor, movie_processor)
return {
"db": db,
# "nfo_manager": nfo_manager, # Phase 3: Removed
"nfo_manager": nfo_manager,
"path_mapper": path_mapper,
"tv_processor": tv_processor,
"movie_processor": movie_processor,
"batcher": batcher,
"start_time": start_time,
"config": config,
"version": get_version(),
"shutdown_event": shutdown_event
"version": get_version()
}
def signal_handler(signum, frame):
"""Handle shutdown signals gracefully"""
_log("INFO", f"Received signal {signum}, shutting down gracefully...")
# Set shutdown event to notify background tasks
shutdown_event.set()
# Get the global dependencies if they exist
if hasattr(signal_handler, 'dependencies') and signal_handler.dependencies:
deps = signal_handler.dependencies
# Shutdown webhook batcher cleanly
if 'batcher' in deps:
try:
_log("INFO", "Shutting down webhook batcher...")
deps['batcher'].shutdown()
except Exception as e:
_log("WARNING", f"Error during batcher shutdown: {e}")
# Close database connection
if 'db' in deps:
try:
_log("INFO", "Closing database connection...")
deps['db'].close()
except Exception as e:
_log("WARNING", f"Error closing database: {e}")
_log("INFO", "Graceful shutdown complete")
# Force exit after 2 seconds if graceful shutdown doesn't work
import threading
def force_exit():
import time
time.sleep(2)
_log("WARNING", "Force exiting after timeout")
os._exit(0)
force_thread = threading.Thread(target=force_exit, daemon=True)
force_thread.start()
sys.exit(0)
@@ -173,11 +129,7 @@ def main():
_log("INFO", f"Version: {version}")
_log("INFO", f"TV paths: {[str(p) for p in config.tv_paths]}")
_log("INFO", f"Movie paths: {[str(p) for p in config.movie_paths]}")
if config.db_type == "postgresql":
_log("INFO", f"Database: PostgreSQL at {config.db_host}:{config.db_port}/{config.db_name}")
_log("INFO", f"Database user: {config.db_user}")
else:
_log("INFO", f"Database: {config.db_path}")
_log("INFO", f"Database: {config.db_path}")
_log("INFO", f"Config: manage_nfo={config.manage_nfo}, fix_mtimes={config.fix_dir_mtimes}")
_log("INFO", f"Movie priority: {config.movie_priority}")
@@ -187,52 +139,21 @@ def main():
# Initialize components
dependencies = initialize_components()
# Note: Authentication and web interface handled by separate nfoguard-web container
_log("INFO", "Core API: Authentication handled by separate web container")
# Store dependencies globally for signal handler access
signal_handler.dependencies = dependencies
# Register routes
register_routes(app, dependencies)
try:
# Core API configuration (webhooks, processing, database management)
core_host = config.core_api_host if hasattr(config, 'core_api_host') else "0.0.0.0"
core_port = config.core_api_port if hasattr(config, 'core_api_port') else 8080
_log("INFO", f"🚀 Starting NFOGuard Core API on {core_host}:{core_port}")
uvicorn.run(
app,
host=core_host,
port=core_port,
reload=False,
access_log=False, # Reduce logging overhead
server_header=False, # Reduce response overhead
timeout_graceful_shutdown=15 # Give more time for graceful shutdown
host="0.0.0.0",
port=int(os.environ.get("PORT", "8080")),
reload=False
)
except KeyboardInterrupt:
_log("INFO", "NFOGuard stopped by user")
except Exception as e:
_log("ERROR", f"NFOGuard crashed: {e}")
sys.exit(1)
finally:
# Ensure cleanup happens even if uvicorn doesn't trigger signal handler
if hasattr(signal_handler, 'dependencies') and signal_handler.dependencies:
deps = signal_handler.dependencies
if 'batcher' in deps:
try:
deps['batcher'].shutdown()
except Exception:
pass
if 'db' in deps:
try:
deps['db'].close()
except Exception:
pass
if __name__ == "__main__":
monitoring/__init__.py
View File
monitoring/health.py
-474
View File
@@ -1,474 +0,0 @@
"""
Health Check System for NFOGuard
Provides health and readiness endpoints for monitoring and orchestration
"""
import time
import asyncio
from typing import Dict, Any, List, Optional
from dataclasses import dataclass
from enum import Enum
from pathlib import Path
from config.runtime_validator import RuntimeValidator, HealthCheckResult
from monitoring.metrics import metrics
class HealthStatus(Enum):
"""Health check status levels"""
HEALTHY = "healthy"
DEGRADED = "degraded"
UNHEALTHY = "unhealthy"
@dataclass
class HealthCheck:
"""Individual health check result"""
name: str
status: HealthStatus
message: str
duration_ms: float
details: Dict[str, Any] = None
def to_dict(self) -> Dict[str, Any]:
return {
"name": self.name,
"status": self.status.value,
"message": self.message,
"duration_ms": round(self.duration_ms, 2),
"details": self.details or {}
}
@dataclass
class OverallHealth:
"""Overall system health status"""
status: HealthStatus
checks: List[HealthCheck]
timestamp: float
uptime_seconds: float
version: str = "2.0.0"
def to_dict(self) -> Dict[str, Any]:
return {
"status": self.status.value,
"timestamp": self.timestamp,
"uptime_seconds": round(self.uptime_seconds, 2),
"version": self.version,
"checks": [check.to_dict() for check in self.checks],
"summary": {
"total_checks": len(self.checks),
"healthy_checks": len([c for c in self.checks if c.status == HealthStatus.HEALTHY]),
"degraded_checks": len([c for c in self.checks if c.status == HealthStatus.DEGRADED]),
"unhealthy_checks": len([c for c in self.checks if c.status == HealthStatus.UNHEALTHY])
}
}
class HealthChecker:
"""Comprehensive health checking system"""
def __init__(self):
self.start_time = time.time()
self._last_health_check = None
self._health_check_cache_ttl = 30 # Cache for 30 seconds
self._runtime_validator = None
def _get_runtime_validator(self):
"""Get runtime validator instance"""
if self._runtime_validator is None:
try:
from config.settings import config
self._runtime_validator = RuntimeValidator(config)
except Exception as e:
# Create a dummy validator if config fails
self._runtime_validator = None
return self._runtime_validator
async def check_basic_health(self) -> HealthCheck:
"""Basic health check - always succeeds if service is running"""
start_time = time.time()
try:
# Basic service availability
uptime = time.time() - self.start_time
if uptime < 30:
status = HealthStatus.DEGRADED
message = f"Service starting up (uptime: {uptime:.1f}s)"
else:
status = HealthStatus.HEALTHY
message = f"Service running normally (uptime: {uptime:.1f}s)"
return HealthCheck(
name="basic",
status=status,
message=message,
duration_ms=(time.time() - start_time) * 1000,
details={"uptime_seconds": uptime}
)
except Exception as e:
return HealthCheck(
name="basic",
status=HealthStatus.UNHEALTHY,
message=f"Basic health check failed: {e}",
duration_ms=(time.time() - start_time) * 1000
)
async def check_filesystem_health(self) -> HealthCheck:
"""Check filesystem access for media paths"""
start_time = time.time()
try:
from config.settings import config
accessible_paths = 0
total_paths = len(config.tv_paths) + len(config.movie_paths)
issues = []
# Check TV paths
for path in config.tv_paths:
try:
if path.exists() and path.is_dir():
# Try to read directory
list(path.iterdir())
accessible_paths += 1
else:
issues.append(f"TV path not accessible: {path}")
except PermissionError:
issues.append(f"TV path permission denied: {path}")
except Exception as e:
issues.append(f"TV path error {path}: {e}")
# Check movie paths
for path in config.movie_paths:
try:
if path.exists() and path.is_dir():
list(path.iterdir())
accessible_paths += 1
else:
issues.append(f"Movie path not accessible: {path}")
except PermissionError:
issues.append(f"Movie path permission denied: {path}")
except Exception as e:
issues.append(f"Movie path error {path}: {e}")
# Determine status
if accessible_paths == total_paths:
status = HealthStatus.HEALTHY
message = f"All {total_paths} media paths accessible"
elif accessible_paths > 0:
status = HealthStatus.DEGRADED
message = f"{accessible_paths}/{total_paths} media paths accessible"
else:
status = HealthStatus.UNHEALTHY
message = "No media paths accessible"
return HealthCheck(
name="filesystem",
status=status,
message=message,
duration_ms=(time.time() - start_time) * 1000,
details={
"accessible_paths": accessible_paths,
"total_paths": total_paths,
"issues": issues[:5] # Limit to first 5 issues
}
)
except Exception as e:
return HealthCheck(
name="filesystem",
status=HealthStatus.UNHEALTHY,
message=f"Filesystem check failed: {e}",
duration_ms=(time.time() - start_time) * 1000
)
async def check_database_health(self) -> HealthCheck:
"""Check database connectivity and performance"""
start_time = time.time()
try:
import sqlite3
from config.settings import config
# Test local database
db_path = config.db_path
def test_db():
with sqlite3.connect(str(db_path), timeout=5) as conn:
cursor = conn.cursor()
cursor.execute("SELECT COUNT(*) FROM sqlite_master WHERE type='table'")
table_count = cursor.fetchone()[0]
return table_count
# Run database test
table_count = await asyncio.get_event_loop().run_in_executor(None, test_db)
duration = (time.time() - start_time) * 1000
if duration < 100: # < 100ms is good
status = HealthStatus.HEALTHY
message = f"Database responsive ({duration:.1f}ms, {table_count} tables)"
elif duration < 1000: # < 1s is acceptable
status = HealthStatus.DEGRADED
message = f"Database slow ({duration:.1f}ms, {table_count} tables)"
else:
status = HealthStatus.UNHEALTHY
message = f"Database very slow ({duration:.1f}ms)"
return HealthCheck(
name="database",
status=status,
message=message,
duration_ms=duration,
details={
"db_path": str(db_path),
"table_count": table_count,
"response_time_category": "fast" if duration < 100 else "slow" if duration < 1000 else "very_slow"
}
)
except Exception as e:
return HealthCheck(
name="database",
status=HealthStatus.UNHEALTHY,
message=f"Database check failed: {e}",
duration_ms=(time.time() - start_time) * 1000,
details={"error": str(e)}
)
async def check_external_apis_health(self) -> HealthCheck:
"""Check external API connectivity"""
start_time = time.time()
try:
import aiohttp
from config.settings import config
api_results = []
apis_tested = 0
apis_healthy = 0
timeout = aiohttp.ClientTimeout(total=5)
async with aiohttp.ClientSession(timeout=timeout) as session:
# Test Radarr API if configured
if hasattr(config, 'radarr_url') and config.radarr_url:
apis_tested += 1
try:
test_url = f"{config.radarr_url.rstrip('/')}/api/v3/health"
async with session.get(test_url) as response:
if response.status == 200:
apis_healthy += 1
api_results.append({"api": "radarr", "status": "healthy"})
else:
api_results.append({"api": "radarr", "status": f"unhealthy (HTTP {response.status})"})
except Exception as e:
api_results.append({"api": "radarr", "status": f"error: {str(e)[:50]}"})
# Test Sonarr API if configured
if hasattr(config, 'sonarr_url') and config.sonarr_url:
apis_tested += 1
try:
test_url = f"{config.sonarr_url.rstrip('/')}/api/v3/health"
async with session.get(test_url) as response:
if response.status == 200:
apis_healthy += 1
api_results.append({"api": "sonarr", "status": "healthy"})
else:
api_results.append({"api": "sonarr", "status": f"unhealthy (HTTP {response.status})"})
except Exception as e:
api_results.append({"api": "sonarr", "status": f"error: {str(e)[:50]}"})
# Determine overall API health
if apis_tested == 0:
status = HealthStatus.HEALTHY
message = "No external APIs configured"
elif apis_healthy == apis_tested:
status = HealthStatus.HEALTHY
message = f"All {apis_tested} external APIs healthy"
elif apis_healthy > 0:
status = HealthStatus.DEGRADED
message = f"{apis_healthy}/{apis_tested} external APIs healthy"
else:
status = HealthStatus.UNHEALTHY
message = "No external APIs responding"
return HealthCheck(
name="external_apis",
status=status,
message=message,
duration_ms=(time.time() - start_time) * 1000,
details={
"apis_tested": apis_tested,
"apis_healthy": apis_healthy,
"api_results": api_results
}
)
except Exception as e:
return HealthCheck(
name="external_apis",
status=HealthStatus.UNHEALTHY,
message=f"API health check failed: {e}",
duration_ms=(time.time() - start_time) * 1000
)
async def check_performance_health(self) -> HealthCheck:
"""Check system performance metrics"""
start_time = time.time()
try:
system_metrics = metrics.get_system_metrics()
processing_metrics = metrics.get_processing_metrics()
issues = []
warnings = []
# Check CPU usage
cpu_percent = system_metrics.get("cpu_percent", 0)
if cpu_percent > 90:
issues.append(f"High CPU usage: {cpu_percent:.1f}%")
elif cpu_percent > 70:
warnings.append(f"Elevated CPU usage: {cpu_percent:.1f}%")
# Check memory usage
memory_percent = system_metrics.get("memory_percent", 0)
if memory_percent > 90:
issues.append(f"High memory usage: {memory_percent:.1f}%")
elif memory_percent > 80:
warnings.append(f"Elevated memory usage: {memory_percent:.1f}%")
# Check disk space
if "db_disk_free" in system_metrics and system_metrics["db_disk_free"]:
free_space_gb = system_metrics["db_disk_free"] / (1024**3)
if free_space_gb < 1:
issues.append(f"Low disk space: {free_space_gb:.1f}GB free")
elif free_space_gb < 5:
warnings.append(f"Low disk space: {free_space_gb:.1f}GB free")
# Check active operations
active_ops = system_metrics.get("active_operations", 0)
if active_ops > 10:
warnings.append(f"High concurrent operations: {active_ops}")
# Determine status
if issues:
status = HealthStatus.UNHEALTHY
message = f"Performance issues detected: {', '.join(issues[:2])}"
elif warnings:
status = HealthStatus.DEGRADED
message = f"Performance warnings: {', '.join(warnings[:2])}"
else:
status = HealthStatus.HEALTHY
message = "System performance normal"
return HealthCheck(
name="performance",
status=status,
message=message,
duration_ms=(time.time() - start_time) * 1000,
details={
"cpu_percent": cpu_percent,
"memory_percent": memory_percent,
"active_operations": active_ops,
"issues": issues,
"warnings": warnings
}
)
except Exception as e:
return HealthCheck(
name="performance",
status=HealthStatus.DEGRADED,
message=f"Performance check failed: {e}",
duration_ms=(time.time() - start_time) * 1000
)
async def get_full_health_status(self) -> OverallHealth:
"""Get comprehensive health status"""
start_time = time.time()
# Run all health checks concurrently
checks = await asyncio.gather(
self.check_basic_health(),
self.check_filesystem_health(),
self.check_database_health(),
self.check_external_apis_health(),
self.check_performance_health(),
return_exceptions=True
)
# Filter out any exceptions and convert to HealthCheck objects
valid_checks = []
for check in checks:
if isinstance(check, HealthCheck):
valid_checks.append(check)
elif isinstance(check, Exception):
valid_checks.append(HealthCheck(
name="unknown",
status=HealthStatus.UNHEALTHY,
message=f"Health check exception: {check}",
duration_ms=0
))
# Determine overall status
unhealthy_count = len([c for c in valid_checks if c.status == HealthStatus.UNHEALTHY])
degraded_count = len([c for c in valid_checks if c.status == HealthStatus.DEGRADED])
if unhealthy_count > 0:
overall_status = HealthStatus.UNHEALTHY
elif degraded_count > 0:
overall_status = HealthStatus.DEGRADED
else:
overall_status = HealthStatus.HEALTHY
return OverallHealth(
status=overall_status,
checks=valid_checks,
timestamp=start_time,
uptime_seconds=time.time() - self.start_time
)
async def get_readiness_status(self) -> Dict[str, Any]:
"""Get readiness status for Kubernetes readiness probes"""
# Readiness is simpler - just check critical components
checks = await asyncio.gather(
self.check_basic_health(),
self.check_filesystem_health(),
self.check_database_health(),
return_exceptions=True
)
critical_failures = 0
for check in checks:
if isinstance(check, HealthCheck) and check.status == HealthStatus.UNHEALTHY:
critical_failures += 1
is_ready = critical_failures == 0
return {
"ready": is_ready,
"timestamp": time.time(),
"critical_failures": critical_failures,
"message": "Service ready" if is_ready else f"{critical_failures} critical failures"
}
async def get_liveness_status(self) -> Dict[str, Any]:
"""Get liveness status for Kubernetes liveness probes"""
# Liveness is even simpler - just check if service is responsive
basic_check = await self.check_basic_health()
is_alive = basic_check.status != HealthStatus.UNHEALTHY
return {
"alive": is_alive,
"timestamp": time.time(),
"uptime_seconds": time.time() - self.start_time,
"message": basic_check.message
}
# Global health checker instance
health_checker = HealthChecker()
monitoring/logging_enhanced.py
-404
View File
@@ -1,404 +0,0 @@
"""
Enhanced Logging System for NFOGuard
Provides structured logging with correlation IDs, request tracing, and monitoring integration
"""
import logging
import json
import time
import uuid
import threading
from typing import Dict, Any, Optional, List, Union
from dataclasses import dataclass, field
from contextlib import contextmanager
from datetime import datetime
import sys
import traceback
from monitoring.metrics import metrics
# Thread-local storage for correlation context
_context = threading.local()
@dataclass
class LogContext:
"""Logging context with correlation and tracing information"""
correlation_id: str
request_id: Optional[str] = None
user_id: Optional[str] = None
operation: Optional[str] = None
media_type: Optional[str] = None
media_title: Optional[str] = None
webhook_type: Optional[str] = None
processing_stage: Optional[str] = None
additional_fields: Dict[str, Any] = field(default_factory=dict)
def to_dict(self) -> Dict[str, Any]:
"""Convert context to dictionary for logging"""
context = {
"correlation_id": self.correlation_id,
"timestamp": datetime.utcnow().isoformat(),
}
# Add non-None fields
for field_name in ["request_id", "user_id", "operation", "media_type",
"media_title", "webhook_type", "processing_stage"]:
value = getattr(self, field_name)
if value is not None:
context[field_name] = value
# Add additional fields
context.update(self.additional_fields)
return context
class StructuredFormatter(logging.Formatter):
"""JSON formatter for structured logging"""
def __init__(self, include_context: bool = True):
super().__init__()
self.include_context = include_context
def format(self, record: logging.LogRecord) -> str:
# Base log entry
log_entry = {
"timestamp": datetime.utcnow().isoformat(),
"level": record.levelname,
"logger": record.name,
"message": record.getMessage(),
"module": record.module,
"function": record.funcName,
"line": record.lineno,
}
# Add thread information
log_entry["thread"] = {
"id": record.thread,
"name": record.threadName
}
# Add correlation context if available
if self.include_context and hasattr(_context, 'log_context'):
log_entry["context"] = _context.log_context.to_dict()
# Add exception information if present
if record.exc_info:
log_entry["exception"] = {
"type": record.exc_info[0].__name__,
"message": str(record.exc_info[1]),
"traceback": traceback.format_exception(*record.exc_info)
}
# Add any extra fields passed to log call
if hasattr(record, 'extra_fields') and record.extra_fields:
log_entry["extra"] = record.extra_fields
# Add performance metrics if available
if hasattr(record, 'performance_data') and record.performance_data:
log_entry["performance"] = record.performance_data
return json.dumps(log_entry, default=str, ensure_ascii=False)
class CorrelationIDFilter(logging.Filter):
"""Filter to add correlation ID to log records"""
def filter(self, record: logging.LogRecord) -> bool:
# Add correlation ID to record if available
if hasattr(_context, 'log_context'):
record.correlation_id = _context.log_context.correlation_id
else:
record.correlation_id = "no-correlation"
return True
class EnhancedLogger:
"""Enhanced logger with correlation IDs and structured logging"""
def __init__(self, name: str):
self.logger = logging.getLogger(name)
self.name = name
# Track log events for metrics
self._log_counts = {"debug": 0, "info": 0, "warning": 0, "error": 0, "critical": 0}
def _log_with_context(self, level: int, message: str, extra_fields: Optional[Dict[str, Any]] = None,
performance_data: Optional[Dict[str, Any]] = None, **kwargs):
"""Log with enhanced context and metrics tracking"""
# Track log counts for metrics
level_name = logging.getLevelName(level).lower()
if level_name in self._log_counts:
self._log_counts[level_name] += 1
metrics.increment_counter(f"log_messages", 1, {"level": level_name, "logger": self.name})
# Create log record with extra data
extra = {}
if extra_fields:
extra['extra_fields'] = extra_fields
if performance_data:
extra['performance_data'] = performance_data
# Log the message
self.logger.log(level, message, extra=extra, **kwargs)
# Track errors in metrics
if level >= logging.ERROR:
metrics.record_error("logging_error", message, self.name)
def debug(self, message: str, **kwargs):
"""Log debug message"""
self._log_with_context(logging.DEBUG, message, **kwargs)
def info(self, message: str, **kwargs):
"""Log info message"""
self._log_with_context(logging.INFO, message, **kwargs)
def warning(self, message: str, **kwargs):
"""Log warning message"""
self._log_with_context(logging.WARNING, message, **kwargs)
def error(self, message: str, **kwargs):
"""Log error message"""
self._log_with_context(logging.ERROR, message, **kwargs)
def critical(self, message: str, **kwargs):
"""Log critical message"""
self._log_with_context(logging.CRITICAL, message, **kwargs)
def exception(self, message: str, **kwargs):
"""Log exception with traceback"""
kwargs['exc_info'] = True
self._log_with_context(logging.ERROR, message, **kwargs)
def log_operation_start(self, operation: str, **context_fields):
"""Log the start of an operation"""
self.info(f"Starting operation: {operation}",
extra_fields={"operation_event": "start", "operation": operation, **context_fields})
def log_operation_end(self, operation: str, success: bool = True, duration: Optional[float] = None, **context_fields):
"""Log the end of an operation"""
outcome = "success" if success else "failure"
extra = {"operation_event": "end", "operation": operation, "outcome": outcome, **context_fields}
if duration is not None:
extra["duration_seconds"] = duration
level = logging.INFO if success else logging.ERROR
self._log_with_context(level, f"Operation {outcome}: {operation}", extra_fields=extra)
def log_webhook_received(self, webhook_type: str, payload_size: int, **context_fields):
"""Log webhook reception"""
self.info(f"Webhook received: {webhook_type}",
extra_fields={
"event_type": "webhook_received",
"webhook_type": webhook_type,
"payload_size_bytes": payload_size,
**context_fields
})
def log_nfo_operation(self, operation: str, file_path: str, success: bool = True, **context_fields):
"""Log NFO file operations"""
outcome = "success" if success else "failure"
level = logging.INFO if success else logging.ERROR
self._log_with_context(level, f"NFO {operation} {outcome}: {file_path}",
extra_fields={
"event_type": "nfo_operation",
"nfo_operation": operation,
"file_path": file_path,
"outcome": outcome,
**context_fields
})
def log_performance_metrics(self, operation: str, duration: float, success: bool = True, **metrics_data):
"""Log performance metrics"""
self.debug(f"Performance: {operation} took {duration:.3f}s",
performance_data={
"operation": operation,
"duration_seconds": duration,
"success": success,
**metrics_data
})
def get_log_stats(self) -> Dict[str, int]:
"""Get logging statistics"""
return self._log_counts.copy()
def setup_enhanced_logging(
log_level: str = "INFO",
structured: bool = True,
log_file: Optional[str] = None,
max_bytes: int = 10 * 1024 * 1024, # 10MB
backup_count: int = 5
) -> None:
"""Setup enhanced logging configuration"""
# Configure root logger
root_logger = logging.getLogger()
root_logger.setLevel(getattr(logging, log_level.upper()))
# Clear existing handlers
root_logger.handlers.clear()
# Create console handler
console_handler = logging.StreamHandler(sys.stdout)
if structured:
# Use structured JSON formatter
formatter = StructuredFormatter(include_context=True)
else:
# Use simple text formatter with correlation ID
formatter = logging.Formatter(
'%(asctime)s [%(correlation_id)s] %(levelname)s %(name)s: %(message)s'
)
console_handler.setFormatter(formatter)
console_handler.addFilter(CorrelationIDFilter())
root_logger.addHandler(console_handler)
# Add file handler if specified
if log_file:
from logging.handlers import RotatingFileHandler
file_handler = RotatingFileHandler(
log_file, maxBytes=max_bytes, backupCount=backup_count
)
file_handler.setFormatter(formatter)
file_handler.addFilter(CorrelationIDFilter())
root_logger.addHandler(file_handler)
# Reduce noise from external libraries
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("aiohttp").setLevel(logging.WARNING)
def get_enhanced_logger(name: str) -> EnhancedLogger:
"""Get enhanced logger instance"""
return EnhancedLogger(name)
def set_log_context(
correlation_id: Optional[str] = None,
request_id: Optional[str] = None,
operation: Optional[str] = None,
**kwargs
) -> LogContext:
"""Set logging context for current thread"""
if correlation_id is None:
correlation_id = str(uuid.uuid4())
context = LogContext(
correlation_id=correlation_id,
request_id=request_id,
operation=operation,
**kwargs
)
_context.log_context = context
return context
def get_log_context() -> Optional[LogContext]:
"""Get current logging context"""
return getattr(_context, 'log_context', None)
def clear_log_context():
"""Clear logging context for current thread"""
if hasattr(_context, 'log_context'):
delattr(_context, 'log_context')
@contextmanager
def log_context(correlation_id: Optional[str] = None, **context_fields):
"""Context manager for scoped logging context"""
original_context = get_log_context()
try:
# Set new context
new_context = set_log_context(correlation_id=correlation_id, **context_fields)
yield new_context
finally:
# Restore original context
if original_context:
_context.log_context = original_context
else:
clear_log_context()
@contextmanager
def log_operation(operation: str, logger: Optional[EnhancedLogger] = None, **context_fields):
"""Context manager for logging operation start/end with timing"""
if logger is None:
logger = get_enhanced_logger(__name__)
start_time = time.time()
success = True
# Update context with operation
current_context = get_log_context()
if current_context:
current_context.operation = operation
current_context.processing_stage = "executing"
logger.log_operation_start(operation, **context_fields)
try:
yield
except Exception as e:
success = False
logger.exception(f"Operation failed: {operation}",
extra_fields={"operation": operation, "error": str(e), **context_fields})
raise
finally:
duration = time.time() - start_time
logger.log_operation_end(operation, success, duration, **context_fields)
# Update metrics
metrics.record_operation_duration(operation, duration, success)
def trace_request(request_id: Optional[str] = None, **context_fields):
"""Decorator/context manager for request tracing"""
def decorator(func):
def wrapper(*args, **kwargs):
correlation_id = str(uuid.uuid4())
req_id = request_id or f"req_{int(time.time())}"
with log_context(correlation_id=correlation_id, request_id=req_id, **context_fields):
return func(*args, **kwargs)
return wrapper
# Can be used as context manager or decorator
if request_id is None and len(context_fields) == 1 and callable(list(context_fields.values())[0]):
# Used as decorator without parentheses
func = list(context_fields.values())[0]
return decorator(func)
else:
# Used as decorator with parameters or context manager
return decorator
# Module-level logger for this module
logger = get_enhanced_logger(__name__)
def get_logging_stats() -> Dict[str, Any]:
"""Get comprehensive logging statistics"""
# Collect stats from all enhanced loggers
total_stats = {"debug": 0, "info": 0, "warning": 0, "error": 0, "critical": 0}
# This is a simplified version - in practice you'd track all logger instances
return {
"total_log_messages": sum(total_stats.values()),
"by_level": total_stats,
"structured_logging_enabled": True,
"correlation_tracking_enabled": True
}
monitoring/metrics.py
-354
View File
@@ -1,354 +0,0 @@
"""
Metrics Collection System for NFOGuard
Provides performance monitoring, counters, and operational metrics
"""
import time
import psutil
import threading
from datetime import datetime, timedelta
from typing import Dict, Any, List, Optional
from dataclasses import dataclass, field
from collections import defaultdict, deque
from contextlib import contextmanager
import asyncio
@dataclass
class MetricValue:
"""Individual metric value with timestamp"""
value: float
timestamp: float = field(default_factory=time.time)
labels: Dict[str, str] = field(default_factory=dict)
@dataclass
class TimeSeriesMetric:
"""Time series metric with historical data"""
name: str
values: deque = field(default_factory=lambda: deque(maxlen=1000))
total: float = 0.0
count: int = 0
def add_value(self, value: float, labels: Optional[Dict[str, str]] = None):
"""Add a new metric value"""
metric_value = MetricValue(value, labels=labels or {})
self.values.append(metric_value)
self.total += value
self.count += 1
def get_average(self, window_seconds: int = 300) -> float:
"""Get average value over time window"""
cutoff_time = time.time() - window_seconds
recent_values = [v.value for v in self.values if v.timestamp > cutoff_time]
return sum(recent_values) / len(recent_values) if recent_values else 0.0
def get_rate_per_minute(self, window_seconds: int = 300) -> float:
"""Get rate per minute over time window"""
cutoff_time = time.time() - window_seconds
recent_count = len([v for v in self.values if v.timestamp > cutoff_time])
return (recent_count / window_seconds) * 60 if window_seconds > 0 else 0.0
class MetricsCollector:
"""Central metrics collection system"""
def __init__(self):
self._metrics: Dict[str, TimeSeriesMetric] = {}
self._counters: Dict[str, int] = defaultdict(int)
self._gauges: Dict[str, float] = {}
self._histograms: Dict[str, List[float]] = defaultdict(list)
self._start_time = time.time()
self._lock = threading.RLock()
# Processing metrics
self._active_operations = 0
self._operation_durations = deque(maxlen=1000)
# Error tracking
self._error_counts = defaultdict(int)
self._last_errors = deque(maxlen=100)
# System metrics
self._system_stats_cache = {}
self._system_stats_last_update = 0
self._system_stats_cache_ttl = 30 # 30 seconds
def increment_counter(self, name: str, value: int = 1, labels: Optional[Dict[str, str]] = None):
"""Increment a counter metric"""
with self._lock:
full_name = self._build_metric_name(name, labels)
self._counters[full_name] += value
# Also track in time series for rate calculations
if name not in self._metrics:
self._metrics[name] = TimeSeriesMetric(name)
self._metrics[name].add_value(value, labels)
def set_gauge(self, name: str, value: float, labels: Optional[Dict[str, str]] = None):
"""Set a gauge metric value"""
with self._lock:
full_name = self._build_metric_name(name, labels)
self._gauges[full_name] = value
def record_histogram(self, name: str, value: float, labels: Optional[Dict[str, str]] = None):
"""Record a histogram value"""
with self._lock:
full_name = self._build_metric_name(name, labels)
self._histograms[full_name].append(value)
# Keep only recent values (last 1000)
if len(self._histograms[full_name]) > 1000:
self._histograms[full_name] = self._histograms[full_name][-1000:]
# Also track in time series
if name not in self._metrics:
self._metrics[name] = TimeSeriesMetric(name)
self._metrics[name].add_value(value, labels)
def record_operation_duration(self, operation: str, duration: float, success: bool = True):
"""Record operation duration and outcome"""
with self._lock:
# Record duration
self.record_histogram(f"operation_duration_{operation}", duration)
# Record outcome
outcome = "success" if success else "error"
self.increment_counter(f"operation_total", 1, {"operation": operation, "outcome": outcome})
# Track active operations
if operation.endswith("_start"):
self._active_operations += 1
elif operation.endswith("_end"):
self._active_operations = max(0, self._active_operations - 1)
def record_error(self, error_type: str, error_message: str, operation: Optional[str] = None):
"""Record an error occurrence"""
with self._lock:
self._error_counts[error_type] += 1
error_info = {
"type": error_type,
"message": error_message,
"operation": operation,
"timestamp": time.time()
}
self._last_errors.append(error_info)
# Increment error counter
labels = {"error_type": error_type}
if operation:
labels["operation"] = operation
self.increment_counter("errors_total", 1, labels)
@contextmanager
def operation_timer(self, operation: str):
"""Context manager for timing operations"""
start_time = time.time()
success = True
try:
self.record_operation_duration(f"{operation}_start", 0)
yield
except Exception as e:
success = False
self.record_error("operation_error", str(e), operation)
raise
finally:
duration = time.time() - start_time
self.record_operation_duration(operation, duration, success)
self.record_operation_duration(f"{operation}_end", 0)
def get_system_metrics(self) -> Dict[str, Any]:
"""Get current system resource metrics"""
now = time.time()
# Use cached values if recent
if (now - self._system_stats_last_update) < self._system_stats_cache_ttl:
return self._system_stats_cache
try:
# CPU metrics
cpu_percent = psutil.cpu_percent(interval=0.1)
cpu_count = psutil.cpu_count()
# Memory metrics
memory = psutil.virtual_memory()
# Disk metrics for database path
try:
from config.settings import config
db_disk = psutil.disk_usage(str(config.db_path.parent))
except:
db_disk = None
# Process metrics
process = psutil.Process()
process_memory = process.memory_info()
self._system_stats_cache = {
"cpu_percent": cpu_percent,
"cpu_count": cpu_count,
"memory_total": memory.total,
"memory_available": memory.available,
"memory_percent": memory.percent,
"process_memory_rss": process_memory.rss,
"process_memory_vms": process_memory.vms,
"db_disk_free": db_disk.free if db_disk else None,
"db_disk_total": db_disk.total if db_disk else None,
"active_operations": self._active_operations,
"uptime_seconds": now - self._start_time
}
self._system_stats_last_update = now
except Exception as e:
# Return basic metrics if detailed collection fails
self._system_stats_cache = {
"uptime_seconds": now - self._start_time,
"active_operations": self._active_operations,
"error": str(e)
}
return self._system_stats_cache
def get_processing_metrics(self) -> Dict[str, Any]:
"""Get processing-related metrics"""
with self._lock:
# Calculate rates and averages
webhook_rate = self._metrics.get("webhooks_received", TimeSeriesMetric("webhooks_received")).get_rate_per_minute()
nfo_rate = self._metrics.get("nfo_created", TimeSeriesMetric("nfo_created")).get_rate_per_minute()
avg_processing_time = 0.0
if "processing_duration" in self._metrics:
avg_processing_time = self._metrics["processing_duration"].get_average()
return {
"webhooks_received_per_minute": webhook_rate,
"nfo_files_created_per_minute": nfo_rate,
"average_processing_time_seconds": avg_processing_time,
"active_operations": self._active_operations,
"total_webhooks": self._counters.get("webhooks_received", 0),
"total_nfo_created": self._counters.get("nfo_created", 0),
"total_errors": sum(self._error_counts.values())
}
def get_error_metrics(self) -> Dict[str, Any]:
"""Get error-related metrics"""
with self._lock:
recent_errors = []
cutoff_time = time.time() - 3600 # Last hour
for error in self._last_errors:
if error["timestamp"] > cutoff_time:
recent_errors.append({
"type": error["type"],
"message": error["message"][:100], # Truncate long messages
"operation": error["operation"],
"timestamp": error["timestamp"]
})
return {
"error_counts_by_type": dict(self._error_counts),
"recent_errors": recent_errors[-10:], # Last 10 errors
"total_errors": sum(self._error_counts.values()),
"error_rate_per_minute": len([e for e in self._last_errors if e["timestamp"] > time.time() - 300]) / 5
}
def get_prometheus_metrics(self) -> str:
"""Generate Prometheus-compatible metrics format"""
lines = []
# Add help and type information
lines.append("# HELP nfoguard_webhooks_total Total number of webhooks received")
lines.append("# TYPE nfoguard_webhooks_total counter")
with self._lock:
# Counters
for name, value in self._counters.items():
metric_name = f"nfoguard_{name.replace('-', '_')}"
lines.append(f"{metric_name} {value}")
# Gauges
lines.append("# HELP nfoguard_active_operations Current number of active operations")
lines.append("# TYPE nfoguard_active_operations gauge")
lines.append(f"nfoguard_active_operations {self._active_operations}")
# System metrics
system_metrics = self.get_system_metrics()
for key, value in system_metrics.items():
if isinstance(value, (int, float)) and value is not None:
metric_name = f"nfoguard_system_{key}"
lines.append(f"{metric_name} {value}")
return "\n".join(lines)
def get_all_metrics(self) -> Dict[str, Any]:
"""Get all metrics in a structured format"""
return {
"system": self.get_system_metrics(),
"processing": self.get_processing_metrics(),
"errors": self.get_error_metrics(),
"timestamp": time.time(),
"uptime_seconds": time.time() - self._start_time
}
def reset_metrics(self, metric_types: Optional[List[str]] = None):
"""Reset specific metric types or all metrics"""
with self._lock:
if not metric_types or "counters" in metric_types:
self._counters.clear()
if not metric_types or "histograms" in metric_types:
self._histograms.clear()
if not metric_types or "errors" in metric_types:
self._error_counts.clear()
self._last_errors.clear()
if not metric_types or "timeseries" in metric_types:
self._metrics.clear()
def _build_metric_name(self, name: str, labels: Optional[Dict[str, str]]) -> str:
"""Build metric name with labels"""
if not labels:
return name
label_str = ",".join(f"{k}={v}" for k, v in sorted(labels.items()))
return f"{name}{{{label_str}}}"
# Global metrics collector instance
metrics = MetricsCollector()
# Convenience functions for common operations
def track_webhook_received(webhook_type: str):
"""Track webhook received"""
metrics.increment_counter("webhooks_received", 1, {"type": webhook_type})
def track_nfo_created(media_type: str, success: bool = True):
"""Track NFO file creation"""
outcome = "success" if success else "error"
metrics.increment_counter("nfo_created", 1, {"media_type": media_type, "outcome": outcome})
def track_api_call(api_name: str, duration: float, success: bool = True):
"""Track external API call"""
metrics.record_histogram(f"api_call_duration", duration, {"api": api_name})
outcome = "success" if success else "error"
metrics.increment_counter("api_calls_total", 1, {"api": api_name, "outcome": outcome})
def track_database_operation(operation: str, duration: float, success: bool = True):
"""Track database operation"""
metrics.record_histogram("database_operation_duration", duration, {"operation": operation})
outcome = "success" if success else "error"
metrics.increment_counter("database_operations_total", 1, {"operation": operation, "outcome": outcome})
def track_file_operation(operation: str, duration: float, success: bool = True):
"""Track file system operation"""
metrics.record_histogram("file_operation_duration", duration, {"operation": operation})
outcome = "success" if success else "error"
metrics.increment_counter("file_operations_total", 1, {"operation": operation, "outcome": outcome})
monitoring/performance.py
-413
View File
@@ -1,413 +0,0 @@
"""
Performance Monitoring and Profiling for NFOGuard
Provides detailed performance analysis and optimization insights
"""
import time
import asyncio
import threading
import functools
from typing import Dict, Any, List, Optional, Callable, TypeVar, Union
from dataclasses import dataclass, field
from collections import defaultdict, deque
from contextlib import asynccontextmanager, contextmanager
import traceback
import sys
from monitoring.metrics import metrics
T = TypeVar('T')
@dataclass
class PerformanceProfile:
"""Performance profile for an operation"""
operation_name: str
total_calls: int = 0
total_duration: float = 0.0
min_duration: float = float('inf')
max_duration: float = 0.0
recent_durations: deque = field(default_factory=lambda: deque(maxlen=100))
error_count: int = 0
concurrent_calls: int = 0
def add_measurement(self, duration: float, success: bool = True):
"""Add a performance measurement"""
self.total_calls += 1
self.total_duration += duration
self.min_duration = min(self.min_duration, duration)
self.max_duration = max(self.max_duration, duration)
self.recent_durations.append(duration)
if not success:
self.error_count += 1
def get_average_duration(self) -> float:
"""Get average duration across all calls"""
return self.total_duration / self.total_calls if self.total_calls > 0 else 0.0
def get_recent_average(self, window: int = 50) -> float:
"""Get average of recent calls"""
recent = list(self.recent_durations)[-window:]
return sum(recent) / len(recent) if recent else 0.0
def get_percentiles(self) -> Dict[str, float]:
"""Get duration percentiles for recent calls"""
recent = sorted(list(self.recent_durations))
if not recent:
return {"p50": 0, "p95": 0, "p99": 0}
length = len(recent)
return {
"p50": recent[int(length * 0.5)] if length > 0 else 0,
"p95": recent[int(length * 0.95)] if length > 0 else 0,
"p99": recent[int(length * 0.99)] if length > 0 else 0
}
def to_dict(self) -> Dict[str, Any]:
"""Convert to dictionary for API responses"""
percentiles = self.get_percentiles()
return {
"operation_name": self.operation_name,
"total_calls": self.total_calls,
"error_count": self.error_count,
"error_rate": self.error_count / self.total_calls if self.total_calls > 0 else 0,
"concurrent_calls": self.concurrent_calls,
"duration_stats": {
"average": round(self.get_average_duration(), 4),
"recent_average": round(self.get_recent_average(), 4),
"min": round(self.min_duration if self.min_duration != float('inf') else 0, 4),
"max": round(self.max_duration, 4),
"p50": round(percentiles["p50"], 4),
"p95": round(percentiles["p95"], 4),
"p99": round(percentiles["p99"], 4)
},
"performance_rating": self._get_performance_rating()
}
def _get_performance_rating(self) -> str:
"""Get performance rating based on metrics"""
avg_duration = self.get_recent_average()
error_rate = self.error_count / self.total_calls if self.total_calls > 0 else 0
if error_rate > 0.1: # >10% error rate
return "poor"
elif avg_duration > 5.0: # >5 seconds average
return "slow"
elif avg_duration > 1.0: # >1 second average
return "acceptable"
else:
return "excellent"
class PerformanceMonitor:
"""Advanced performance monitoring system"""
def __init__(self):
self._profiles: Dict[str, PerformanceProfile] = {}
self._active_operations: Dict[str, float] = {} # operation_id -> start_time
self._lock = threading.RLock()
# Slow operation tracking
self._slow_operation_threshold = 1.0 # 1 second
self._slow_operations = deque(maxlen=100)
# Memory monitoring
self._memory_samples = deque(maxlen=1000)
self._memory_monitoring_enabled = True
# Async operation tracking
self._async_tasks = {}
self._task_counter = 0
def get_profile(self, operation_name: str) -> PerformanceProfile:
"""Get or create performance profile for operation"""
with self._lock:
if operation_name not in self._profiles:
self._profiles[operation_name] = PerformanceProfile(operation_name)
return self._profiles[operation_name]
@contextmanager
def monitor_operation(self, operation_name: str, **kwargs):
"""Context manager for monitoring synchronous operations"""
start_time = time.time()
operation_id = f"{operation_name}_{id(threading.current_thread())}_{time.time()}"
success = True
profile = self.get_profile(operation_name)
with self._lock:
profile.concurrent_calls += 1
self._active_operations[operation_id] = start_time
try:
yield
except Exception as e:
success = False
metrics.record_error("performance_monitor", str(e), operation_name)
raise
finally:
end_time = time.time()
duration = end_time - start_time
with self._lock:
profile.concurrent_calls = max(0, profile.concurrent_calls - 1)
self._active_operations.pop(operation_id, None)
# Record measurement
profile.add_measurement(duration, success)
# Track slow operations
if duration > self._slow_operation_threshold:
self._slow_operations.append({
"operation": operation_name,
"duration": duration,
"timestamp": end_time,
"success": success,
"metadata": kwargs
})
# Update metrics
metrics.record_histogram(f"operation_duration", duration, {"operation": operation_name})
if not success:
metrics.increment_counter("operation_errors", 1, {"operation": operation_name})
@asynccontextmanager
async def monitor_async_operation(self, operation_name: str, **kwargs):
"""Context manager for monitoring asynchronous operations"""
start_time = time.time()
task_id = f"{operation_name}_{self._task_counter}"
self._task_counter += 1
success = True
profile = self.get_profile(operation_name)
with self._lock:
profile.concurrent_calls += 1
self._async_tasks[task_id] = {
"operation": operation_name,
"start_time": start_time,
"metadata": kwargs
}
try:
yield
except Exception as e:
success = False
metrics.record_error("async_performance_monitor", str(e), operation_name)
raise
finally:
end_time = time.time()
duration = end_time - start_time
with self._lock:
profile.concurrent_calls = max(0, profile.concurrent_calls - 1)
self._async_tasks.pop(task_id, None)
# Record measurement
profile.add_measurement(duration, success)
# Track slow operations
if duration > self._slow_operation_threshold:
self._slow_operations.append({
"operation": operation_name,
"duration": duration,
"timestamp": end_time,
"success": success,
"async": True,
"metadata": kwargs
})
# Update metrics
metrics.record_histogram(f"async_operation_duration", duration, {"operation": operation_name})
if not success:
metrics.increment_counter("async_operation_errors", 1, {"operation": operation_name})
def monitor_function(self, operation_name: Optional[str] = None):
"""Decorator for monitoring function performance"""
def decorator(func: Callable[..., T]) -> Callable[..., T]:
name = operation_name or f"{func.__module__}.{func.__name__}"
if asyncio.iscoroutinefunction(func):
@functools.wraps(func)
async def async_wrapper(*args, **kwargs):
async with self.monitor_async_operation(name):
return await func(*args, **kwargs)
return async_wrapper
else:
@functools.wraps(func)
def sync_wrapper(*args, **kwargs):
with self.monitor_operation(name):
return func(*args, **kwargs)
return sync_wrapper
return decorator
def get_performance_summary(self) -> Dict[str, Any]:
"""Get comprehensive performance summary"""
with self._lock:
# Get top operations by various metrics
profiles = list(self._profiles.values())
# Sort by total calls
most_called = sorted(profiles, key=lambda p: p.total_calls, reverse=True)[:10]
# Sort by average duration
slowest_avg = sorted(profiles, key=lambda p: p.get_average_duration(), reverse=True)[:10]
# Sort by recent average
slowest_recent = sorted(profiles, key=lambda p: p.get_recent_average(), reverse=True)[:10]
# Sort by error rate
highest_errors = sorted(
[p for p in profiles if p.total_calls > 0],
key=lambda p: p.error_count / p.total_calls,
reverse=True
)[:10]
# Get active operations count
total_active = sum(p.concurrent_calls for p in profiles)
# Get slow operations
recent_slow = list(self._slow_operations)[-20:] # Last 20 slow operations
return {
"overview": {
"total_operations_tracked": len(profiles),
"total_active_operations": total_active,
"slow_operation_threshold_seconds": self._slow_operation_threshold,
"total_slow_operations": len(self._slow_operations)
},
"top_operations": {
"most_called": [p.to_dict() for p in most_called],
"slowest_average": [p.to_dict() for p in slowest_avg],
"slowest_recent": [p.to_dict() for p in slowest_recent],
"highest_error_rate": [p.to_dict() for p in highest_errors]
},
"recent_slow_operations": recent_slow,
"performance_insights": self._generate_performance_insights(profiles)
}
def get_operation_detail(self, operation_name: str) -> Optional[Dict[str, Any]]:
"""Get detailed performance data for specific operation"""
with self._lock:
if operation_name not in self._profiles:
return None
profile = self._profiles[operation_name]
# Get related slow operations
related_slow = [
op for op in self._slow_operations
if op["operation"] == operation_name
]
detail = profile.to_dict()
detail.update({
"detailed_stats": {
"total_duration": round(profile.total_duration, 4),
"recent_durations": list(profile.recent_durations)[-20:], # Last 20 calls
"slow_operations_count": len(related_slow),
"recent_slow_operations": related_slow[-10:] # Last 10 slow calls
},
"recommendations": self._get_operation_recommendations(profile)
})
return detail
def _generate_performance_insights(self, profiles: List[PerformanceProfile]) -> List[str]:
"""Generate performance optimization insights"""
insights = []
# Check for very slow operations
very_slow = [p for p in profiles if p.get_recent_average() > 5.0]
if very_slow:
insights.append(f"Found {len(very_slow)} operations with >5s average duration - consider optimization")
# Check for high error rates
high_error_rate = [p for p in profiles if p.total_calls > 10 and (p.error_count / p.total_calls) > 0.1]
if high_error_rate:
insights.append(f"Found {len(high_error_rate)} operations with >10% error rate - investigate failures")
# Check for high concurrency
high_concurrency = [p for p in profiles if p.concurrent_calls > 5]
if high_concurrency:
insights.append(f"Found {len(high_concurrency)} operations with high concurrency - may need rate limiting")
# Check total active operations
total_active = sum(p.concurrent_calls for p in profiles)
if total_active > 20:
insights.append(f"High total concurrent operations ({total_active}) - system may be under load")
# Performance trends
recent_slow_count = len([op for op in self._slow_operations if op["timestamp"] > time.time() - 300])
if recent_slow_count > 10:
insights.append(f"Many slow operations recently ({recent_slow_count} in last 5 minutes)")
if not insights:
insights.append("No significant performance issues detected")
return insights
def _get_operation_recommendations(self, profile: PerformanceProfile) -> List[str]:
"""Get recommendations for optimizing specific operation"""
recommendations = []
avg_duration = profile.get_recent_average()
error_rate = profile.error_count / profile.total_calls if profile.total_calls > 0 else 0
if avg_duration > 5.0:
recommendations.append("Consider breaking down this operation into smaller parts")
recommendations.append("Review database queries and file I/O for optimization opportunities")
elif avg_duration > 1.0:
recommendations.append("Monitor for potential optimization opportunities")
if error_rate > 0.1:
recommendations.append("High error rate - investigate common failure causes")
recommendations.append("Consider adding retry logic or better error handling")
if profile.concurrent_calls > 5:
recommendations.append("High concurrency - consider adding rate limiting")
recommendations.append("Review resource usage and potential bottlenecks")
percentiles = profile.get_percentiles()
if percentiles["p99"] > percentiles["p50"] * 3:
recommendations.append("High latency variance - investigate outlier causes")
if not recommendations:
recommendations.append("Performance appears optimal for this operation")
return recommendations
def set_slow_operation_threshold(self, threshold_seconds: float):
"""Set threshold for what constitutes a slow operation"""
with self._lock:
self._slow_operation_threshold = threshold_seconds
def clear_profiles(self, operation_names: Optional[List[str]] = None):
"""Clear performance profiles for specific operations or all"""
with self._lock:
if operation_names:
for name in operation_names:
self._profiles.pop(name, None)
else:
self._profiles.clear()
self._slow_operations.clear()
# Global performance monitor instance
performance_monitor = PerformanceMonitor()
# Decorator shortcuts
def monitor_performance(operation_name: Optional[str] = None):
"""Shortcut decorator for performance monitoring"""
return performance_monitor.monitor_function(operation_name)
def monitor_sync_operation(operation_name: str, **kwargs):
"""Shortcut for synchronous operation monitoring"""
return performance_monitor.monitor_operation(operation_name, **kwargs)
def monitor_async_operation(operation_name: str, **kwargs):
"""Shortcut for asynchronous operation monitoring"""
return performance_monitor.monitor_async_operation(operation_name, **kwargs)
nfoguard-web/__init__.py
-1
View File
@@ -1 +0,0 @@
# NFOGuard Web Interface Package
nfoguard-web/api/__init__.py
-1
View File
@@ -1 +0,0 @@
# NFOGuard Web API Package
nfoguard-web/api/auth.py
-182
View File
@@ -1,182 +0,0 @@
"""
Simple authentication middleware for NFOGuard web interface
Provides basic HTTP auth and session management for web interface protection
"""
import secrets
import hashlib
from datetime import datetime, timedelta
from typing import Optional, Dict, Any
from fastapi import HTTPException, status, Request, Response
from fastapi.security import HTTPBasic, HTTPBasicCredentials
from starlette.middleware.base import BaseHTTPMiddleware
class AuthSession:
"""Simple session management for web interface"""
def __init__(self, timeout_seconds: int = 3600):
self.sessions: Dict[str, Dict[str, Any]] = {}
self.timeout_seconds = timeout_seconds
def create_session(self, username: str) -> str:
"""Create a new session and return session token"""
session_token = secrets.token_urlsafe(32)
self.sessions[session_token] = {
"username": username,
"created_at": datetime.utcnow(),
"last_activity": datetime.utcnow()
}
return session_token
def validate_session(self, session_token: str) -> bool:
"""Validate session token and update last activity"""
if not session_token or session_token not in self.sessions:
return False
session = self.sessions[session_token]
now = datetime.utcnow()
# Check if session expired
if (now - session["last_activity"]).seconds > self.timeout_seconds:
del self.sessions[session_token]
return False
# Update last activity
session["last_activity"] = now
return True
def get_session_user(self, session_token: str) -> Optional[str]:
"""Get username from valid session"""
if self.validate_session(session_token):
return self.sessions[session_token]["username"]
return None
def delete_session(self, session_token: str) -> None:
"""Delete a session (logout)"""
if session_token in self.sessions:
del self.sessions[session_token]
def cleanup_expired_sessions(self) -> None:
"""Remove expired sessions"""
now = datetime.utcnow()
expired_tokens = []
for token, session in self.sessions.items():
if (now - session["last_activity"]).seconds > self.timeout_seconds:
expired_tokens.append(token)
for token in expired_tokens:
del self.sessions[token]
class SimpleAuthMiddleware(BaseHTTPMiddleware):
"""Simple authentication middleware for web interface routes"""
def __init__(self, app, config):
super().__init__(app)
self.config = config
self.session_manager = AuthSession(config.web_auth_session_timeout)
self.security = HTTPBasic()
# Routes that require authentication (web interface)
self.protected_routes = [
"/", # Main web interface
"/static/", # Static files (CSS, JS)
"/api/movies", # Web API endpoints
"/api/series",
"/api/episodes",
"/api/dashboard"
]
# Routes that are always public (webhooks, health checks, API endpoints)
self.public_routes = [
"/webhook/",
"/health",
"/ping",
"/api/v1/health",
"/api/v1/metrics",
"/database/", # Database management endpoints (API access)
"/manual/", # Manual scan endpoints (API access)
"/debug/", # Debug endpoints (API access)
"/test/", # Test endpoints (API access)
"/bulk/" # Bulk operation endpoints (API access)
]
async def dispatch(self, request: Request, call_next):
"""Process request through authentication middleware"""
# Skip authentication if disabled
if not self.config.web_auth_enabled:
return await call_next(request)
# Check if route requires authentication
path = request.url.path
needs_auth = any(path.startswith(route) for route in self.protected_routes)
is_public = any(path.startswith(route) for route in self.public_routes)
if is_public or not needs_auth:
return await call_next(request)
# Check for existing session
session_token = request.cookies.get("nfoguard_session")
if session_token and self.session_manager.validate_session(session_token):
# Valid session, proceed
return await call_next(request)
# Check for HTTP Basic Auth
auth_header = request.headers.get("authorization")
if auth_header and auth_header.startswith("Basic "):
credentials = self._parse_basic_auth(auth_header)
if credentials and self._validate_credentials(credentials.username, credentials.password):
# Create session for successful login
session_token = self.session_manager.create_session(credentials.username)
response = await call_next(request)
response.set_cookie(
key="nfoguard_session",
value=session_token,
max_age=self.config.web_auth_session_timeout,
httponly=True,
secure=False # Set to True if using HTTPS
)
return response
# Authentication required
return self._auth_required_response()
def _parse_basic_auth(self, auth_header: str) -> Optional[HTTPBasicCredentials]:
"""Parse HTTP Basic Auth header"""
try:
import base64
encoded_credentials = auth_header.split(" ")[1]
decoded_credentials = base64.b64decode(encoded_credentials).decode('utf-8')
username, password = decoded_credentials.split(":", 1)
return HTTPBasicCredentials(username=username, password=password)
except Exception:
return None
def _validate_credentials(self, username: str, password: str) -> bool:
"""Validate username and password"""
return (username == self.config.web_auth_username and
password == self.config.web_auth_password)
def _auth_required_response(self) -> Response:
"""Return 401 response with WWW-Authenticate header"""
return Response(
content="Authentication required",
status_code=status.HTTP_401_UNAUTHORIZED,
headers={"WWW-Authenticate": "Basic realm=\"NFOGuard Web Interface\""}
)
def create_auth_dependencies(config) -> Dict[str, Any]:
"""Create authentication-related dependencies for dependency injection"""
session_manager = AuthSession(config.web_auth_session_timeout)
return {
"session_manager": session_manager,
"auth_enabled": config.web_auth_enabled,
"auth_config": {
"username": config.web_auth_username,
"timeout": config.web_auth_session_timeout
}
}
nfoguard-web/api/web_routes.py
-1292
View File
File diff suppressed because it is too large Load Diff
nfoguard-web/config/web_settings.py
-70
View File
@@ -1,70 +0,0 @@
"""
NFOGuard Web Interface Configuration
Lightweight configuration for web-only container
"""
import os
def _bool_env(name: str, default: bool = False) -> bool:
"""Convert environment variable to boolean"""
value = os.environ.get(name, "").lower()
return value in ("true", "1", "yes", "on")
class WebConfig:
"""Configuration for NFOGuard Web Interface"""
def __init__(self):
self._load_server_settings()
self._load_database_settings()
self._load_auth_settings()
self._load_ui_settings()
def _load_server_settings(self) -> None:
"""Load web server configuration"""
self.web_host = os.environ.get("WEB_HOST", "0.0.0.0")
self.web_port = int(os.environ.get("WEB_PORT", "8081"))
self.web_workers = int(os.environ.get("WEB_WORKERS", "1"))
self.web_debug = _bool_env("WEB_DEBUG", False)
# Core NFOGuard API connection (for some operations)
self.core_api_host = os.environ.get("CORE_API_HOST", "nfoguard")
self.core_api_port = int(os.environ.get("CORE_API_PORT", "8080"))
self.core_api_url = f"http://{self.core_api_host}:{self.core_api_port}"
def _load_database_settings(self) -> None:
"""Load database configuration (read-only access)"""
self.db_type = os.environ.get("DB_TYPE", "postgresql").lower()
self.db_host = os.environ.get("DB_HOST", "nfoguard-db")
self.db_port = int(os.environ.get("DB_PORT", "5432"))
self.db_name = os.environ.get("DB_NAME", "nfoguard")
self.db_user = os.environ.get("DB_USER", "nfoguard")
self.db_password = os.environ.get("DB_PASSWORD", "")
if not self.db_password:
raise ValueError("DB_PASSWORD must be set for web interface database access")
def _load_auth_settings(self) -> None:
"""Load web interface authentication settings"""
self.web_auth_enabled = _bool_env("WEB_AUTH_ENABLED", False)
self.web_auth_username = os.environ.get("WEB_AUTH_USERNAME", "admin")
self.web_auth_password = os.environ.get("WEB_AUTH_PASSWORD", "")
self.web_auth_session_timeout = int(os.environ.get("WEB_AUTH_SESSION_TIMEOUT", "3600"))
if self.web_auth_enabled and not self.web_auth_password:
raise ValueError("WEB_AUTH_PASSWORD must be set when authentication is enabled")
def _load_ui_settings(self) -> None:
"""Load UI-specific settings"""
self.app_title = os.environ.get("APP_TITLE", "NFOGuard")
self.app_subtitle = os.environ.get("APP_SUBTITLE", "Database Management & Reporting")
self.pagination_limit = int(os.environ.get("PAGINATION_LIMIT", "50"))
self.refresh_interval = int(os.environ.get("REFRESH_INTERVAL", "30")) # seconds
# Logo configuration
self.logo_enabled = _bool_env("LOGO_ENABLED", True)
self.logo_path = "/static/logo/NFOguardLogoPlain.png"
# Global config instance
web_config = WebConfig()
nfoguard-web/core/__init__.py
-1
View File
@@ -1 +0,0 @@
# NFOGuard Web Core Components
nfoguard-web/core/web_database.py
-385
View File
@@ -1,385 +0,0 @@
"""
NFOGuard Web Database - Lightweight Read-Only Database Access
Optimized for web interface queries with minimal dependencies
"""
import psycopg2
import psycopg2.extras
from typing import Dict, List, Optional, Any, Tuple
import logging
logger = logging.getLogger(__name__)
class WebDatabase:
"""Lightweight database access for web interface"""
def __init__(self, db_type: str, host: str, port: int, database: str, user: str, password: str):
self.db_type = db_type.lower()
self.host = host
self.port = port
self.database = database
self.user = user
self.password = password
self.connection = None
# Connect to database
self._connect()
def _connect(self):
"""Connect to PostgreSQL database"""
if self.db_type != "postgresql":
raise ValueError("Web interface only supports PostgreSQL")
try:
self.connection = psycopg2.connect(
host=self.host,
port=self.port,
database=self.database,
user=self.user,
password=self.password,
cursor_factory=psycopg2.extras.RealDictCursor
)
# Set to autocommit for read operations
self.connection.autocommit = True
logger.info(f"Connected to PostgreSQL: {self.host}:{self.port}/{self.database}")
except Exception as e:
logger.error(f"Failed to connect to database: {e}")
raise
def execute_query(self, query: str, params: Optional[Tuple] = None) -> List[Dict[str, Any]]:
"""Execute a SELECT query and return results"""
try:
with self.connection.cursor() as cursor:
cursor.execute(query, params)
return [dict(row) for row in cursor.fetchall()]
except Exception as e:
logger.error(f"Query failed: {query[:100]}... Error: {e}")
raise
def execute_single(self, query: str, params: Optional[Tuple] = None) -> Optional[Dict[str, Any]]:
"""Execute a query and return single result"""
results = self.execute_query(query, params)
return results[0] if results else None
def execute_scalar(self, query: str, params: Optional[Tuple] = None) -> Any:
"""Execute a query and return single value"""
result = self.execute_single(query, params)
return list(result.values())[0] if result else None
# Dashboard Statistics
def get_dashboard_stats(self) -> Dict[str, Any]:
"""Get dashboard statistics"""
stats = {}
# Movie statistics
movie_query = """
SELECT
COUNT(*) as total_movies,
COUNT(CASE WHEN dateadded IS NOT NULL AND source != 'unknown' THEN 1 END) as movies_with_dates,
COUNT(CASE WHEN dateadded IS NULL OR source = 'unknown' THEN 1 END) as movies_without_dates
FROM movies
"""
movie_stats = self.execute_single(movie_query)
stats.update(movie_stats)
# TV statistics
tv_query = """
SELECT
COUNT(DISTINCT imdb_id) as total_series,
COUNT(*) as total_episodes,
COUNT(CASE WHEN dateadded IS NOT NULL AND source != 'unknown' THEN 1 END) as episodes_with_dates,
COUNT(CASE WHEN dateadded IS NULL OR source = 'unknown' THEN 1 END) as episodes_without_dates
FROM episodes
"""
tv_stats = self.execute_single(tv_query)
stats.update(tv_stats)
return stats
# Movie queries
def get_movies(self, skip: int = 0, limit: int = 50, has_date: Optional[bool] = None) -> List[Dict[str, Any]]:
"""Get movies with pagination"""
where_clause = ""
params = []
if has_date is not None:
if has_date:
where_clause = "WHERE dateadded IS NOT NULL AND source != 'unknown'"
else:
where_clause = "WHERE dateadded IS NULL OR source = 'unknown'"
query = f"""
SELECT imdb_id, title, year, dateadded, released, source, last_updated
FROM movies
{where_clause}
ORDER BY title, year
LIMIT %s OFFSET %s
"""
params.extend([limit, skip])
return self.execute_query(query, params)
def get_movie_count(self, has_date: Optional[bool] = None) -> int:
"""Get total movie count"""
where_clause = ""
params = []
if has_date is not None:
if has_date:
where_clause = "WHERE dateadded IS NOT NULL AND source != 'unknown'"
else:
where_clause = "WHERE dateadded IS NULL OR source = 'unknown'"
query = f"SELECT COUNT(*) FROM movies {where_clause}"
return self.execute_scalar(query, params)
# TV Series queries
def get_series(self, skip: int = 0, limit: int = 50, date_filter: str = "none") -> List[Dict[str, Any]]:
"""Get TV series with episode statistics"""
where_clause = ""
if date_filter == "complete":
where_clause = """
WHERE NOT EXISTS (
SELECT 1 FROM episodes e2
WHERE e2.imdb_id = e.imdb_id
AND (e2.dateadded IS NULL OR e2.source = 'unknown')
)
"""
elif date_filter == "incomplete":
where_clause = """
WHERE EXISTS (
SELECT 1 FROM episodes e2
WHERE e2.imdb_id = e.imdb_id
AND (e2.dateadded IS NULL OR e2.source = 'unknown')
)
"""
query = f"""
SELECT
e.imdb_id,
e.series_title,
COUNT(*) as total_episodes,
COUNT(CASE WHEN e.dateadded IS NOT NULL AND e.source != 'unknown' THEN 1 END) as episodes_with_dates,
COUNT(CASE WHEN e.dateadded IS NULL OR e.source = 'unknown' THEN 1 END) as episodes_without_dates,
MAX(e.last_updated) as last_updated
FROM episodes e
{where_clause}
GROUP BY e.imdb_id, e.series_title
ORDER BY e.series_title
LIMIT %s OFFSET %s
"""
return self.execute_query(query, [limit, skip])
def get_series_count(self, date_filter: str = "none") -> int:
"""Get total series count"""
where_clause = ""
if date_filter == "complete":
where_clause = """
WHERE NOT EXISTS (
SELECT 1 FROM episodes e2
WHERE e2.imdb_id = e.imdb_id
AND (e2.dateadded IS NULL OR e2.source = 'unknown')
)
"""
elif date_filter == "incomplete":
where_clause = """
WHERE EXISTS (
SELECT 1 FROM episodes e2
WHERE e2.imdb_id = e.imdb_id
AND (e2.dateadded IS NULL OR e2.source = 'unknown')
)
"""
query = f"""
SELECT COUNT(DISTINCT imdb_id)
FROM episodes e
{where_clause}
"""
return self.execute_scalar(query)
def get_episodes_for_series(self, imdb_id: str) -> List[Dict[str, Any]]:
"""Get all episodes for a series"""
query = """
SELECT imdb_id, series_title, season, episode, episode_title,
dateadded, source, last_updated
FROM episodes
WHERE imdb_id = %s
ORDER BY season, episode
"""
return self.execute_query(query, [imdb_id])
# Source statistics
def get_series_sources(self) -> List[Dict[str, Any]]:
"""Get source statistics for series"""
query = """
SELECT
source,
COUNT(DISTINCT imdb_id) as series_count,
COUNT(*) as episode_count
FROM episodes
WHERE source != 'unknown'
GROUP BY source
ORDER BY series_count DESC, episode_count DESC
"""
return self.execute_query(query)
# Episode-specific methods for web interface
def get_episode_date(self, imdb_id: str, season: int, episode: int) -> Optional[Dict]:
"""Get episode data including dates"""
query = """
SELECT imdb_id, season, episode, aired, dateadded, source, has_video_file, last_updated
FROM episodes
WHERE imdb_id = %s AND season = %s AND episode = %s
"""
return self.execute_single(query, (imdb_id, season, episode))
def upsert_episode_date(self, imdb_id: str, season: int, episode: int,
aired: Optional[str], dateadded: Optional[str],
source: str, has_video_file: bool = False) -> None:
"""Update or insert episode date information"""
# First check if episode exists
existing = self.get_episode_date(imdb_id, season, episode)
# Temporarily disable autocommit for the transaction
original_autocommit = self.connection.autocommit
self.connection.autocommit = False
try:
with self.connection.cursor() as cursor:
if existing:
# Update existing episode
query = """
UPDATE episodes
SET aired = %s, dateadded = %s, source = %s, has_video_file = %s, last_updated = NOW()
WHERE imdb_id = %s AND season = %s AND episode = %s
"""
cursor.execute(query, (aired, dateadded, source, has_video_file, imdb_id, season, episode))
else:
# Insert new episode
query = """
INSERT INTO episodes (imdb_id, season, episode, aired, dateadded, source, has_video_file, last_updated)
VALUES (%s, %s, %s, %s, %s, %s, %s, NOW())
"""
cursor.execute(query, (imdb_id, season, episode, aired, dateadded, source, has_video_file))
self.connection.commit()
except Exception as e:
self.connection.rollback()
logger.error(f"Failed to upsert episode date: {e}")
raise
finally:
# Restore original autocommit setting
self.connection.autocommit = original_autocommit
def get_movie_dates(self, imdb_id: str) -> Optional[Dict]:
"""Get movie data including dates"""
query = """
SELECT imdb_id, path, released, dateadded, source, has_video_file, last_updated
FROM movies
WHERE imdb_id = %s
"""
return self.execute_single(query, (imdb_id,))
def upsert_movie_dates(self, imdb_id: str, released: Optional[str],
dateadded: Optional[str], source: str,
has_video_file: bool = False, path: str = "") -> None:
"""Update or insert movie date information"""
# First check if movie exists
existing = self.get_movie_dates(imdb_id)
# Temporarily disable autocommit for the transaction
original_autocommit = self.connection.autocommit
self.connection.autocommit = False
try:
with self.connection.cursor() as cursor:
if existing:
# Update existing movie
query = """
UPDATE movies
SET released = %s, dateadded = %s, source = %s, has_video_file = %s, last_updated = NOW()
WHERE imdb_id = %s
"""
cursor.execute(query, (released, dateadded, source, has_video_file, imdb_id))
else:
# Insert new movie
query = """
INSERT INTO movies (imdb_id, path, released, dateadded, source, has_video_file, last_updated)
VALUES (%s, %s, %s, %s, %s, %s, NOW())
"""
cursor.execute(query, (imdb_id, path, released, dateadded, source, has_video_file))
self.connection.commit()
except Exception as e:
self.connection.rollback()
logger.error(f"Failed to upsert movie dates: {e}")
raise
finally:
# Restore original autocommit setting
self.connection.autocommit = original_autocommit
def get_connection(self):
"""Get database connection for advanced operations"""
return self.connection
def _get_first_value(self, row):
"""Extract first value from a database row (compatibility method)"""
if row is None:
return None
if isinstance(row, dict):
return list(row.values())[0] if row else None
return row[0] if row else None
def get_stats(self) -> Dict[str, Any]:
"""Get basic database statistics (compatibility method)"""
return self.get_dashboard_stats()
def add_processing_history(self, imdb_id: str, media_type: str, event_type: str, details: Dict) -> None:
"""Add processing history entry (simplified for web interface)"""
# Temporarily disable autocommit for the transaction
original_autocommit = self.connection.autocommit
self.connection.autocommit = False
try:
with self.connection.cursor() as cursor:
# Check if processing_history table exists
cursor.execute("""
SELECT EXISTS (
SELECT FROM information_schema.tables
WHERE table_name = 'processing_history'
)
""")
table_exists = cursor.fetchone()[0]
if table_exists:
query = """
INSERT INTO processing_history (imdb_id, media_type, event_type, details, processed_at)
VALUES (%s, %s, %s, %s, NOW())
"""
import json
cursor.execute(query, (imdb_id, media_type, event_type, json.dumps(details)))
self.connection.commit()
else:
# Table doesn't exist, skip logging
logger.debug("Processing history table not found, skipping log entry")
except Exception as e:
self.connection.rollback()
logger.error(f"Failed to add processing history: {e}")
# Don't raise, this is non-critical
finally:
# Restore original autocommit setting
self.connection.autocommit = original_autocommit
def close(self):
"""Close database connection"""
if self.connection:
self.connection.close()
logger.info("Database connection closed")
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
nfoguard-web/logo/NFOguardLogoPlain.png
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 926 KiB

nfoguard-web/main_web.py
-142
View File
@@ -1,142 +0,0 @@
"""
NFOGuard Web Interface - Separated Web Application
Lightweight FastAPI application for web interface only
"""
import asyncio
import signal
import sys
import os
from pathlib import Path
import uvicorn
from fastapi import FastAPI
from fastapi.staticfiles import StaticFiles
from fastapi.responses import FileResponse
# Add current directory and parent directory to path for imports
sys.path.append(str(Path(__file__).parent))
sys.path.append(str(Path(__file__).parent.parent))
# Import web-specific configuration
from config.web_settings import web_config
# Import database (lightweight, read-only access)
from core.web_database import WebDatabase
# Import web routes and authentication
from api.web_routes import register_web_routes
from api.auth import SimpleAuthMiddleware, create_auth_dependencies
def create_web_app() -> FastAPI:
"""Create FastAPI web application"""
app = FastAPI(
title="NFOGuard Web Interface",
description="Web interface for NFOGuard media database management",
version="2.9.0-fixes-only-files",
docs_url="/docs" if web_config.web_debug else None,
redoc_url="/redoc" if web_config.web_debug else None
)
return app
def initialize_web_database() -> WebDatabase:
"""Initialize web database connection (read-only optimized)"""
return WebDatabase(
db_type=web_config.db_type,
host=web_config.db_host,
port=web_config.db_port,
database=web_config.db_name,
user=web_config.db_user,
password=web_config.db_password
)
def setup_static_files(app: FastAPI) -> None:
"""Mount static file directories"""
# Mount main static files
app.mount("/static", StaticFiles(directory="static"), name="static")
# Mount logo separately for easy access
app.mount("/logo", StaticFiles(directory="logo"), name="logo")
# Serve index.html at root
@app.get("/")
async def serve_index():
return FileResponse("static/index.html")
def setup_signal_handlers():
"""Setup graceful shutdown signal handlers"""
def signal_handler(signum, frame):
print(f"\n🛑 Received signal {signum}, shutting down web interface...")
# Web interface can shutdown immediately (no background processing)
sys.exit(0)
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGINT, signal_handler)
def main():
"""Main entry point for NFOGuard Web Interface"""
print("🌐 Starting NFOGuard Web Interface...")
print(f"📊 Configuration: Port {web_config.web_port}, Auth: {'Enabled' if web_config.web_auth_enabled else 'Disabled'}")
# Setup signal handlers
setup_signal_handlers()
# Create FastAPI app
app = create_web_app()
# Initialize database
try:
db = initialize_web_database()
print(f"✅ Connected to database: {web_config.db_host}:{web_config.db_port}/{web_config.db_name}")
except Exception as e:
print(f"❌ Failed to connect to database: {e}")
sys.exit(1)
# Create dependencies for dependency injection
dependencies = {
"db": db,
"config": web_config
}
# Add authentication dependencies if enabled
if web_config.web_auth_enabled:
auth_deps = create_auth_dependencies(web_config)
dependencies.update(auth_deps)
# Add authentication middleware
app.add_middleware(SimpleAuthMiddleware, config=web_config)
print(f"🔐 Web authentication enabled for user: {web_config.web_auth_username}")
else:
print("🔓 Web authentication disabled - interface is public")
# Setup static files and routes
setup_static_files(app)
# Register web routes
register_web_routes(app, dependencies)
print(f"🚀 Starting web server on {web_config.web_host}:{web_config.web_port}")
try:
uvicorn.run(
app,
host=web_config.web_host,
port=web_config.web_port,
workers=web_config.web_workers,
log_level="debug" if web_config.web_debug else "info",
access_log=web_config.web_debug
)
except KeyboardInterrupt:
print("\n🛑 Web interface shutdown by user")
except Exception as e:
print(f"❌ Web interface failed: {e}")
sys.exit(1)
if __name__ == "__main__":
main()
nfoguard-web/static/css/styles.css
-902
View File
@@ -1,902 +0,0 @@
/* NFOGuard Web Interface Styles */
:root {
--primary-color: #007bff;
--secondary-color: #6c757d;
--success-color: #28a745;
--warning-color: #ffc107;
--danger-color: #dc3545;
--dark-color: #343a40;
--light-color: #f8f9fa;
--border-color: #dee2e6;
--text-muted: #6c757d;
--shadow: 0 0.125rem 0.25rem rgba(0, 0, 0, 0.075);
--shadow-lg: 0 0.5rem 1rem rgba(0, 0, 0, 0.15);
}
* {
margin: 0;
padding: 0;
box-sizing: border-box;
}
body {
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;
font-size: 14px;
line-height: 1.5;
color: var(--dark-color);
background-color: #f5f5f5;
}
.app-container {
min-height: 100vh;
display: flex;
flex-direction: column;
}
/* Header */
.app-header {
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
color: white;
padding: 1rem 0;
box-shadow: var(--shadow-lg);
position: relative;
}
.header-content {
max-width: 1200px;
margin: 0 auto;
padding: 0 1rem;
text-align: center;
}
.header-content h1 {
font-size: 2rem;
font-weight: 300;
margin-bottom: 0.5rem;
}
.header-content h1 i {
margin-right: 0.5rem;
}
.header-content p {
opacity: 0.9;
font-size: 1rem;
}
/* Authentication Status */
.auth-status {
position: absolute;
top: 1rem;
right: 1rem;
display: flex;
align-items: center;
gap: 1rem;
color: white;
font-size: 0.9rem;
}
.auth-user {
display: flex;
align-items: center;
gap: 0.5rem;
opacity: 0.9;
}
.auth-logout {
background: rgba(255, 255, 255, 0.2);
color: white;
border: 1px solid rgba(255, 255, 255, 0.3);
padding: 0.5rem 1rem;
border-radius: 0.25rem;
cursor: pointer;
font-size: 0.85rem;
display: flex;
align-items: center;
gap: 0.5rem;
transition: all 0.2s ease;
}
.auth-logout:hover {
background: rgba(255, 255, 255, 0.3);
border-color: rgba(255, 255, 255, 0.5);
transform: translateY(-1px);
}
.nav-tabs {
max-width: 1200px;
margin: 1rem auto 0;
padding: 0 1rem;
display: flex;
gap: 0.5rem;
flex-wrap: wrap;
justify-content: center;
}
.nav-tab {
background: rgba(255, 255, 255, 0.1);
border: none;
color: white;
padding: 0.75rem 1.5rem;
border-radius: 0.5rem;
cursor: pointer;
transition: all 0.3s ease;
font-size: 0.9rem;
display: flex;
align-items: center;
gap: 0.5rem;
}
.nav-tab:hover {
background: rgba(255, 255, 255, 0.2);
transform: translateY(-1px);
}
.nav-tab.active {
background: rgba(255, 255, 255, 0.9);
color: var(--dark-color);
}
/* Main Content */
.main-content {
flex: 1;
max-width: 1200px;
margin: 0 auto;
padding: 2rem 1rem;
width: 100%;
}
.tab-content {
display: none;
}
.tab-content.active {
display: block;
}
/* Dashboard */
.dashboard-grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
gap: 1.5rem;
margin-bottom: 2rem;
}
.stat-card {
background: white;
border-radius: 0.5rem;
padding: 1.5rem;
box-shadow: var(--shadow);
display: flex;
align-items: center;
gap: 1rem;
}
.stat-icon {
width: 60px;
height: 60px;
border-radius: 50%;
display: flex;
align-items: center;
justify-content: center;
font-size: 1.5rem;
color: white;
}
.stat-icon.movies { background: linear-gradient(135deg, #667eea, #764ba2); }
.stat-icon.tv { background: linear-gradient(135deg, #f093fb, #f5576c); }
.stat-icon.missing { background: linear-gradient(135deg, #ffecd2, #fcb69f); }
.stat-icon.activity { background: linear-gradient(135deg, #a8edea, #fed6e3); }
.stat-info h3 {
font-size: 2rem;
font-weight: 700;
margin-bottom: 0.25rem;
}
.stat-info p {
font-weight: 500;
margin-bottom: 0.25rem;
}
.stat-info small {
color: var(--text-muted);
font-size: 0.85rem;
}
.dashboard-charts {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
gap: 1.5rem;
}
.chart-card {
background: white;
border-radius: 0.5rem;
padding: 1.5rem;
box-shadow: var(--shadow);
}
.chart-card h3 {
margin-bottom: 1rem;
color: var(--dark-color);
}
.chart-container {
height: 200px;
display: flex;
align-items: center;
justify-content: center;
background: var(--light-color);
border-radius: 0.25rem;
color: var(--text-muted);
}
/* Content Header */
.content-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 1.5rem;
flex-wrap: wrap;
gap: 1rem;
}
.content-header h2 {
color: var(--dark-color);
font-weight: 600;
}
.content-controls {
display: flex;
gap: 1rem;
align-items: center;
flex-wrap: wrap;
}
.search-controls {
display: flex;
gap: 0.5rem;
flex-wrap: wrap;
}
.filter-controls {
display: flex;
gap: 0.5rem;
align-items: center;
flex-wrap: wrap;
}
.search-box {
position: relative;
display: flex;
align-items: center;
}
.search-box i {
position: absolute;
left: 0.75rem;
color: var(--text-muted);
}
.search-box input {
padding: 0.5rem 0.75rem 0.5rem 2.5rem;
border: 1px solid var(--border-color);
border-radius: 0.25rem;
font-size: 0.9rem;
width: 250px;
}
.search-box input:focus {
outline: none;
border-color: var(--primary-color);
box-shadow: 0 0 0 0.2rem rgba(0, 123, 255, 0.25);
}
/* Buttons */
.btn {
padding: 0.5rem 1rem;
border: none;
border-radius: 0.25rem;
cursor: pointer;
font-size: 0.9rem;
transition: all 0.2s ease;
display: inline-flex;
align-items: center;
gap: 0.5rem;
text-decoration: none;
}
.btn-primary {
background-color: var(--primary-color);
color: white;
}
.btn-primary:hover {
background-color: #0056b3;
transform: translateY(-1px);
}
.btn-secondary {
background-color: var(--secondary-color);
color: white;
}
.btn-secondary:hover {
background-color: #545b62;
}
.btn-success {
background-color: var(--success-color);
color: white;
}
.btn-success:hover {
background-color: #1e7e34;
}
.btn-warning {
background-color: var(--warning-color);
color: var(--dark-color);
}
.btn-warning:hover {
background-color: #e0a800;
}
.btn-danger {
background-color: var(--danger-color);
color: white;
}
.btn-danger:hover {
background-color: #c82333;
}
.btn-sm {
padding: 0.25rem 0.5rem;
font-size: 0.8rem;
}
/* Tables */
.table-container {
background: white;
border-radius: 0.5rem;
box-shadow: var(--shadow);
overflow: hidden;
margin-bottom: 1rem;
}
.data-table {
width: 100%;
border-collapse: collapse;
}
.data-table th,
.data-table td {
padding: 0.75rem;
text-align: left;
border-bottom: 1px solid var(--border-color);
}
.data-table th {
background-color: var(--light-color);
font-weight: 600;
color: var(--dark-color);
position: sticky;
top: 0;
}
.data-table tr:hover {
background-color: rgba(0, 123, 255, 0.05);
}
.data-table .loading {
text-align: center;
color: var(--text-muted);
font-style: italic;
padding: 2rem;
}
/* Status badges */
.badge {
padding: 0.25rem 0.5rem;
border-radius: 0.25rem;
font-size: 0.75rem;
font-weight: 600;
text-transform: uppercase;
}
.badge-success {
background-color: #d4edda;
color: #155724;
}
.badge-warning {
background-color: #fff3cd;
color: #856404;
}
.badge-danger {
background-color: #f8d7da;
color: #721c24;
}
.badge-secondary {
background-color: #e9ecef;
color: #495057;
}
/* Pagination */
.pagination {
display: flex;
justify-content: center;
align-items: center;
gap: 0.5rem;
margin-top: 1rem;
}
.pagination .btn {
padding: 0.5rem 0.75rem;
}
.pagination .page-info {
margin: 0 1rem;
color: var(--text-muted);
}
/* Forms */
.form-group {
margin-bottom: 1rem;
}
.form-group label {
display: block;
margin-bottom: 0.25rem;
font-weight: 500;
}
.form-group input,
.form-group select,
.form-group textarea {
width: 100%;
padding: 0.5rem;
border: 1px solid var(--border-color);
border-radius: 0.25rem;
font-size: 0.9rem;
}
.form-group input:focus,
.form-group select:focus,
.form-group textarea:focus {
outline: none;
border-color: var(--primary-color);
box-shadow: 0 0 0 0.2rem rgba(0, 123, 255, 0.25);
}
.form-group small {
display: block;
margin-top: 0.25rem;
color: var(--text-muted);
font-size: 0.8rem;
}
.form-actions {
display: flex;
gap: 0.5rem;
justify-content: flex-end;
margin-top: 1.5rem;
}
/* Modal */
.modal {
display: none;
position: fixed;
z-index: 1000;
left: 0;
top: 0;
width: 100%;
height: 100%;
background-color: rgba(0, 0, 0, 0.5);
}
.modal.active {
display: flex;
align-items: center;
justify-content: center;
}
.modal-content {
background: white;
border-radius: 0.5rem;
max-width: 500px;
width: 90%;
max-height: 90vh;
overflow-y: auto;
box-shadow: var(--shadow-lg);
}
.modal-header {
padding: 1rem 1.5rem;
border-bottom: 1px solid var(--border-color);
display: flex;
justify-content: space-between;
align-items: center;
}
.modal-header h3 {
margin: 0;
}
.modal-close {
background: none;
border: none;
font-size: 1.5rem;
cursor: pointer;
color: var(--text-muted);
}
.modal-close:hover {
color: var(--dark-color);
}
.modal-body {
padding: 1.5rem;
}
/* Higher z-index for edit modals that appear on top of other modals */
#edit-modal, #smart-fix-modal {
z-index: 1100 !important;
}
/* Reports */
.report-summary {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
gap: 1rem;
margin-bottom: 2rem;
}
.summary-card {
background: white;
padding: 1.5rem;
border-radius: 0.5rem;
box-shadow: var(--shadow);
text-align: center;
}
.summary-card h3 {
margin-bottom: 1rem;
color: var(--dark-color);
}
.summary-card p {
margin-bottom: 0.5rem;
font-size: 1.1rem;
}
.summary-card span {
font-weight: 700;
color: var(--primary-color);
}
.report-section {
margin-bottom: 2rem;
}
.report-section h3 {
margin-bottom: 1rem;
color: var(--dark-color);
}
/* Tools */
.tools-grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
gap: 1.5rem;
}
.tool-card {
background: white;
border-radius: 0.5rem;
padding: 1.5rem;
box-shadow: var(--shadow);
}
.tool-card h3 {
margin-bottom: 0.5rem;
color: var(--dark-color);
}
.tool-card p {
margin-bottom: 1.5rem;
color: var(--text-muted);
}
.stats-display {
background: var(--light-color);
padding: 1rem;
border-radius: 0.25rem;
margin-bottom: 1rem;
min-height: 100px;
}
/* Toast notifications */
.toast-container {
position: fixed;
top: 20px;
right: 20px;
z-index: 1050;
}
.toast {
background: white;
border-radius: 0.25rem;
box-shadow: var(--shadow-lg);
margin-bottom: 0.5rem;
padding: 0.75rem 1rem;
min-width: 300px;
border-left: 4px solid var(--primary-color);
animation: slideIn 0.3s ease;
}
.toast.success {
border-left-color: var(--success-color);
}
.toast.warning {
border-left-color: var(--warning-color);
}
.toast.error {
border-left-color: var(--danger-color);
}
@keyframes slideIn {
from {
transform: translateX(100%);
opacity: 0;
}
to {
transform: translateX(0);
opacity: 1;
}
}
/* Smart Fix Modal */
.smart-fix-options {
display: flex;
flex-direction: column;
gap: 1rem;
margin-bottom: 1rem;
}
.option-card {
border: 2px solid var(--border-color);
border-radius: 0.5rem;
transition: all 0.2s ease;
}
.option-card:hover {
border-color: var(--primary-color);
box-shadow: var(--shadow);
}
.option-label {
display: block;
padding: 1rem;
cursor: pointer;
margin: 0;
}
.option-label input[type="radio"] {
margin-right: 0.75rem;
margin-top: 0.1rem;
width: auto;
}
.option-content h4 {
margin: 0 0 0.5rem 0;
color: var(--dark-color);
font-size: 1rem;
}
.option-content p {
margin: 0 0 0.5rem 0;
color: var(--text-muted);
font-size: 0.9rem;
}
.option-content small {
color: var(--text-muted);
font-size: 0.8rem;
}
.manual-date-input {
width: 100% !important;
margin-top: 0.5rem !important;
}
.option-card input[type="radio"]:checked + .option-content {
color: var(--primary-color);
}
.option-card:has(input[type="radio"]:checked) {
border-color: var(--primary-color);
background-color: rgba(0, 123, 255, 0.05);
}
/* Additional badge styles */
.badge-info {
background-color: #d1ecf1;
color: #0c5460;
}
/* Enhanced Edit Modal Date Options */
.date-options {
display: flex;
flex-direction: column;
gap: 0.75rem;
margin-bottom: 1rem;
}
.date-option-card {
border: 1px solid var(--border-color);
border-radius: 0.375rem;
transition: all 0.2s ease;
}
.date-option-card:hover {
border-color: var(--primary-color);
box-shadow: 0 0 0 2px rgba(0, 123, 255, 0.1);
}
.date-option-label {
display: block;
padding: 0.75rem;
cursor: pointer;
margin: 0;
}
.date-option-label input[type="radio"] {
margin-right: 0.5rem;
margin-top: 0.1rem;
width: auto;
}
.date-option-content h4 {
margin: 0 0 0.25rem 0;
color: var(--dark-color);
font-size: 0.9rem;
font-weight: 600;
}
.date-option-content p {
margin: 0 0 0.25rem 0;
color: var(--text-muted);
font-size: 0.8rem;
}
.date-option-content small {
color: var(--primary-color);
font-size: 0.75rem;
font-weight: 500;
}
.date-option-card input[type="radio"]:checked + .date-option-content h4 {
color: var(--primary-color);
}
.date-option-card:has(input[type="radio"]:checked) {
border-color: var(--primary-color);
background-color: rgba(0, 123, 255, 0.03);
}
/* Responsive */
@media (max-width: 768px) {
.content-header {
flex-direction: column;
align-items: stretch;
}
.content-controls {
justify-content: center;
}
.search-box input {
width: 200px;
}
.nav-tabs {
flex-direction: column;
gap: 0.25rem;
}
.data-table {
font-size: 0.8rem;
}
.data-table th,
.data-table td {
padding: 0.5rem 0.25rem;
}
.dashboard-grid {
grid-template-columns: 1fr;
}
.tools-grid {
grid-template-columns: 1fr;
}
}
/* Utility classes */
.text-center { text-align: center; }
.text-muted { color: var(--text-muted); }
.mb-0 { margin-bottom: 0; }
.mb-1 { margin-bottom: 0.5rem; }
.mb-2 { margin-bottom: 1rem; }
.mt-1 { margin-top: 0.5rem; }
.mt-2 { margin-top: 1rem; }
.d-none { display: none; }
.d-block { display: block; }
.d-flex { display: flex; }
.justify-content-between { justify-content: space-between; }
.align-items-center { align-items: center; }
/* Manual Scan Styles */
.scan-status {
margin-top: 1rem;
padding: 1rem;
background-color: var(--light-color);
border: 1px solid var(--border-color);
border-radius: 0.375rem;
}
.scan-progress {
margin-bottom: 1rem;
}
.progress-bar {
width: 100%;
height: 1.5rem;
background-color: #e9ecef;
border-radius: 0.375rem;
overflow: hidden;
margin-bottom: 0.5rem;
}
.progress-fill {
height: 100%;
background: linear-gradient(90deg, var(--primary-color), var(--success-color));
transition: width 0.3s ease;
width: 0%;
}
.scan-info {
display: flex;
justify-content: space-between;
align-items: center;
font-size: 0.875rem;
}
.scan-info span:first-child {
color: var(--text-muted);
}
.scan-info span:last-child {
font-weight: 600;
color: var(--primary-color);
}
.form-group small {
display: block;
margin-top: 0.25rem;
color: var(--text-muted);
font-size: 0.875rem;
}
.btn-sm {
padding: 0.375rem 0.75rem;
font-size: 0.875rem;
}
nfoguard-web/static/index.html
-732
View File
@@ -1,732 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>NFOGuard - Database Management</title>
<link rel="stylesheet" href="/static/css/styles.css?v=2.10.0-skipped-imdb-edit-v2">
<link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0/css/all.min.css" rel="stylesheet">
</head>
<body>
<div class="app-container">
<!-- Header -->
<header class="app-header">
<div class="header-content">
<h1><i class="fas fa-shield-alt"></i> NFOGuard <span style="font-size: 0.5em; color: #888;">v2.10.0-skipped-imdb-edit-v2</span></h1>
<p>Database Management & Reporting</p>
</div>
<div class="auth-status" id="auth-status" style="display: none;">
<span class="auth-user">
<i class="fas fa-user"></i> <span id="auth-username">Loading...</span>
</span>
<button class="auth-logout" id="logout-btn" onclick="logout()">
<i class="fas fa-sign-out-alt"></i> Logout
</button>
</div>
<nav class="nav-tabs">
<button class="nav-tab active" data-tab="dashboard">
<i class="fas fa-tachometer-alt"></i> Dashboard
</button>
<button class="nav-tab" data-tab="movies">
<i class="fas fa-film"></i> Movies
</button>
<button class="nav-tab" data-tab="tv">
<i class="fas fa-tv"></i> TV Series
</button>
<button class="nav-tab" data-tab="reports">
<i class="fas fa-chart-bar"></i> Reports
</button>
<button class="nav-tab" data-tab="scheduled-scans">
<i class="fas fa-clock"></i> Scheduled Scans
</button>
<button class="nav-tab" data-tab="tools">
<i class="fas fa-tools"></i> Tools
</button>
</nav>
</header>
<!-- Main Content -->
<main class="main-content">
<!-- Dashboard Tab -->
<div class="tab-content active" id="dashboard">
<div class="dashboard-grid">
<div class="stat-card">
<div class="stat-icon movies">
<i class="fas fa-film"></i>
</div>
<div class="stat-info">
<h3 id="movies-total">-</h3>
<p>Total Movies</p>
<small id="movies-with-dates">- with dates</small>
</div>
</div>
<div class="stat-card">
<div class="stat-icon tv">
<i class="fas fa-tv"></i>
</div>
<div class="stat-info">
<h3 id="series-total">-</h3>
<p>TV Series</p>
<small id="episodes-total">- episodes</small>
</div>
</div>
<div class="stat-card">
<div class="stat-icon missing">
<i class="fas fa-exclamation-triangle"></i>
</div>
<div class="stat-info">
<h3 id="missing-dates-total">-</h3>
<p>Missing Dates</p>
<small id="no-valid-source-total">- no valid source</small>
</div>
</div>
<div class="stat-card">
<div class="stat-icon activity">
<i class="fas fa-history"></i>
</div>
<div class="stat-info">
<h3 id="recent-activity">-</h3>
<p>Recent Activity</p>
<small>Last 7 days</small>
</div>
</div>
<div class="stat-card">
<div class="stat-icon" style="background: linear-gradient(135deg, #f39c12 0%, #e67e22 100%);">
<i class="fas fa-ban"></i>
</div>
<div class="stat-info">
<h3 id="skipped-total">-</h3>
<p>Skipped Items</p>
<small id="skipped-breakdown">- movies, - episodes</small>
</div>
</div>
</div>
<div class="dashboard-charts">
<div class="chart-card">
<h3><i class="fas fa-chart-pie"></i> Movie Sources</h3>
<div id="movie-sources-chart" class="chart-container"></div>
</div>
<div class="chart-card">
<h3><i class="fas fa-chart-pie"></i> Episode Sources</h3>
<div id="episode-sources-chart" class="chart-container"></div>
</div>
</div>
</div>
<!-- Movies Tab -->
<div class="tab-content" id="movies">
<div class="content-header">
<h2><i class="fas fa-film"></i> Movies Database</h2>
<div class="content-controls">
<div class="search-controls">
<div class="search-box">
<i class="fas fa-search"></i>
<input type="text" id="movies-search" placeholder="Search title/path...">
</div>
<div class="search-box">
<i class="fas fa-hashtag"></i>
<input type="text" id="movies-imdb-search" placeholder="Search IMDb ID...">
</div>
</div>
<div class="filter-controls">
<select id="movies-filter-date">
<option value="">All Movies</option>
<option value="true">With Dates</option>
<option value="false">Missing Dates</option>
<option value="skipped">Skipped</option>
</select>
<select id="movies-filter-source">
<option value="">All Sources</option>
</select>
<button class="btn btn-primary" onclick="refreshMovies()">
<i class="fas fa-sync"></i> Refresh
</button>
</div>
</div>
</div>
<div class="table-container">
<table class="data-table sortable-table" id="movies-table">
<thead>
<tr>
<th class="sortable" onclick="sortTable('movies-tbody', 0, 'text')" style="cursor: pointer;">
Title <i class="fas fa-sort"></i>
</th>
<th class="sortable" onclick="sortTable('movies-tbody', 1, 'text')" style="cursor: pointer;">
IMDb ID <i class="fas fa-sort"></i>
</th>
<th class="sortable" onclick="sortTable('movies-tbody', 2, 'date')" style="cursor: pointer;">
Movie Released <i class="fas fa-sort"></i>
</th>
<th class="sortable" onclick="sortTable('movies-tbody', 3, 'date')" style="cursor: pointer;">
Date Added to Library <i class="fas fa-sort"></i>
</th>
<th class="sortable" onclick="sortTable('movies-tbody', 4, 'text')" style="cursor: pointer;">
Source <i class="fas fa-sort"></i>
</th>
<th class="sortable" onclick="sortTable('movies-tbody', 5, 'text')" style="cursor: pointer;">
Date Type <i class="fas fa-sort"></i>
</th>
<th class="sortable" onclick="sortTable('movies-tbody', 6, 'text')" style="cursor: pointer;">
Video File <i class="fas fa-sort"></i>
</th>
<th>Actions</th>
</tr>
</thead>
<tbody id="movies-tbody">
<tr>
<td colspan="8" class="loading">Loading movies...</td>
</tr>
</tbody>
</table>
</div>
<div class="pagination" id="movies-pagination"></div>
</div>
<!-- TV Series Tab -->
<div class="tab-content" id="tv">
<div class="content-header">
<h2><i class="fas fa-tv"></i> TV Series Database</h2>
<div class="content-controls">
<div class="search-controls">
<div class="search-box">
<i class="fas fa-search"></i>
<input type="text" id="series-search" placeholder="Search title/path...">
</div>
<div class="search-box">
<i class="fas fa-hashtag"></i>
<input type="text" id="series-imdb-search" placeholder="Search IMDb ID...">
</div>
</div>
<div class="filter-controls">
<select id="series-filter-date">
<option value="">All Series</option>
<option value="complete">Fully Dated</option>
<option value="incomplete">Missing Dates</option>
<option value="none">No Dates</option>
<option value="skipped">Skipped</option>
</select>
<select id="series-filter-source">
<option value="">All Sources</option>
</select>
<button class="btn btn-primary" onclick="refreshSeries()">
<i class="fas fa-sync"></i> Refresh
</button>
</div>
</div>
</div>
<div class="table-container">
<table class="data-table sortable-table" id="series-table">
<thead>
<tr>
<th class="sortable" onclick="sortTable('series-tbody', 0, 'text')" style="cursor: pointer;">
Series Title <i class="fas fa-sort"></i>
</th>
<th class="sortable" onclick="sortTable('series-tbody', 1, 'text')" style="cursor: pointer;">
IMDb ID <i class="fas fa-sort"></i>
</th>
<th class="sortable" onclick="sortTable('series-tbody', 2, 'number')" style="cursor: pointer;">
Episodes <i class="fas fa-sort"></i>
</th>
<th class="sortable" onclick="sortTable('series-tbody', 3, 'number')" style="cursor: pointer;">
With Dates <i class="fas fa-sort"></i>
</th>
<th class="sortable" onclick="sortTable('series-tbody', 4, 'number')" style="cursor: pointer;">
With Video <i class="fas fa-sort"></i>
</th>
<th class="sortable" onclick="sortTable('series-tbody', 5, 'number')" style="cursor: pointer;">
Skipped <i class="fas fa-sort"></i>
</th>
<th>Actions</th>
</tr>
</thead>
<tbody id="series-tbody">
<tr>
<td colspan="6" class="loading">Loading series...</td>
</tr>
</tbody>
</table>
</div>
<div class="pagination" id="series-pagination"></div>
</div>
<!-- Reports Tab -->
<div class="tab-content" id="reports">
<div class="content-header">
<h2><i class="fas fa-chart-bar"></i> Missing Dates Report</h2>
<div class="content-controls">
<button class="btn btn-primary" onclick="refreshReport()">
<i class="fas fa-sync"></i> Refresh Report
</button>
</div>
</div>
<div class="report-summary" id="report-summary">
<div class="summary-card">
<h3>Movies</h3>
<p><span id="report-movies-with">-</span> with dates</p>
<p><span id="report-movies-missing">-</span> missing dates</p>
</div>
<div class="summary-card">
<h3>Episodes</h3>
<p><span id="report-episodes-with">-</span> with dates</p>
<p><span id="report-episodes-missing">-</span> missing dates</p>
</div>
</div>
<div class="report-content">
<div class="report-section">
<h3><i class="fas fa-film"></i> Movies Missing Dates</h3>
<div class="table-container">
<table class="data-table">
<thead>
<tr>
<th>Title</th>
<th>IMDb ID</th>
<th>Released</th>
<th>Source</th>
<th>Smart Fix</th>
</tr>
</thead>
<tbody id="report-movies-tbody">
<tr>
<td colspan="5" class="loading">Loading report...</td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="report-section">
<h3><i class="fas fa-tv"></i> Episodes Missing Dates</h3>
<div class="table-container">
<table class="data-table">
<thead>
<tr>
<th>Series</th>
<th>Episode</th>
<th>IMDb ID</th>
<th>Aired</th>
<th>Source</th>
<th>Smart Fix</th>
</tr>
</thead>
<tbody id="report-episodes-tbody">
<tr>
<td colspan="6" class="loading">Loading report...</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
</div>
<!-- Scheduled Scans Tab -->
<div class="tab-content" id="scheduled-scans">
<div class="content-header">
<h2><i class="fas fa-clock"></i> Scheduled Scans</h2>
<button class="btn btn-primary" id="add-schedule-btn">
<i class="fas fa-plus"></i> Add Schedule
</button>
</div>
<!-- Active Schedules Section -->
<div class="section-card">
<h3><i class="fas fa-list"></i> Active Schedules</h3>
<div class="table-container">
<table class="data-table" id="schedules-table">
<thead>
<tr>
<th>Name</th>
<th>Type</th>
<th>Mode</th>
<th>Schedule</th>
<th>Last Run</th>
<th>Next Run</th>
<th>Status</th>
<th>Actions</th>
</tr>
</thead>
<tbody id="schedules-table-body">
<!-- Schedules will be loaded here -->
</tbody>
</table>
</div>
</div>
<!-- Execution History Section -->
<div class="section-card">
<h3><i class="fas fa-history"></i> Recent Executions</h3>
<div class="table-container">
<table class="data-table" id="executions-table">
<thead>
<tr>
<th>Schedule</th>
<th>Started</th>
<th>Duration</th>
<th>Status</th>
<th>Items Processed</th>
<th>Items Skipped</th>
<th>Items Failed</th>
<th>Actions</th>
</tr>
</thead>
<tbody id="executions-table-body">
<!-- Executions will be loaded here -->
</tbody>
</table>
</div>
</div>
</div>
<!-- Tools Tab -->
<div class="tab-content" id="tools">
<div class="content-header">
<h2><i class="fas fa-tools"></i> Database Tools</h2>
</div>
<div class="tools-grid">
<div class="tool-card">
<h3><i class="fas fa-exchange-alt"></i> Bulk Source Update</h3>
<p>Change source for multiple items at once</p>
<form id="bulk-update-form">
<div class="form-group">
<label>Media Type:</label>
<select id="bulk-media-type" required>
<option value="">Select type...</option>
<option value="movies">Movies</option>
<option value="episodes">Episodes</option>
</select>
</div>
<div class="form-group">
<label>From Source:</label>
<input type="text" id="bulk-old-source" placeholder="e.g., no_valid_date_source" required>
</div>
<div class="form-group">
<label>To Source:</label>
<select id="bulk-new-source" required>
<option value="">Select new source...</option>
<option value="airdate">Air Date</option>
<option value="digital_release">Digital Release</option>
<option value="manual">Manual</option>
<option value="radarr:db.history.import">Radarr Import</option>
<option value="sonarr:history.import">Sonarr Import</option>
</select>
</div>
<button type="submit" class="btn btn-warning">
<i class="fas fa-exchange-alt"></i> Update Sources
</button>
</form>
</div>
<div class="tool-card">
<h3><i class="fas fa-search"></i> Manual Scan</h3>
<p>Scan specific folders or perform full library scans</p>
<form id="manual-scan-form">
<div class="form-group">
<label>Scan Type:</label>
<select id="scan-type" required>
<option value="both">TV Shows & Movies</option>
<option value="tv">TV Shows Only</option>
<option value="movies">Movies Only</option>
</select>
</div>
<div class="form-group">
<label>Scan Mode:</label>
<select id="scan-mode" required>
<option value="smart">Smart (Recommended)</option>
<option value="full">Full Scan</option>
<option value="incomplete">Incomplete Only</option>
</select>
</div>
<div class="form-group">
<label>Specific Path (Optional):</label>
<input type="text" id="scan-path" placeholder="e.g., /mnt/unionfs/Media/TV/Series Name" title="Leave empty for full library scan">
<small>Leave empty to scan entire library</small>
</div>
<button type="submit" class="btn btn-primary">
<i class="fas fa-play"></i> Start Scan
</button>
</form>
<div id="scan-status" class="scan-status" style="display: none;">
<div class="scan-progress">
<div class="progress-bar">
<div class="progress-fill" id="scan-progress-bar"></div>
</div>
<div class="scan-info">
<span id="scan-current-operation">Initializing...</span>
<span id="scan-progress-text">0%</span>
</div>
</div>
<button class="btn btn-secondary btn-sm" onclick="stopScanPolling()">
<i class="fas fa-times"></i> Hide Status
</button>
</div>
</div>
<div class="tool-card">
<h3><i class="fas fa-database"></i> Database Statistics</h3>
<p>View detailed database information</p>
<div class="stats-display" id="detailed-stats">
<p>Click refresh to load detailed statistics</p>
</div>
<button class="btn btn-secondary" onclick="loadDetailedStats()">
<i class="fas fa-sync"></i> Refresh Stats
</button>
</div>
<div class="tool-card">
<h3><i class="fas fa-upload"></i> Populate Database</h3>
<p>Bulk import data from Radarr/Sonarr into NFOGuard database</p>
<form id="populate-form">
<div class="form-group">
<label>Media Type:</label>
<select id="populate-media-type" required>
<option value="both">Movies & TV Shows</option>
<option value="movies">Movies Only</option>
<option value="tv">TV Shows Only</option>
</select>
</div>
<button type="submit" class="btn btn-primary">
<i class="fas fa-play"></i> Start Population
</button>
</form>
<div id="populate-status" class="scan-status" style="display: none;">
<div class="scan-progress">
<div class="progress-bar">
<div class="progress-fill" id="populate-progress-bar"></div>
</div>
<div class="scan-info">
<span id="populate-current-operation">Running...</span>
<span id="populate-progress-text">In Progress</span>
</div>
</div>
<div id="populate-results" class="stats-display" style="margin-top: 10px;"></div>
<button class="btn btn-secondary btn-sm" onclick="stopPopulatePolling()">
<i class="fas fa-times"></i> Hide Status
</button>
</div>
</div>
</div>
</div>
</main>
</div>
<!-- Smart Fix Modal -->
<div class="modal" id="smart-fix-modal">
<div class="modal-content">
<div class="modal-header">
<h3 id="smart-fix-title">Choose Date Source</h3>
<button class="modal-close" onclick="closeSmartFixModal()">&times;</button>
</div>
<div class="modal-body">
<div id="smart-fix-content">
<p>Loading available options...</p>
</div>
<div class="form-actions">
<button type="button" class="btn btn-secondary" onclick="closeSmartFixModal()">Cancel</button>
</div>
</div>
</div>
</div>
<!-- Edit Modal -->
<div class="modal" id="edit-modal">
<div class="modal-content">
<div class="modal-header">
<h3 id="modal-title">Edit Entry</h3>
<button class="modal-close" onclick="closeModal()">&times;</button>
</div>
<div class="modal-body">
<form id="edit-form">
<input type="hidden" id="edit-imdb-id">
<input type="hidden" id="edit-season">
<input type="hidden" id="edit-episode">
<input type="hidden" id="edit-media-type">
<div class="form-group">
<label for="edit-dateadded">Date Added:</label>
<input type="datetime-local" id="edit-dateadded">
<small>Leave empty to clear date</small>
</div>
<div class="form-group">
<label for="edit-source">Source:</label>
<select id="edit-source" required>
<option value="manual">Manual</option>
<option value="airdate">Air Date</option>
<option value="digital_release">Digital Release</option>
<option value="radarr:db.history.import">Radarr Import</option>
<option value="sonarr:history.import">Sonarr Import</option>
<option value="no_valid_date_source">No Valid Source</option>
</select>
</div>
<div class="form-actions">
<button type="button" class="btn btn-secondary" onclick="closeModal()">Cancel</button>
<button type="submit" class="btn btn-primary">Save Changes</button>
</div>
</form>
</div>
</div>
</div>
<!-- Schedule Modal -->
<div class="modal" id="schedule-modal">
<div class="modal-content">
<div class="modal-header">
<h3 id="schedule-modal-title">Add New Schedule</h3>
<button class="modal-close" onclick="closeScheduleModal()">&times;</button>
</div>
<div class="modal-body">
<form id="schedule-form">
<input type="hidden" id="schedule-id">
<div class="form-group">
<label for="schedule-name">Schedule Name:</label>
<input type="text" id="schedule-name" required placeholder="e.g., Daily TV Incomplete Scan">
</div>
<div class="form-group">
<label for="schedule-description">Description:</label>
<textarea id="schedule-description" placeholder="Optional description of what this schedule does"></textarea>
</div>
<div class="form-row">
<div class="form-group">
<label for="schedule-media-type">Media Type:</label>
<select id="schedule-media-type" required>
<option value="both">Both TV Shows & Movies</option>
<option value="tv">TV Shows Only</option>
<option value="movies">Movies Only</option>
</select>
</div>
<div class="form-group">
<label for="schedule-scan-mode">Scan Mode:</label>
<select id="schedule-scan-mode" required>
<option value="smart">Smart (Recommended)</option>
<option value="incomplete">Incomplete Only</option>
<option value="full">Full Scan</option>
</select>
</div>
</div>
<div class="form-group">
<label for="schedule-cron">Schedule (Cron Expression):</label>
<div class="cron-input-container">
<input type="text" id="schedule-cron" required placeholder="0 2 * * *" pattern="^(\*|[0-5]?\d|\*\/[0-9]+)(\s+(\*|[0-5]?\d|\*\/[0-9]+)){4}$">
<button type="button" class="btn btn-secondary btn-sm" id="cron-builder-btn">
<i class="fas fa-magic"></i> Builder
</button>
</div>
<small class="help-text">
Examples: "0 2 * * *" (daily at 2 AM), "0 2 * * 0" (weekly on Sunday at 2 AM)
</small>
</div>
<div class="form-group">
<label for="schedule-paths">Specific Paths (Optional):</label>
<textarea id="schedule-paths" placeholder="Leave empty to scan entire library, or specify paths separated by commas"></textarea>
<small class="help-text">
Example: /media/TV/Series Name, /media/Movies/Movie Name
</small>
</div>
<div class="form-group">
<label class="checkbox-label">
<input type="checkbox" id="schedule-enabled" checked>
<span class="checkmark"></span>
Enable this schedule
</label>
</div>
<div class="form-actions">
<button type="button" class="btn btn-secondary" onclick="closeScheduleModal()">Cancel</button>
<button type="submit" class="btn btn-primary">
<span id="schedule-submit-text">Create Schedule</span>
</button>
</div>
</form>
</div>
</div>
</div>
<!-- Cron Builder Modal -->
<div class="modal" id="cron-builder-modal">
<div class="modal-content">
<div class="modal-header">
<h3>Cron Expression Builder</h3>
<button class="modal-close" onclick="closeCronBuilder()">&times;</button>
</div>
<div class="modal-body">
<div class="cron-builder">
<div class="cron-presets">
<h4>Quick Presets:</h4>
<div class="preset-buttons">
<button type="button" class="btn btn-outline" onclick="setCronPreset('0 2 * * *')">Daily at 2 AM</button>
<button type="button" class="btn btn-outline" onclick="setCronPreset('0 2 * * 0')">Weekly (Sunday 2 AM)</button>
<button type="button" class="btn btn-outline" onclick="setCronPreset('0 2 1 * *')">Monthly (1st at 2 AM)</button>
<button type="button" class="btn btn-outline" onclick="setCronPreset('0 */6 * * *')">Every 6 Hours</button>
<button type="button" class="btn btn-outline" onclick="setCronPreset('0 */12 * * *')">Every 12 Hours</button>
</div>
</div>
<div class="cron-fields">
<h4>Custom Schedule:</h4>
<div class="field-group">
<label>Minute (0-59):</label>
<input type="text" id="cron-minute" value="0" placeholder="0">
</div>
<div class="field-group">
<label>Hour (0-23):</label>
<input type="text" id="cron-hour" value="2" placeholder="2">
</div>
<div class="field-group">
<label>Day of Month (1-31):</label>
<input type="text" id="cron-day" value="*" placeholder="*">
</div>
<div class="field-group">
<label>Month (1-12):</label>
<input type="text" id="cron-month" value="*" placeholder="*">
</div>
<div class="field-group">
<label>Day of Week (0-6):</label>
<input type="text" id="cron-dow" value="*" placeholder="*">
</div>
</div>
<div class="cron-preview">
<h4>Preview:</h4>
<div class="cron-expression">
<code id="cron-preview-text">0 2 * * *</code>
</div>
<div class="cron-description" id="cron-description">
Runs daily at 2:00 AM
</div>
</div>
</div>
<div class="form-actions">
<button type="button" class="btn btn-secondary" onclick="closeCronBuilder()">Cancel</button>
<button type="button" class="btn btn-primary" onclick="applyCronExpression()">Use This Schedule</button>
</div>
</div>
</div>
</div>
<!-- Toast Notifications -->
<div class="toast-container" id="toast-container"></div>
<script src="/static/js/app.js?v=2.10.0-skipped-imdb-edit-v2"></script>
</body>
</html>
nfoguard-web/static/js/app.js
-2458
View File
File diff suppressed because it is too large Load Diff
nfoguard.py
+2448
View File
File diff suppressed because it is too large Load Diff
processors/movie_processor.py
+163 -232
View File
@@ -11,12 +11,12 @@ from datetime import datetime, timezone
from zoneinfo import ZoneInfo
from core.database import NFOGuardDatabase
from core.nfo_manager import NFOManager
from core.path_mapper import PathMapper
from clients.radarr_client import RadarrClient
from clients.external_clients import ExternalClientManager
from config.settings import config
from utils.logging import _log
from utils.imdb_utils import find_imdb_in_directory # Phase 3: Replaced NFOManager
from utils.file_utils import find_media_path_by_imdb_and_title
@@ -68,9 +68,9 @@ def convert_utc_to_local(utc_iso_string: str) -> str:
class MovieProcessor:
"""Handles movie processing"""
def __init__(self, db: NFOGuardDatabase, nfo_manager, path_mapper: PathMapper):
# nfo_manager parameter kept for backward compatibility but no longer used (Phase 3)
def __init__(self, db: NFOGuardDatabase, nfo_manager: NFOManager, path_mapper: PathMapper):
self.db = db
self.nfo_manager = nfo_manager
self.path_mapper = path_mapper
self.radarr = RadarrClient(
os.environ.get("RADARR_URL", ""),
@@ -88,75 +88,12 @@ class MovieProcessor:
path_mapper=self.path_mapper
)
def should_skip_movie(self, imdb_id: str, movie_name: str = "") -> Tuple[bool, str]:
"""
Determine if we should skip processing this movie based on completion status
Args:
imdb_id: Movie IMDb ID
movie_name: Movie name for logging
Returns:
(should_skip: bool, reason: str)
"""
try:
with self.db.get_connection() as conn:
cursor = conn.cursor()
if self.db.db_type == "postgresql":
cursor.execute("""
SELECT dateadded, source, has_video_file
FROM movies
WHERE imdb_id = %s
""", (imdb_id,))
else:
cursor.execute("""
SELECT dateadded, source, has_video_file
FROM movies
WHERE imdb_id = ?
""", (imdb_id,))
result = cursor.fetchone()
if not result:
return False, "No database record found"
if self.db.db_type == "postgresql":
dateadded = result['dateadded']
source = result['source']
has_video_file = result['has_video_file']
else:
dateadded = result[0] if result[0] else None
source = result[1] if result[1] else None
has_video_file = result[2] if result[2] else False
# Skip if:
# 1. Movie has a valid dateadded timestamp
# 2. Source is valid (not 'unknown' or 'no_valid_date_source')
# 3. Has video file on disk
if (dateadded and
source and
source not in ['unknown', 'no_valid_date_source'] and
has_video_file):
return True, f"Complete: Has valid date '{dateadded}' from source '{source}'"
elif not dateadded:
return False, "Missing dateadded"
elif not source or source in ['unknown', 'no_valid_date_source']:
return False, f"Invalid source: '{source}'"
elif not has_video_file:
return False, "No video file detected"
else:
return False, "Incomplete movie data"
except Exception as e:
_log("ERROR", f"Error checking movie completion for {imdb_id}: {e}")
return False, f"Error checking completion: {e}"
def process_movie(self, movie_path: Path, webhook_mode: bool = False, force_scan: bool = False, scan_mode: str = "smart", shutdown_event=None) -> str:
def process_movie(self, movie_path: Path, webhook_mode: bool = False) -> None:
"""Process a movie directory"""
imdb_id = find_imdb_in_directory(movie_path) # Phase 3: Using imdb_utils instead of NFOManager
imdb_id = self.nfo_manager.find_movie_imdb_id(movie_path)
if not imdb_id:
_log("ERROR", f"No IMDb ID found in movie directory, filenames, or NFO file: {movie_path}")
return "error"
return
# Handle TMDB ID fallback case
is_tmdb_fallback = imdb_id.startswith("tmdb-")
@@ -165,27 +102,6 @@ class MovieProcessor:
else:
_log("INFO", f"Processing movie: {movie_path.name} (IMDb: {imdb_id})")
# Check if we should skip this movie (unless forced, webhook mode, or incomplete mode)
# Skip database optimization for incomplete mode since we need to check NFO files first
if not force_scan and not webhook_mode and scan_mode != "incomplete":
should_skip, reason = self.should_skip_movie(imdb_id, movie_path.name)
if should_skip:
_log("INFO", f"⏭️ SKIPPING MOVIE: {movie_path.name} [{imdb_id}] - {reason}")
# Still update the movie record to track that we've seen it
self.db.upsert_movie(imdb_id, str(movie_path))
return "skipped"
else:
_log("INFO", f"🎬 PROCESSING MOVIE: {movie_path.name} [{imdb_id}] - {reason}")
elif force_scan:
_log("INFO", f"🔄 FORCE PROCESSING MOVIE: {movie_path.name} [{imdb_id}] - Force scan enabled")
else:
_log("INFO", f"📥 WEBHOOK PROCESSING MOVIE: {movie_path.name} [{imdb_id}] - Webhook mode")
# Check for shutdown signal early in processing
if shutdown_event and shutdown_event.is_set():
_log("INFO", f"⚠️ SHUTDOWN SIGNAL RECEIVED - Stopping movie processing: {movie_path.name}")
return "shutdown"
# Update database
self.db.upsert_movie(imdb_id, str(movie_path))
@@ -194,73 +110,109 @@ class MovieProcessor:
has_video = any(f.is_file() and f.suffix.lower() in video_exts for f in movie_path.iterdir())
if not has_video:
_log("WARNING", f"No video files found in: {movie_path} - skipping database entry")
return "no_video_files"
_log("WARNING", f"No video files found in: {movie_path}")
self.db.upsert_movie_dates(imdb_id, None, None, None, False)
return
# For incomplete mode: Start with NFO check to find missing dateadded elements
if scan_mode == "incomplete":
return self._process_movie_nfo_first(movie_path, imdb_id, shutdown_event)
# For smart/full modes: Use database-first optimization
# TIER 1: Check database first (fastest - local lookup)
# TIER 1: Check if NFO file already has NFOGuard data (fastest - no DB or API calls)
nfo_path = movie_path / "movie.nfo"
nfo_data = self.nfo_manager.extract_nfoguard_dates_from_nfo(nfo_path)
if nfo_data:
_log("INFO", f"🚀 Using existing NFOGuard data from NFO file: {nfo_data['dateadded']} (source: {nfo_data['source']})")
dateadded = nfo_data["dateadded"]
source = nfo_data["source"]
released = nfo_data.get("released")
# Update file mtimes if enabled (NFO is already correct)
if config.fix_dir_mtimes and dateadded:
self.nfo_manager.update_movie_files_mtime(movie_path, dateadded)
_log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [nfo-only]")
return
# TIER 1.5: Special handling for TMDB-only movies - extract dates from existing NFO
if is_tmdb_fallback:
tmdb_nfo_data = self._extract_dates_from_tmdb_nfo(nfo_path)
if tmdb_nfo_data:
_log("INFO", f"🎬 Using TMDB data from existing NFO file: {tmdb_nfo_data['dateadded']} (source: {tmdb_nfo_data['source']})")
dateadded = tmdb_nfo_data["dateadded"]
source = tmdb_nfo_data["source"]
released = tmdb_nfo_data.get("released")
# Create NFO with NFOGuard fields added
if config.manage_nfo:
self.nfo_manager.create_movie_nfo(
movie_path, imdb_id, dateadded, released, source, config.lock_metadata
)
# Update file mtimes if enabled
if config.fix_dir_mtimes and dateadded:
self.nfo_manager.update_movie_files_mtime(movie_path, dateadded)
# Save to database
self.db.upsert_movie_dates(imdb_id, released, dateadded, source, True)
_log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [tmdb-nfo]")
return
# TIER 2: Check database for existing data
existing = self.db.get_movie_dates(imdb_id)
_log("DEBUG", f"Database lookup for {imdb_id}: {existing}")
# Enhanced debug for database state
if existing:
has_dateadded = bool(existing.get("dateadded"))
source_value = existing.get("source")
_log("INFO", f"🔍 TIER 1 DEBUG - {imdb_id}: has_dateadded={has_dateadded}, source='{source_value}', dateadded='{existing.get('dateadded')}'")
else:
_log("INFO", f"🔍 TIER 1 DEBUG - {imdb_id}: No database record found")
# If we have complete data in database, use it and skip all other checks
# If we have complete data in database, use it and skip expensive API calls
if existing and existing.get("dateadded") and existing.get("source") != "no_valid_date_source":
_log("INFO", f"✅ TIER 1 - Using complete database data for {imdb_id}: {existing['dateadded']} (source: {existing['source']})")
_log("INFO", f"✅ Using complete database data for {imdb_id}: {existing['dateadded']} (source: {existing['source']})")
# Still create NFO and update files but skip API queries
dateadded, source, released = existing["dateadded"], existing["source"], existing.get("released")
# Convert datetime objects to strings for NFO manager
if hasattr(dateadded, 'isoformat'):
dateadded = dateadded.isoformat()
if released and hasattr(released, 'isoformat'):
released = released.isoformat()
# Create NFO with existing data
if config.manage_nfo:
self.nfo_manager.create_movie_nfo(
movie_path, imdb_id, dateadded, released, source, config.lock_metadata
)
# NFO file operations removed - database is now the single source of truth
# (Phase 1: Remove NFO file write operations)
# Update file mtimes if enabled
if config.fix_dir_mtimes and dateadded:
self.nfo_manager.update_movie_files_mtime(movie_path, dateadded)
_log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [database-cached]")
return "processed"
else:
_log("INFO", f"🔍 TIER 1 SKIP - {imdb_id}: Database incomplete, proceeding to Tier 2")
_log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [database-only]")
return
# TIER 2: Query external APIs directly (NFO layer removed in Phase 2)
_log("INFO", f"🔍 TIER 2 - No database cache, querying external APIs")
# Check for shutdown signal before expensive API operations
if shutdown_event and shutdown_event.is_set():
_log("INFO", f"⚠️ SHUTDOWN SIGNAL RECEIVED - Stopping movie processing before API calls: {movie_path.name}")
return "shutdown"
# TIER 3: No cached data found - determine if we should query APIs
# Handle webhook mode - prioritize database, then use proper date logic
if webhook_mode:
_log("INFO", f"Webhook processing - no cached data found, using full date decision logic")
should_query = True # Always query for webhooks when no cached data exists
_log("DEBUG", f"Webhook mode: existing={bool(existing)}, has_dateadded={bool(existing and existing.get('dateadded')) if existing else 'N/A'}")
if existing and existing.get("dateadded"):
_log("INFO", f"Webhook processing - using existing database entry: {existing['dateadded']} (source: {existing.get('source', 'unknown')})")
dateadded, source, released = existing["dateadded"], existing["source"], existing.get("released")
else:
if existing:
_log("INFO", f"Webhook processing - database entry exists but no dateadded field: {existing}")
else:
_log("INFO", f"Webhook processing - no database entry found for {imdb_id}")
_log("INFO", f"Using full date decision logic")
# Use same logic as manual scan to check Radarr import dates, release dates, etc.
should_query = True # Always query for webhooks when no database entry exists
dateadded, source, released = self._decide_movie_dates(imdb_id, movie_path, should_query, existing)
# Only if ALL date sources fail, fall back to current timestamp
if dateadded is None:
local_tz = _get_local_timezone()
current_time = datetime.now(local_tz).isoformat(timespec="seconds")
_log("INFO", f"Webhook processing - all date sources failed, using current timestamp as last resort: {current_time}")
dateadded, source = current_time, "webhook:fallback_timestamp"
else:
# Manual scan mode - determine if we should query APIs
should_query = config.movie_poll_mode == "always"
_log("DEBUG", f"Movie {imdb_id}: should_query={should_query}, poll_mode={config.movie_poll_mode}")
# Use existing movie date decision logic
# Pass NFO fallback data if available for cases where external APIs don't have import history
nfo_fallback = locals().get('nfo_fallback_data', None)
dateadded, source, released = self._decide_movie_dates(imdb_id, movie_path, should_query, nfo_fallback)
# Webhook fallback: if ALL date sources fail, use current timestamp
if webhook_mode and dateadded is None:
local_tz = _get_local_timezone()
current_time = datetime.now(local_tz).isoformat(timespec="seconds")
_log("INFO", f"Webhook processing - all date sources failed, using current timestamp as last resort: {current_time}")
dateadded, source = current_time, "webhook:fallback_timestamp"
should_query = (
config.movie_poll_mode == "always" or
(config.movie_poll_mode == "if_missing" and not existing) or
(config.movie_poll_mode == "if_missing" and existing and existing.get("source") == "file:mtime") or
(config.movie_poll_mode == "if_missing" and existing and not existing.get("dateadded"))
)
_log("DEBUG", f"Movie {imdb_id}: should_query={should_query}, poll_mode={config.movie_poll_mode}, existing={bool(existing)}, has_dateadded={bool(existing and existing.get('dateadded')) if existing else False}")
# Use existing movie date decision logic
dateadded, source, released = self._decide_movie_dates(imdb_id, movie_path, should_query, existing)
# If we don't have an import/download date but we have a release date, use it as dateadded
# This ensures we save digital release dates, theatrical dates, etc. to the database
@@ -272,14 +224,17 @@ class MovieProcessor:
final_source = f"{source}_as_dateadded" if source else "release_date_fallback"
_log("INFO", f"Using release date as dateadded: {final_dateadded} (source: {final_source})")
# NFO file operations removed - database is now the single source of truth
# (Phase 1: Remove NFO file write operations)
# Create NFO regardless of date availability (preserves existing metadata)
if config.manage_nfo:
self.nfo_manager.create_movie_nfo(
movie_path, imdb_id, final_dateadded, released, final_source, config.lock_metadata
)
# Skip remaining processing if no valid date found and file dates disabled
if final_dateadded is None:
_log("WARNING", f"Movie {movie_path.name} - no valid date source available, but NFO was still processed")
self.db.upsert_movie_dates(imdb_id, released, None, source, True)
return "processed"
return
# Update dateadded and source for the rest of processing
dateadded = final_dateadded
@@ -287,12 +242,12 @@ class MovieProcessor:
_log("DEBUG", f"Movie {movie_path.name} proceeding to save: dateadded={dateadded}, source={source}")
# File mtime operations removed - database is now the single source of truth
# (Phase 1: Remove NFO file write operations)
# Update file mtimes (only if we have a valid date)
if config.fix_dir_mtimes and dateadded and dateadded != "MANUAL_REVIEW_NEEDED":
self.nfo_manager.update_movie_files_mtime(movie_path, dateadded)
_log("DEBUG", f"Movie processing reached file mtime section: fix_dir_mtimes={config.fix_dir_mtimes}, dateadded={dateadded}")
# Save to database
_log("DEBUG", f"About to save to database: imdb_id={imdb_id}, dateadded={dateadded}")
try:
@@ -303,78 +258,32 @@ class MovieProcessor:
raise
_log("INFO", f"Completed processing movie: {movie_path.name} (source: {source})")
return "processed"
def _process_movie_nfo_first(self, movie_path: Path, imdb_id: str, shutdown_event=None) -> str:
"""Process movie for incomplete mode: Database-first then API (NFO checks removed in Phase 2)"""
_log("INFO", f"🔍 INCOMPLETE MODE: Checking movie for missing data: {movie_path.name}")
# Check for shutdown signal
if shutdown_event and shutdown_event.is_set():
_log("INFO", f"⚠️ SHUTDOWN SIGNAL RECEIVED - Stopping movie processing: {movie_path.name}")
return "shutdown"
# STEP 1: Check database for existing data (Phase 2: NFO check removed)
_log("DEBUG", f"STEP 1 - Checking database for existing data")
existing = self.db.get_movie_dates(imdb_id)
if existing and existing.get("dateadded") and existing.get("source") != "no_valid_date_source":
# Found in database - data is complete
_log("INFO", f"✅ Database has dateadded={existing['dateadded']}")
dateadded, source, released = existing["dateadded"], existing["source"], existing.get("released")
# Convert datetime objects to strings
if hasattr(dateadded, 'isoformat'):
dateadded = dateadded.isoformat()
if released and hasattr(released, 'isoformat'):
released = released.isoformat()
_log("INFO", f"Completed processing movie: {movie_path.name} (source: {source}) [database-cached]")
return "processed"
# STEP 2: Database incomplete or missing, query APIs
_log("DEBUG", f"STEP 2 - Querying APIs for missing data")
# Check for shutdown signal before API calls
if shutdown_event and shutdown_event.is_set():
_log("INFO", f"⚠️ SHUTDOWN SIGNAL RECEIVED - Stopping before API calls: {movie_path.name}")
return "shutdown"
# Handle TMDB ID fallback case
is_tmdb_fallback = imdb_id.startswith("tmdb-")
if is_tmdb_fallback:
# TMDB fallback processing - use file modification time
_log("INFO", f"🔍 TMDB fallback processing for {imdb_id}")
dateadded, source, released = self._get_file_mtime_date(movie_path)
_log("INFO", f"Using file mtime for TMDB movie: {dateadded}")
else:
# Standard IMDb processing
# Try to get digital release date from external APIs
digital_date, digital_source = self._get_digital_release_date(imdb_id)
def _extract_dates_from_tmdb_nfo(self, nfo_path: Path) -> Optional[Dict[str, str]]:
"""Extract date information from TMDB-based NFO file"""
if not nfo_path.exists():
return None
if digital_date:
dateadded = digital_date
source = digital_source
released = digital_date # For movies, digital release is often the key date
_log("INFO", f"Got digital release date from APIs: {dateadded} (source: {source})")
else:
# Last resort: file modification time
dateadded, source, released = self._get_file_mtime_date(movie_path)
_log("INFO", f"Using file mtime as fallback: {dateadded}")
# Save to database only (NFO operations removed in Phase 1)
if dateadded:
self.db.upsert_movie_dates(imdb_id, released, dateadded, source, True)
try:
root = self.nfo_manager._parse_nfo_with_tolerance(nfo_path)
_log("INFO", f"🔍 INCOMPLETE MODE COMPLETE: {movie_path.name} (source: {source})")
return "processed"
else:
_log("WARNING", f"Could not determine dateadded for movie: {movie_path.name}")
return "error"
# Look for premiered date (from TMDB)
premiered_elem = root.find('.//premiered')
if premiered_elem is not None and premiered_elem.text:
premiered_date = premiered_elem.text.strip()
print(f"✅ Found TMDB premiered date: {premiered_date}")
return {
"dateadded": premiered_date,
"source": "tmdb:premiered_from_nfo",
"released": premiered_date
}
except (ET.ParseError, Exception) as e:
print(f"⚠️ Error parsing TMDB NFO for dates: {e}")
return None
# NFO helper methods removed in Phase 2 - database is the single source of truth
def _decide_movie_dates(self, imdb_id: str, movie_path: Path, should_query: bool, existing: Optional[Dict]) -> Tuple[str, str, Optional[str]]:
"""Decide movie dates based on configuration and available data"""
_log("DEBUG", f"_decide_movie_dates for {imdb_id}: should_query={should_query}, existing={existing}")
@@ -422,8 +331,7 @@ class MovieProcessor:
# Compare dates - prefer release date if it's reasonable
if self._should_prefer_release_over_file_date(digital_date, digital_source, released, imdb_id):
_log("INFO", f"✅ Movie {imdb_id}: Preferring digital release date {digital_date} over file date")
# When using digital release date, store it as both dateadded and released
return digital_date, digital_source, digital_date
return digital_date, digital_source, released
else:
# Convert file date to local timezone for NFO files
local_file_date = convert_utc_to_local(import_date)
@@ -438,17 +346,21 @@ class MovieProcessor:
return local_import_date, import_source, released
elif digital_date:
_log("INFO", f"✅ Movie {imdb_id}: Using digital release date {digital_date} from {digital_source}")
# When using digital release date, store it as both dateadded and released
return digital_date, digital_source, digital_date
return digital_date, digital_source, released
else:
_log("WARNING", f"⚠️ Movie {imdb_id}: No import date OR digital release date found")
_log("WARNING", f"⚠️ Movie {imdb_id}: No import date OR digital release date found - trying additional fallbacks")
# Try Radarr's own NFO premiered date as fallback
radarr_premiered = self._get_radarr_nfo_premiered_date(movie_path)
if radarr_premiered:
_log("INFO", f"✅ Movie {imdb_id}: Using Radarr NFO premiered date {radarr_premiered}")
return radarr_premiered, "radarr:nfo.premiered", released
else: # digital_then_import
# Try digital release first
digital_date, digital_source = self._get_digital_release_date(imdb_id)
if digital_date:
# When using digital release date, store it as both dateadded and released
return digital_date, digital_source, digital_date
return digital_date, digital_source, released
# Fall back to import history
if radarr_movie:
@@ -460,11 +372,6 @@ class MovieProcessor:
local_import_date = convert_utc_to_local(import_date)
return local_import_date, import_source, released
# Last resort: check if we have NFO fallback data (when external APIs don't have import history)
if existing and existing.get('dateadded'):
_log("INFO", f"✅ Movie {imdb_id}: External APIs don't have import history, using NFO fallback date: {existing['dateadded']} (source: {existing['source']})")
return existing["dateadded"], f"nfo_fallback:{existing['source']}", existing.get("released")
# Last resort: file mtime (if allowed)
if config.allow_file_date_fallback:
return self._get_file_mtime_date(movie_path)
@@ -500,8 +407,32 @@ class MovieProcessor:
_log("ERROR", f"External clients error for {imdb_id}: {e}")
return None, f"release:error:{str(e)}"
# _get_radarr_nfo_premiered_date() removed in Phase 2 - no longer reading NFO files
def _get_radarr_nfo_premiered_date(self, movie_path: Path) -> Optional[str]:
"""Extract premiered date from Radarr's existing movie.nfo file"""
try:
nfo_path = movie_path / "movie.nfo"
if not nfo_path.exists():
_log("DEBUG", f"No existing NFO file found at {nfo_path}")
return None
nfo_content = nfo_path.read_text(encoding='utf-8')
# Look for <premiered>YYYY-MM-DD</premiered>
match = re.search(r'<premiered>(\d{4}-\d{2}-\d{2})</premiered>', nfo_content)
if match:
premiered_date = match.group(1)
# Convert to ISO format with timezone
iso_date = f"{premiered_date}T00:00:00+00:00"
_log("INFO", f"✅ Found Radarr NFO premiered date: {premiered_date}")
return iso_date
else:
_log("DEBUG", f"No <premiered> tag found in existing NFO")
return None
except Exception as e:
_log("ERROR", f"Error reading Radarr NFO file: {e}")
return None
def _log_failed_movie(self, movie_path: Path, imdb_id: str, reason: str, available_countries: List[str] = None):
"""Log movies that failed to get valid dates to a debug file"""
try:
processors/tv_processor.py
+85 -868
View File
File diff suppressed because it is too large Load Diff
processors/tv_series_processor.py
+36 -112
View File
@@ -19,8 +19,8 @@ from core.logging import _log, convert_utc_to_local
class TVSeriesProcessor:
"""Clean TV series processor with video filename matching"""
def __init__(self, db: NFOGuardDatabase, nfo_manager: NFOManager,
def __init__(self, db: NFOGuardDatabase, nfo_manager: NFOManager,
path_mapper: PathMapper, sonarr_client: SonarrClient):
self.db = db
self.nfo_manager = nfo_manager # Keep for series/season NFOs
@@ -28,19 +28,6 @@ class TVSeriesProcessor:
self.path_mapper = path_mapper
self.sonarr = sonarr_client
self.external_manager = ExternalClientManager()
# Initialize Sonarr database client for high-performance queries
self.sonarr_db = None
try:
from clients.sonarr_db_client import SonarrDbClient
self.sonarr_db = SonarrDbClient.from_env()
if self.sonarr_db:
_log("INFO", "✅ SONARR DB MODE: Direct database access enabled")
else:
_log("INFO", "⚠️ SONARR API MODE: Using API (configure SONARR_DB_TYPE for better performance)")
except Exception as e:
_log("WARNING", f"Sonarr DB client initialization failed, using API: {e}")
self.sonarr_db = None
def process_series_manual_scan(self, series_path: Path) -> bool:
"""Process a TV series during manual scan"""
@@ -205,19 +192,10 @@ class TVSeriesProcessor:
aired, dateadded, source = self._get_episode_dates_from_sonarr(imdb_id, season_num, episode_num)
# Step 4: Create/update NFO and database
if dateadded or aired:
if dateadded:
# Get episode metadata for title/plot
title, plot = self._get_episode_metadata_from_sonarr(imdb_id, season_num, episode_num)
# Use aired date as dateadded if no import date found (user requirement)
if not dateadded and aired:
dateadded = aired
if source == "no_data_found":
source = "tmdb:air_date_fallback"
else:
source = f"{source}_used_as_dateadded"
_log("INFO", f"Using aired date as dateadded for S{season_num:02d}E{episode_num:02d}: {dateadded}")
# Create/update NFO with video filename
success = self.episode_nfo_manager.create_episode_nfo(
season_dir, season_num, episode_num, aired, dateadded, source, title, plot
@@ -232,66 +210,15 @@ class TVSeriesProcessor:
return False
def _get_episode_dates_from_sonarr(self, imdb_id: str, season_num: int, episode_num: int) -> Tuple[Optional[str], Optional[str], str]:
"""Get episode dates from Sonarr (DB first, then API fallback)"""
"""Get episode dates from Sonarr API with proper fallback"""
aired = None
dateadded = None
source = "no_data_found"
# TIER 1: Try Sonarr database (high performance)
if self.sonarr_db:
try:
# Find series by IMDb
series = self.sonarr_db.get_series_by_imdb(imdb_id)
if series:
series_id = series['id']
# Get all episodes for series
episodes = self.sonarr_db.get_all_episodes_for_series(series_id)
# Find target episode
target_episode = None
for ep in episodes:
if ep['season'] == season_num and ep['episode'] == episode_num:
target_episode = ep
break
if target_episode:
# Get air date
aired = target_episode.get('air_date')
# Try to get import date from history
episode_id = target_episode['id']
import_date, import_source = self.sonarr_db.get_episode_import_date(episode_id)
if import_date:
dateadded = import_date
source = import_source
_log("INFO", f"✅ DB: Found import date for S{season_num:02d}E{episode_num:02d}: {dateadded}")
return aired, dateadded, source
# Fallback to episode file date
file_date = self.sonarr_db.get_episode_file_date(series_id, season_num, episode_num)
if file_date:
dateadded = file_date
source = "sonarr:db.file.dateAdded"
_log("INFO", f"✅ DB: Using file date for S{season_num:02d}E{episode_num:02d}: {dateadded}")
return aired, dateadded, source
# Use air date as fallback
if aired:
dateadded = convert_utc_to_local(aired)
source = "sonarr:db.airDate"
_log("WARNING", f"⚠️ DB: No import date, using air date for S{season_num:02d}E{episode_num:02d}: {dateadded}")
return aired, dateadded, source
except Exception as e:
_log("WARNING", f"Sonarr DB query failed for S{season_num:02d}E{episode_num:02d}, falling back to API: {e}")
# TIER 2: Fall back to Sonarr API
if not self.sonarr.enabled:
_log("WARNING", "Sonarr not enabled, cannot get episode dates")
return aired, dateadded, source
try:
# Find series in Sonarr using lookup endpoint first
series = self.sonarr.series_by_imdb(imdb_id)
@@ -303,44 +230,42 @@ class TVSeriesProcessor:
if not series:
_log("WARNING", f"Series not found via direct search either for IMDb: {imdb_id}")
# Fall through to external API fallback
if series:
# Get episodes for series
episodes = self.sonarr.episodes_for_series(series["id"])
target_episode = None
for episode in episodes:
if (episode.get("seasonNumber") == season_num and
if (episode.get("seasonNumber") == season_num and
episode.get("episodeNumber") == episode_num):
target_episode = episode
break
if not target_episode:
_log("WARNING", f"Episode S{season_num:02d}E{episode_num:02d} not found in Sonarr")
# Don't return here - fall through to external API fallback
else:
# Get airdate
aired = target_episode.get("airDateUtc")
# Try to get import history
episode_id = target_episode.get("id")
if episode_id:
import_date = self.sonarr.get_episode_import_history(episode_id)
if import_date:
# Sonarr import dates are already in local timezone despite 'Z' suffix
# Remove 'Z' and use as-is to avoid double timezone conversion
dateadded = import_date.replace('Z', '') if 'Z' in import_date else import_date
source = "sonarr:api.history.import"
_log("INFO", f"✅ API: Found import date for S{season_num:02d}E{episode_num:02d}: {dateadded}")
dateadded = convert_utc_to_local(import_date)
source = "sonarr:history.import"
_log("INFO", f"Found import date for S{season_num:02d}E{episode_num:02d}: {dateadded}")
return aired, dateadded, source
# Fallback to airdate if no import history
if aired:
dateadded = convert_utc_to_local(aired)
source = "sonarr:api.episode.airDateUtc"
_log("WARNING", f"⚠️ API: No import history for S{season_num:02d}E{episode_num:02d}, using airdate: {dateadded}")
source = "sonarr:episode.airDateUtc"
_log("WARNING", f"No import history for S{season_num:02d}E{episode_num:02d}, using airdate: {dateadded}")
return aired, dateadded, source
except Exception as e:
_log("ERROR", f"Sonarr API error for S{season_num:02d}E{episode_num:02d}: {e}")
@@ -435,22 +360,21 @@ class TVSeriesProcessor:
return None, "no_external_data"
def _create_series_nfos(self, series_path: Path, imdb_id: str):
"""DISABLED: Skip creating tvshow.nfo and season.nfo files (user only wants episode NFOs)"""
# # Create tvshow.nfo only if it doesn't exist
# tvshow_nfo = series_path / "tvshow.nfo"
# if not tvshow_nfo.exists():
# self.nfo_manager.create_tvshow_nfo(series_path, imdb_id)
# else:
# _log("DEBUG", f"Skipping tvshow.nfo creation - already exists: {tvshow_nfo}")
#
# # Create season.nfo for each season directory only if they don't exist
# for season_dir in series_path.iterdir():
# if season_dir.is_dir() and self._is_season_directory(season_dir.name):
# season_num = self._extract_season_number(season_dir.name)
# if season_num is not None:
# season_nfo = season_dir / "season.nfo"
# if not season_nfo.exists():
# self.nfo_manager.create_season_nfo(season_dir, season_num)
# else:
# _log("DEBUG", f"Skipping season.nfo creation - already exists: {season_nfo}")
pass # Function disabled - only process episode NFOs
"""Create tvshow.nfo and season.nfo files only if they don't exist"""
# Create tvshow.nfo only if it doesn't exist
tvshow_nfo = series_path / "tvshow.nfo"
if not tvshow_nfo.exists():
self.nfo_manager.create_tvshow_nfo(series_path, imdb_id)
else:
_log("DEBUG", f"Skipping tvshow.nfo creation - already exists: {tvshow_nfo}")
# Create season.nfo for each season directory only if they don't exist
for season_dir in series_path.iterdir():
if season_dir.is_dir() and self._is_season_directory(season_dir.name):
season_num = self._extract_season_number(season_dir.name)
if season_num is not None:
season_nfo = season_dir / "season.nfo"
if not season_nfo.exists():
self.nfo_manager.create_season_nfo(season_dir, season_num)
else:
_log("DEBUG", f"Skipping season.nfo creation - already exists: {season_nfo}")
requirements.txt
-4
View File
@@ -5,8 +5,4 @@ psycopg2-binary==2.9.7
requests==2.31.0
python-multipart==0.0.6
aiofiles==23.2.1
aiohttp==3.8.6
psutil==5.9.6
python-dotenv==1.0.0
APScheduler==3.10.4
croniter==1.4.1
scheduler/__init__.py
-1
View File
@@ -1 +0,0 @@
# Scheduler module for NFOGuard
scheduler/scheduler.py
-396
View File
@@ -1,396 +0,0 @@
"""
NFOGuard Background Scheduler
Manages scheduled scans using APScheduler with cron-like functionality
"""
import logging
import asyncio
from datetime import datetime, timezone
from typing import Dict, Any, Optional
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from apscheduler.triggers.cron import CronTrigger
from apscheduler.jobstores.memory import MemoryJobStore
from apscheduler.executors.asyncio import AsyncIOExecutor
logger = logging.getLogger(__name__)
class NFOGuardScheduler:
"""
Background scheduler for NFOGuard that manages scheduled scans
"""
def __init__(self, dependencies: Dict[str, Any]):
"""Initialize the scheduler with dependencies"""
self.dependencies = dependencies
self.scheduler = None
self.running = False
# Configure APScheduler
jobstores = {
'default': MemoryJobStore()
}
executors = {
'default': AsyncIOExecutor()
}
job_defaults = {
'coalesce': False,
'max_instances': 1,
'misfire_grace_time': 300 # 5 minutes
}
self.scheduler = AsyncIOScheduler(
jobstores=jobstores,
executors=executors,
job_defaults=job_defaults,
timezone='UTC'
)
async def start(self):
"""Start the scheduler and load existing schedules"""
if self.running:
logger.warning("Scheduler is already running")
return
try:
self.scheduler.start()
self.running = True
logger.info("✅ NFOGuard Scheduler started successfully")
# Load existing scheduled scans from database
await self.load_schedules()
except Exception as e:
logger.error(f"Failed to start scheduler: {e}")
raise
async def stop(self):
"""Stop the scheduler gracefully"""
if not self.running:
return
try:
self.scheduler.shutdown()
self.running = False
logger.info("✅ NFOGuard Scheduler stopped successfully")
except Exception as e:
logger.error(f"Error stopping scheduler: {e}")
async def load_schedules(self):
"""Load all enabled scheduled scans from database and add them to scheduler"""
try:
db = self.dependencies.get("db")
if not db:
logger.error("Database not available for loading schedules")
return
# Get all enabled scheduled scans
scheduled_scans = db.get_scheduled_scans(enabled_only=True)
for scan in scheduled_scans:
await self.add_schedule(scan)
logger.info(f"Loaded {len(scheduled_scans)} scheduled scans")
except Exception as e:
logger.error(f"Failed to load schedules: {e}")
async def add_schedule(self, scan: Dict[str, Any]):
"""Add a scheduled scan to the scheduler"""
try:
job_id = f"scan_{scan['id']}"
# Remove existing job if it exists
if self.scheduler.get_job(job_id):
self.scheduler.remove_job(job_id)
# Create cron trigger
trigger = CronTrigger.from_crontab(scan['cron_expression'])
# Add job to scheduler
self.scheduler.add_job(
func=self._execute_scheduled_scan,
trigger=trigger,
id=job_id,
args=[scan['id']],
name=f"Scheduled Scan: {scan['name']}",
replace_existing=True
)
# Update next run time in database
next_run = self.scheduler.get_job(job_id).next_run_time
if next_run:
db = self.dependencies.get("db")
if db:
db.update_scan_next_run(scan['id'], next_run)
logger.info(f"✅ Added scheduled scan: {scan['name']} ({scan['cron_expression']})")
except Exception as e:
logger.error(f"Failed to add schedule for scan {scan['id']}: {e}")
async def remove_schedule(self, scan_id: int):
"""Remove a scheduled scan from the scheduler"""
try:
job_id = f"scan_{scan_id}"
if self.scheduler.get_job(job_id):
self.scheduler.remove_job(job_id)
logger.info(f"✅ Removed scheduled scan: {scan_id}")
else:
logger.warning(f"No job found for scan ID: {scan_id}")
except Exception as e:
logger.error(f"Failed to remove schedule for scan {scan_id}: {e}")
async def update_schedule(self, scan: Dict[str, Any]):
"""Update an existing scheduled scan"""
try:
# Remove old schedule and add new one
await self.remove_schedule(scan['id'])
if scan['enabled']:
await self.add_schedule(scan)
except Exception as e:
logger.error(f"Failed to update schedule for scan {scan['id']}: {e}")
async def _execute_scheduled_scan(self, scan_id: int):
"""Execute a scheduled scan"""
db = self.dependencies.get("db")
if not db:
logger.error(f"Database not available for executing scan {scan_id}")
return
# Get scan details
scan = db.get_scheduled_scan(scan_id)
if not scan:
logger.error(f"Scheduled scan {scan_id} not found")
return
if not scan['enabled']:
logger.info(f"Skipping disabled scan: {scan['name']}")
return
execution_id = None
try:
logger.info(f"🚀 Starting scheduled scan: {scan['name']} (ID: {scan_id})")
# Create execution record
execution_id = db.create_schedule_execution(
schedule_id=scan_id,
media_type=scan['media_type'],
scan_mode=scan['scan_mode'],
triggered_by="scheduler"
)
# Update last run time
db.update_scan_last_run(scan_id)
# Execute the actual scan
result = await self._run_media_scan(scan, execution_id)
# Update execution with results
db.update_schedule_execution(
execution_id=execution_id,
status="completed",
items_processed=result.get('items_processed', 0),
items_skipped=result.get('items_skipped', 0),
items_failed=result.get('items_failed', 0),
logs=result.get('logs', '')
)
logger.info(f"✅ Completed scheduled scan: {scan['name']} - Processed: {result.get('items_processed', 0)}, Skipped: {result.get('items_skipped', 0)}, Failed: {result.get('items_failed', 0)}")
except Exception as e:
logger.error(f"❌ Failed scheduled scan: {scan['name']} - {e}")
if execution_id:
db.update_schedule_execution(
execution_id=execution_id,
status="failed",
error_message=str(e)
)
async def _run_media_scan(self, scan: Dict[str, Any], execution_id: int) -> Dict[str, Any]:
"""Run the actual media scan based on scan configuration"""
try:
# Import scan functionality from existing modules
from api.routes import run_tv_scan, run_movie_scan
media_type = scan['media_type']
scan_mode = scan['scan_mode']
specific_paths = scan.get('specific_paths', '').strip()
results = {
'items_processed': 0,
'items_skipped': 0,
'items_failed': 0,
'logs': []
}
# Parse specific paths if provided
paths = []
if specific_paths:
paths = [p.strip() for p in specific_paths.split(',') if p.strip()]
# Run TV scan if needed
if media_type in ['tv', 'both']:
logger.info(f"Running TV scan with mode: {scan_mode}")
# Use existing scan infrastructure
tv_result = await self._execute_tv_scan(scan_mode, paths)
results['items_processed'] += tv_result.get('tv_series_processed', 0)
results['items_skipped'] += tv_result.get('tv_series_skipped', 0)
results['items_failed'] += tv_result.get('tv_series_failed', 0)
results['logs'].append(f"TV Scan: {tv_result.get('message', 'Completed')}")
# Run movie scan if needed
if media_type in ['movies', 'both']:
logger.info(f"Running movie scan with mode: {scan_mode}")
# Use existing scan infrastructure
movie_result = await self._execute_movie_scan(scan_mode, paths)
results['items_processed'] += movie_result.get('movies_processed', 0)
results['items_skipped'] += movie_result.get('movies_skipped', 0)
results['items_failed'] += movie_result.get('movies_failed', 0)
results['logs'].append(f"Movie Scan: {movie_result.get('message', 'Completed')}")
results['logs'] = '\n'.join(results['logs'])
return results
except Exception as e:
logger.error(f"Error in media scan execution: {e}")
return {
'items_processed': 0,
'items_skipped': 0,
'items_failed': 1,
'logs': f"Scan failed: {str(e)}"
}
async def _execute_tv_scan(self, scan_mode: str, specific_paths: list = None) -> Dict[str, Any]:
"""Execute TV scan using existing infrastructure"""
try:
# This would integrate with the existing manual scan functionality
# For now, return a placeholder result
return {
'tv_series_processed': 0,
'tv_series_skipped': 0,
'tv_series_failed': 0,
'message': f'TV scan ({scan_mode}) - Integration pending'
}
except Exception as e:
logger.error(f"TV scan execution failed: {e}")
return {
'tv_series_processed': 0,
'tv_series_skipped': 0,
'tv_series_failed': 1,
'message': f'TV scan failed: {str(e)}'
}
async def _execute_movie_scan(self, scan_mode: str, specific_paths: list = None) -> Dict[str, Any]:
"""Execute movie scan using existing infrastructure"""
try:
# This would integrate with the existing manual scan functionality
# For now, return a placeholder result
return {
'movies_processed': 0,
'movies_skipped': 0,
'movies_failed': 0,
'message': f'Movie scan ({scan_mode}) - Integration pending'
}
except Exception as e:
logger.error(f"Movie scan execution failed: {e}")
return {
'movies_processed': 0,
'movies_skipped': 0,
'movies_failed': 1,
'message': f'Movie scan failed: {str(e)}'
}
async def run_manual_scan(self, scan_id: int) -> Dict[str, Any]:
"""Manually trigger a scheduled scan"""
try:
db = self.dependencies.get("db")
scan = db.get_scheduled_scan(scan_id)
if not scan:
return {
'success': False,
'error': 'Scheduled scan not found'
}
# Execute the scan in the background
asyncio.create_task(self._execute_scheduled_scan(scan_id))
return {
'success': True,
'message': f"Manual execution of '{scan['name']}' started"
}
except Exception as e:
logger.error(f"Failed to run manual scan {scan_id}: {e}")
return {
'success': False,
'error': str(e)
}
def get_job_status(self, scan_id: int) -> Optional[Dict[str, Any]]:
"""Get the status of a scheduled job"""
try:
job_id = f"scan_{scan_id}"
job = self.scheduler.get_job(job_id)
if not job:
return None
return {
'id': job.id,
'name': job.name,
'next_run_time': job.next_run_time.isoformat() if job.next_run_time else None,
'trigger': str(job.trigger)
}
except Exception as e:
logger.error(f"Failed to get job status for scan {scan_id}: {e}")
return None
def list_jobs(self) -> list:
"""List all scheduled jobs"""
try:
jobs = []
for job in self.scheduler.get_jobs():
jobs.append({
'id': job.id,
'name': job.name,
'next_run_time': job.next_run_time.isoformat() if job.next_run_time else None,
'trigger': str(job.trigger)
})
return jobs
except Exception as e:
logger.error(f"Failed to list jobs: {e}")
return []
# Global scheduler instance
scheduler_instance: Optional[NFOGuardScheduler] = None
async def get_scheduler(dependencies: Dict[str, Any]) -> NFOGuardScheduler:
"""Get or create the global scheduler instance"""
global scheduler_instance
if scheduler_instance is None:
scheduler_instance = NFOGuardScheduler(dependencies)
await scheduler_instance.start()
return scheduler_instance
async def shutdown_scheduler():
"""Shutdown the global scheduler instance"""
global scheduler_instance
if scheduler_instance:
await scheduler_instance.stop()
scheduler_instance = None
start_web.py
-178
View File
@@ -1,178 +0,0 @@
#!/usr/bin/env python3
"""
NFOGuard Web Interface Starter
Simple script to start web interface using existing config system
"""
import os
import sys
import time
import uvicorn
from fastapi import FastAPI
from fastapi.staticfiles import StaticFiles
from fastapi.responses import FileResponse
# Import existing configuration (keep using core config for simplicity)
from config.settings import config
# Import existing database and components
from core.database import NFOGuardDatabase
# Import web routes from existing system (now includes DELETE route)
from api.web_routes import register_web_routes
# Import authentication system
from api.auth import SimpleAuthMiddleware, AuthSession
def create_web_app() -> FastAPI:
"""Create FastAPI web application"""
app = FastAPI(
title="NFOGuard Web Interface",
description="Web interface for NFOGuard media database management",
version="2.9.0-fixes-only-files",
docs_url=None, # Disable docs in production
redoc_url=None
)
return app
def setup_static_files(app: FastAPI) -> None:
"""Mount static file directories"""
static_path = os.path.join(os.path.dirname(__file__), "nfoguard-web", "static")
logo_path = os.path.join(os.path.dirname(__file__), "logo")
print(f"🔍 Checking static path: {static_path} (exists: {os.path.exists(static_path)})")
print(f"🔍 Checking logo path: {logo_path} (exists: {os.path.exists(logo_path)})")
if os.path.exists(static_path):
app.mount("/static", StaticFiles(directory=static_path), name="static")
print(f"✅ Mounted static files from: {static_path}")
else:
print(f"❌ Static path not found: {static_path}")
if os.path.exists(logo_path):
app.mount("/logo", StaticFiles(directory=logo_path), name="logo")
print(f"✅ Mounted logo files from: {logo_path}")
else:
print(f"❌ Logo path not found: {logo_path}")
# Serve index.html at root
@app.get("/")
async def serve_index():
index_file = os.path.join(static_path, "index.html")
if os.path.exists(index_file):
return FileResponse(index_file)
else:
return {"message": "NFOGuard Web Interface", "status": "running"}
# Serve favicon
@app.get("/favicon.ico")
async def serve_favicon():
# Try to serve favicon from logo directory or static files
favicon_paths = [
os.path.join(logo_path, "favicon.ico"),
os.path.join(static_path, "favicon.ico"),
os.path.join(logo_path, "NFOGuardLogo.png") # Fallback to new logo
]
for favicon_path in favicon_paths:
if os.path.exists(favicon_path):
return FileResponse(favicon_path)
# Return 204 No Content if no favicon found
from fastapi import Response
return Response(status_code=204)
# Health check endpoint for Docker
@app.get("/health")
async def health_check():
"""Health check endpoint for Docker container monitoring"""
try:
# Basic health check - verify the web service is responsive
return {
"status": "healthy",
"service": "nfoguard-web",
"timestamp": time.time(),
"version": "2.9.0-fixes-only-files"
}
except Exception as e:
from fastapi import HTTPException
raise HTTPException(status_code=503, detail=f"Health check failed: {e}")
def main():
"""Main entry point for NFOGuard Web Interface"""
print("🌐 Starting NFOGuard Web Interface...")
# Use existing config system
web_host = os.environ.get("WEB_HOST", "0.0.0.0")
web_port = int(os.environ.get("WEB_PORT", "8081"))
print(f"📊 Configuration: Port {web_port}")
# Create FastAPI app
app = create_web_app()
# Initialize database using existing system
try:
db = NFOGuardDatabase(config)
print(f"✅ Connected to database: {config.db_host}:{config.db_port}/{config.db_name}")
except Exception as e:
print(f"❌ Failed to connect to database: {e}")
sys.exit(1)
# Setup authentication if enabled
auth_enabled = getattr(config, 'web_auth_enabled', False)
session_manager = None
if auth_enabled:
session_timeout = getattr(config, 'web_auth_session_timeout', 3600)
session_manager = AuthSession(timeout_seconds=session_timeout)
print(f"🔐 Web authentication enabled (session timeout: {session_timeout}s)")
else:
print("🌐 Web authentication disabled")
# Create dependencies for dependency injection
dependencies = {
"db": db,
"config": config,
"nfo_manager": None, # Not needed for read-only web interface
"movie_processor": None, # Not needed for read-only web interface
"tv_processor": None, # Not needed for read-only web interface
"auth_enabled": auth_enabled,
"session_manager": session_manager
}
# Add authentication middleware if enabled (BEFORE routes)
if auth_enabled:
app.add_middleware(SimpleAuthMiddleware, config=config, session_manager=session_manager)
print("🔐 Authentication middleware added to web interface")
# Setup static files and routes
setup_static_files(app)
# Register web routes (now includes DELETE /api/episodes/ route)
register_web_routes(app, dependencies)
print("✅ Registered web routes with DELETE /api/episodes/ support")
print(f"🚀 Starting web server on {web_host}:{web_port}")
try:
uvicorn.run(
app,
host=web_host,
port=web_port,
workers=1,
log_level="info",
access_log=False
)
except KeyboardInterrupt:
print("\n🛑 Web interface shutdown by user")
except Exception as e:
print(f"❌ Web interface failed: {e}")
sys.exit(1)
if __name__ == "__main__":
main()
static/css/styles.css
-902
View File
@@ -1,902 +0,0 @@
/* NFOGuard Web Interface Styles */
:root {
--primary-color: #007bff;
--secondary-color: #6c757d;
--success-color: #28a745;
--warning-color: #ffc107;
--danger-color: #dc3545;
--dark-color: #343a40;
--light-color: #f8f9fa;
--border-color: #dee2e6;
--text-muted: #6c757d;
--shadow: 0 0.125rem 0.25rem rgba(0, 0, 0, 0.075);
--shadow-lg: 0 0.5rem 1rem rgba(0, 0, 0, 0.15);
}
* {
margin: 0;
padding: 0;
box-sizing: border-box;
}
body {
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;
font-size: 14px;
line-height: 1.5;
color: var(--dark-color);
background-color: #f5f5f5;
}
.app-container {
min-height: 100vh;
display: flex;
flex-direction: column;
}
/* Header */
.app-header {
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
color: white;
padding: 1rem 0;
box-shadow: var(--shadow-lg);
position: relative;
}
.header-content {
max-width: 1200px;
margin: 0 auto;
padding: 0 1rem;
text-align: center;
}
.header-content h1 {
font-size: 2rem;
font-weight: 300;
margin-bottom: 0.5rem;
}
.header-content h1 i {
margin-right: 0.5rem;
}
.header-content p {
opacity: 0.9;
font-size: 1rem;
}
/* Authentication Status */
.auth-status {
position: absolute;
top: 1rem;
right: 1rem;
display: flex;
align-items: center;
gap: 1rem;
color: white;
font-size: 0.9rem;
}
.auth-user {
display: flex;
align-items: center;
gap: 0.5rem;
opacity: 0.9;
}
.auth-logout {
background: rgba(255, 255, 255, 0.2);
color: white;
border: 1px solid rgba(255, 255, 255, 0.3);
padding: 0.5rem 1rem;
border-radius: 0.25rem;
cursor: pointer;
font-size: 0.85rem;
display: flex;
align-items: center;
gap: 0.5rem;
transition: all 0.2s ease;
}
.auth-logout:hover {
background: rgba(255, 255, 255, 0.3);
border-color: rgba(255, 255, 255, 0.5);
transform: translateY(-1px);
}
.nav-tabs {
max-width: 1200px;
margin: 1rem auto 0;
padding: 0 1rem;
display: flex;
gap: 0.5rem;
flex-wrap: wrap;
justify-content: center;
}
.nav-tab {
background: rgba(255, 255, 255, 0.1);
border: none;
color: white;
padding: 0.75rem 1.5rem;
border-radius: 0.5rem;
cursor: pointer;
transition: all 0.3s ease;
font-size: 0.9rem;
display: flex;
align-items: center;
gap: 0.5rem;
}
.nav-tab:hover {
background: rgba(255, 255, 255, 0.2);
transform: translateY(-1px);
}
.nav-tab.active {
background: rgba(255, 255, 255, 0.9);
color: var(--dark-color);
}
/* Main Content */
.main-content {
flex: 1;
max-width: 1200px;
margin: 0 auto;
padding: 2rem 1rem;
width: 100%;
}
.tab-content {
display: none;
}
.tab-content.active {
display: block;
}
/* Dashboard */
.dashboard-grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
gap: 1.5rem;
margin-bottom: 2rem;
}
.stat-card {
background: white;
border-radius: 0.5rem;
padding: 1.5rem;
box-shadow: var(--shadow);
display: flex;
align-items: center;
gap: 1rem;
}
.stat-icon {
width: 60px;
height: 60px;
border-radius: 50%;
display: flex;
align-items: center;
justify-content: center;
font-size: 1.5rem;
color: white;
}
.stat-icon.movies { background: linear-gradient(135deg, #667eea, #764ba2); }
.stat-icon.tv { background: linear-gradient(135deg, #f093fb, #f5576c); }
.stat-icon.missing { background: linear-gradient(135deg, #ffecd2, #fcb69f); }
.stat-icon.activity { background: linear-gradient(135deg, #a8edea, #fed6e3); }
.stat-info h3 {
font-size: 2rem;
font-weight: 700;
margin-bottom: 0.25rem;
}
.stat-info p {
font-weight: 500;
margin-bottom: 0.25rem;
}
.stat-info small {
color: var(--text-muted);
font-size: 0.85rem;
}
.dashboard-charts {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
gap: 1.5rem;
}
.chart-card {
background: white;
border-radius: 0.5rem;
padding: 1.5rem;
box-shadow: var(--shadow);
}
.chart-card h3 {
margin-bottom: 1rem;
color: var(--dark-color);
}
.chart-container {
height: 200px;
display: flex;
align-items: center;
justify-content: center;
background: var(--light-color);
border-radius: 0.25rem;
color: var(--text-muted);
}
/* Content Header */
.content-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 1.5rem;
flex-wrap: wrap;
gap: 1rem;
}
.content-header h2 {
color: var(--dark-color);
font-weight: 600;
}
.content-controls {
display: flex;
gap: 1rem;
align-items: center;
flex-wrap: wrap;
}
.search-controls {
display: flex;
gap: 0.5rem;
flex-wrap: wrap;
}
.filter-controls {
display: flex;
gap: 0.5rem;
align-items: center;
flex-wrap: wrap;
}
.search-box {
position: relative;
display: flex;
align-items: center;
}
.search-box i {
position: absolute;
left: 0.75rem;
color: var(--text-muted);
}
.search-box input {
padding: 0.5rem 0.75rem 0.5rem 2.5rem;
border: 1px solid var(--border-color);
border-radius: 0.25rem;
font-size: 0.9rem;
width: 250px;
}
.search-box input:focus {
outline: none;
border-color: var(--primary-color);
box-shadow: 0 0 0 0.2rem rgba(0, 123, 255, 0.25);
}
/* Buttons */
.btn {
padding: 0.5rem 1rem;
border: none;
border-radius: 0.25rem;
cursor: pointer;
font-size: 0.9rem;
transition: all 0.2s ease;
display: inline-flex;
align-items: center;
gap: 0.5rem;
text-decoration: none;
}
.btn-primary {
background-color: var(--primary-color);
color: white;
}
.btn-primary:hover {
background-color: #0056b3;
transform: translateY(-1px);
}
.btn-secondary {
background-color: var(--secondary-color);
color: white;
}
.btn-secondary:hover {
background-color: #545b62;
}
.btn-success {
background-color: var(--success-color);
color: white;
}
.btn-success:hover {
background-color: #1e7e34;
}
.btn-warning {
background-color: var(--warning-color);
color: var(--dark-color);
}
.btn-warning:hover {
background-color: #e0a800;
}
.btn-danger {
background-color: var(--danger-color);
color: white;
}
.btn-danger:hover {
background-color: #c82333;
}
.btn-sm {
padding: 0.25rem 0.5rem;
font-size: 0.8rem;
}
/* Tables */
.table-container {
background: white;
border-radius: 0.5rem;
box-shadow: var(--shadow);
overflow: hidden;
margin-bottom: 1rem;
}
.data-table {
width: 100%;
border-collapse: collapse;
}
.data-table th,
.data-table td {
padding: 0.75rem;
text-align: left;
border-bottom: 1px solid var(--border-color);
}
.data-table th {
background-color: var(--light-color);
font-weight: 600;
color: var(--dark-color);
position: sticky;
top: 0;
}
.data-table tr:hover {
background-color: rgba(0, 123, 255, 0.05);
}
.data-table .loading {
text-align: center;
color: var(--text-muted);
font-style: italic;
padding: 2rem;
}
/* Status badges */
.badge {
padding: 0.25rem 0.5rem;
border-radius: 0.25rem;
font-size: 0.75rem;
font-weight: 600;
text-transform: uppercase;
}
.badge-success {
background-color: #d4edda;
color: #155724;
}
.badge-warning {
background-color: #fff3cd;
color: #856404;
}
.badge-danger {
background-color: #f8d7da;
color: #721c24;
}
.badge-secondary {
background-color: #e9ecef;
color: #495057;
}
/* Pagination */
.pagination {
display: flex;
justify-content: center;
align-items: center;
gap: 0.5rem;
margin-top: 1rem;
}
.pagination .btn {
padding: 0.5rem 0.75rem;
}
.pagination .page-info {
margin: 0 1rem;
color: var(--text-muted);
}
/* Forms */
.form-group {
margin-bottom: 1rem;
}
.form-group label {
display: block;
margin-bottom: 0.25rem;
font-weight: 500;
}
.form-group input,
.form-group select,
.form-group textarea {
width: 100%;
padding: 0.5rem;
border: 1px solid var(--border-color);
border-radius: 0.25rem;
font-size: 0.9rem;
}
.form-group input:focus,
.form-group select:focus,
.form-group textarea:focus {
outline: none;
border-color: var(--primary-color);
box-shadow: 0 0 0 0.2rem rgba(0, 123, 255, 0.25);
}
.form-group small {
display: block;
margin-top: 0.25rem;
color: var(--text-muted);
font-size: 0.8rem;
}
.form-actions {
display: flex;
gap: 0.5rem;
justify-content: flex-end;
margin-top: 1.5rem;
}
/* Modal */
.modal {
display: none;
position: fixed;
z-index: 1000;
left: 0;
top: 0;
width: 100%;
height: 100%;
background-color: rgba(0, 0, 0, 0.5);
}
.modal.active {
display: flex;
align-items: center;
justify-content: center;
}
.modal-content {
background: white;
border-radius: 0.5rem;
max-width: 500px;
width: 90%;
max-height: 90vh;
overflow-y: auto;
box-shadow: var(--shadow-lg);
}
.modal-header {
padding: 1rem 1.5rem;
border-bottom: 1px solid var(--border-color);
display: flex;
justify-content: space-between;
align-items: center;
}
.modal-header h3 {
margin: 0;
}
.modal-close {
background: none;
border: none;
font-size: 1.5rem;
cursor: pointer;
color: var(--text-muted);
}
.modal-close:hover {
color: var(--dark-color);
}
.modal-body {
padding: 1.5rem;
}
/* Higher z-index for edit modals that appear on top of other modals */
#edit-modal, #smart-fix-modal {
z-index: 1100 !important;
}
/* Reports */
.report-summary {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
gap: 1rem;
margin-bottom: 2rem;
}
.summary-card {
background: white;
padding: 1.5rem;
border-radius: 0.5rem;
box-shadow: var(--shadow);
text-align: center;
}
.summary-card h3 {
margin-bottom: 1rem;
color: var(--dark-color);
}
.summary-card p {
margin-bottom: 0.5rem;
font-size: 1.1rem;
}
.summary-card span {
font-weight: 700;
color: var(--primary-color);
}
.report-section {
margin-bottom: 2rem;
}
.report-section h3 {
margin-bottom: 1rem;
color: var(--dark-color);
}
/* Tools */
.tools-grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
gap: 1.5rem;
}
.tool-card {
background: white;
border-radius: 0.5rem;
padding: 1.5rem;
box-shadow: var(--shadow);
}
.tool-card h3 {
margin-bottom: 0.5rem;
color: var(--dark-color);
}
.tool-card p {
margin-bottom: 1.5rem;
color: var(--text-muted);
}
.stats-display {
background: var(--light-color);
padding: 1rem;
border-radius: 0.25rem;
margin-bottom: 1rem;
min-height: 100px;
}
/* Toast notifications */
.toast-container {
position: fixed;
top: 20px;
right: 20px;
z-index: 1050;
}
.toast {
background: white;
border-radius: 0.25rem;
box-shadow: var(--shadow-lg);
margin-bottom: 0.5rem;
padding: 0.75rem 1rem;
min-width: 300px;
border-left: 4px solid var(--primary-color);
animation: slideIn 0.3s ease;
}
.toast.success {
border-left-color: var(--success-color);
}
.toast.warning {
border-left-color: var(--warning-color);
}
.toast.error {
border-left-color: var(--danger-color);
}
@keyframes slideIn {
from {
transform: translateX(100%);
opacity: 0;
}
to {
transform: translateX(0);
opacity: 1;
}
}
/* Smart Fix Modal */
.smart-fix-options {
display: flex;
flex-direction: column;
gap: 1rem;
margin-bottom: 1rem;
}
.option-card {
border: 2px solid var(--border-color);
border-radius: 0.5rem;
transition: all 0.2s ease;
}
.option-card:hover {
border-color: var(--primary-color);
box-shadow: var(--shadow);
}
.option-label {
display: block;
padding: 1rem;
cursor: pointer;
margin: 0;
}
.option-label input[type="radio"] {
margin-right: 0.75rem;
margin-top: 0.1rem;
width: auto;
}
.option-content h4 {
margin: 0 0 0.5rem 0;
color: var(--dark-color);
font-size: 1rem;
}
.option-content p {
margin: 0 0 0.5rem 0;
color: var(--text-muted);
font-size: 0.9rem;
}
.option-content small {
color: var(--text-muted);
font-size: 0.8rem;
}
.manual-date-input {
width: 100% !important;
margin-top: 0.5rem !important;
}
.option-card input[type="radio"]:checked + .option-content {
color: var(--primary-color);
}
.option-card:has(input[type="radio"]:checked) {
border-color: var(--primary-color);
background-color: rgba(0, 123, 255, 0.05);
}
/* Additional badge styles */
.badge-info {
background-color: #d1ecf1;
color: #0c5460;
}
/* Enhanced Edit Modal Date Options */
.date-options {
display: flex;
flex-direction: column;
gap: 0.75rem;
margin-bottom: 1rem;
}
.date-option-card {
border: 1px solid var(--border-color);
border-radius: 0.375rem;
transition: all 0.2s ease;
}
.date-option-card:hover {
border-color: var(--primary-color);
box-shadow: 0 0 0 2px rgba(0, 123, 255, 0.1);
}
.date-option-label {
display: block;
padding: 0.75rem;
cursor: pointer;
margin: 0;
}
.date-option-label input[type="radio"] {
margin-right: 0.5rem;
margin-top: 0.1rem;
width: auto;
}
.date-option-content h4 {
margin: 0 0 0.25rem 0;
color: var(--dark-color);
font-size: 0.9rem;
font-weight: 600;
}
.date-option-content p {
margin: 0 0 0.25rem 0;
color: var(--text-muted);
font-size: 0.8rem;
}
.date-option-content small {
color: var(--primary-color);
font-size: 0.75rem;
font-weight: 500;
}
.date-option-card input[type="radio"]:checked + .date-option-content h4 {
color: var(--primary-color);
}
.date-option-card:has(input[type="radio"]:checked) {
border-color: var(--primary-color);
background-color: rgba(0, 123, 255, 0.03);
}
/* Responsive */
@media (max-width: 768px) {
.content-header {
flex-direction: column;
align-items: stretch;
}
.content-controls {
justify-content: center;
}
.search-box input {
width: 200px;
}
.nav-tabs {
flex-direction: column;
gap: 0.25rem;
}
.data-table {
font-size: 0.8rem;
}
.data-table th,
.data-table td {
padding: 0.5rem 0.25rem;
}
.dashboard-grid {
grid-template-columns: 1fr;
}
.tools-grid {
grid-template-columns: 1fr;
}
}
/* Utility classes */
.text-center { text-align: center; }
.text-muted { color: var(--text-muted); }
.mb-0 { margin-bottom: 0; }
.mb-1 { margin-bottom: 0.5rem; }
.mb-2 { margin-bottom: 1rem; }
.mt-1 { margin-top: 0.5rem; }
.mt-2 { margin-top: 1rem; }
.d-none { display: none; }
.d-block { display: block; }
.d-flex { display: flex; }
.justify-content-between { justify-content: space-between; }
.align-items-center { align-items: center; }
/* Manual Scan Styles */
.scan-status {
margin-top: 1rem;
padding: 1rem;
background-color: var(--light-color);
border: 1px solid var(--border-color);
border-radius: 0.375rem;
}
.scan-progress {
margin-bottom: 1rem;
}
.progress-bar {
width: 100%;
height: 1.5rem;
background-color: #e9ecef;
border-radius: 0.375rem;
overflow: hidden;
margin-bottom: 0.5rem;
}
.progress-fill {
height: 100%;
background: linear-gradient(90deg, var(--primary-color), var(--success-color));
transition: width 0.3s ease;
width: 0%;
}
.scan-info {
display: flex;
justify-content: space-between;
align-items: center;
font-size: 0.875rem;
}
.scan-info span:first-child {
color: var(--text-muted);
}
.scan-info span:last-child {
font-weight: 600;
color: var(--primary-color);
}
.form-group small {
display: block;
margin-top: 0.25rem;
color: var(--text-muted);
font-size: 0.875rem;
}
.btn-sm {
padding: 0.375rem 0.75rem;
font-size: 0.875rem;
}
static/index.html
-487
View File
@@ -1,487 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>NFOGuard - Database Management</title>
<link rel="stylesheet" href="/static/css/styles.css?v=manual-scan-ui">
<link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0/css/all.min.css" rel="stylesheet">
</head>
<body>
<div class="app-container">
<!-- Header -->
<header class="app-header">
<div class="header-content">
<h1><i class="fas fa-shield-alt"></i> NFOGuard</h1>
<p>Database Management & Reporting</p>
</div>
<div class="auth-status" id="auth-status" style="display: none;">
<span class="auth-user">
<i class="fas fa-user"></i> <span id="auth-username">Loading...</span>
</span>
<button class="auth-logout" id="logout-btn" onclick="logout()">
<i class="fas fa-sign-out-alt"></i> Logout
</button>
</div>
<nav class="nav-tabs">
<button class="nav-tab active" data-tab="dashboard">
<i class="fas fa-tachometer-alt"></i> Dashboard
</button>
<button class="nav-tab" data-tab="movies">
<i class="fas fa-film"></i> Movies
</button>
<button class="nav-tab" data-tab="tv">
<i class="fas fa-tv"></i> TV Series
</button>
<button class="nav-tab" data-tab="reports">
<i class="fas fa-chart-bar"></i> Reports
</button>
<button class="nav-tab" data-tab="tools">
<i class="fas fa-tools"></i> Tools
</button>
</nav>
</header>
<!-- Main Content -->
<main class="main-content">
<!-- Dashboard Tab -->
<div class="tab-content active" id="dashboard">
<div class="dashboard-grid">
<div class="stat-card">
<div class="stat-icon movies">
<i class="fas fa-film"></i>
</div>
<div class="stat-info">
<h3 id="movies-total">-</h3>
<p>Total Movies</p>
<small id="movies-with-dates">- with dates</small>
</div>
</div>
<div class="stat-card">
<div class="stat-icon tv">
<i class="fas fa-tv"></i>
</div>
<div class="stat-info">
<h3 id="series-total">-</h3>
<p>TV Series</p>
<small id="episodes-total">- episodes</small>
</div>
</div>
<div class="stat-card">
<div class="stat-icon missing">
<i class="fas fa-exclamation-triangle"></i>
</div>
<div class="stat-info">
<h3 id="missing-dates-total">-</h3>
<p>Missing Dates</p>
<small id="no-valid-source-total">- no valid source</small>
</div>
</div>
<div class="stat-card">
<div class="stat-icon activity">
<i class="fas fa-history"></i>
</div>
<div class="stat-info">
<h3 id="recent-activity">-</h3>
<p>Recent Activity</p>
<small>Last 7 days</small>
</div>
</div>
</div>
<div class="dashboard-charts">
<div class="chart-card">
<h3><i class="fas fa-chart-pie"></i> Movie Sources</h3>
<div id="movie-sources-chart" class="chart-container"></div>
</div>
<div class="chart-card">
<h3><i class="fas fa-chart-pie"></i> Episode Sources</h3>
<div id="episode-sources-chart" class="chart-container"></div>
</div>
</div>
</div>
<!-- Movies Tab -->
<div class="tab-content" id="movies">
<div class="content-header">
<h2><i class="fas fa-film"></i> Movies Database</h2>
<div class="content-controls">
<div class="search-controls">
<div class="search-box">
<i class="fas fa-search"></i>
<input type="text" id="movies-search" placeholder="Search title/path...">
</div>
<div class="search-box">
<i class="fas fa-hashtag"></i>
<input type="text" id="movies-imdb-search" placeholder="Search IMDb ID...">
</div>
</div>
<div class="filter-controls">
<select id="movies-filter-date">
<option value="">All Movies</option>
<option value="true">With Dates</option>
<option value="false">Missing Dates</option>
</select>
<select id="movies-filter-source">
<option value="">All Sources</option>
</select>
<button class="btn btn-primary" onclick="refreshMovies()">
<i class="fas fa-sync"></i> Refresh
</button>
</div>
</div>
</div>
<div class="table-container">
<table class="data-table" id="movies-table">
<thead>
<tr>
<th>Title</th>
<th>IMDb ID</th>
<th>Movie Released</th>
<th>Date Added to Library</th>
<th>Source</th>
<th>Date Type</th>
<th>Video File</th>
<th>Actions</th>
</tr>
</thead>
<tbody id="movies-tbody">
<tr>
<td colspan="8" class="loading">Loading movies...</td>
</tr>
</tbody>
</table>
</div>
<div class="pagination" id="movies-pagination"></div>
</div>
<!-- TV Series Tab -->
<div class="tab-content" id="tv">
<div class="content-header">
<h2><i class="fas fa-tv"></i> TV Series Database</h2>
<div class="content-controls">
<div class="search-controls">
<div class="search-box">
<i class="fas fa-search"></i>
<input type="text" id="series-search" placeholder="Search title/path...">
</div>
<div class="search-box">
<i class="fas fa-hashtag"></i>
<input type="text" id="series-imdb-search" placeholder="Search IMDb ID...">
</div>
</div>
<div class="filter-controls">
<select id="series-filter-date">
<option value="">All Series</option>
<option value="complete">Fully Dated</option>
<option value="incomplete">Missing Dates</option>
<option value="none">No Dates</option>
</select>
<select id="series-filter-source">
<option value="">All Sources</option>
</select>
<button class="btn btn-primary" onclick="refreshSeries()">
<i class="fas fa-sync"></i> Refresh
</button>
</div>
</div>
</div>
<div class="table-container">
<table class="data-table" id="series-table">
<thead>
<tr>
<th>Series Title</th>
<th>IMDb ID</th>
<th>Episodes</th>
<th>With Dates</th>
<th>With Video</th>
<th>Actions</th>
</tr>
</thead>
<tbody id="series-tbody">
<tr>
<td colspan="6" class="loading">Loading series...</td>
</tr>
</tbody>
</table>
</div>
<div class="pagination" id="series-pagination"></div>
</div>
<!-- Reports Tab -->
<div class="tab-content" id="reports">
<div class="content-header">
<h2><i class="fas fa-chart-bar"></i> Missing Dates Report</h2>
<div class="content-controls">
<button class="btn btn-primary" onclick="refreshReport()">
<i class="fas fa-sync"></i> Refresh Report
</button>
</div>
</div>
<div class="report-summary" id="report-summary">
<div class="summary-card">
<h3>Movies</h3>
<p><span id="report-movies-with">-</span> with dates</p>
<p><span id="report-movies-missing">-</span> missing dates</p>
</div>
<div class="summary-card">
<h3>Episodes</h3>
<p><span id="report-episodes-with">-</span> with dates</p>
<p><span id="report-episodes-missing">-</span> missing dates</p>
</div>
</div>
<div class="report-content">
<div class="report-section">
<h3><i class="fas fa-film"></i> Movies Missing Dates</h3>
<div class="table-container">
<table class="data-table">
<thead>
<tr>
<th>Title</th>
<th>IMDb ID</th>
<th>Released</th>
<th>Source</th>
<th>Smart Fix</th>
</tr>
</thead>
<tbody id="report-movies-tbody">
<tr>
<td colspan="5" class="loading">Loading report...</td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="report-section">
<h3><i class="fas fa-tv"></i> Episodes Missing Dates</h3>
<div class="table-container">
<table class="data-table">
<thead>
<tr>
<th>Series</th>
<th>Episode</th>
<th>IMDb ID</th>
<th>Aired</th>
<th>Source</th>
<th>Smart Fix</th>
</tr>
</thead>
<tbody id="report-episodes-tbody">
<tr>
<td colspan="6" class="loading">Loading report...</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
</div>
<!-- Tools Tab -->
<div class="tab-content" id="tools">
<div class="content-header">
<h2><i class="fas fa-tools"></i> Database Tools</h2>
</div>
<div class="tools-grid">
<div class="tool-card">
<h3><i class="fas fa-exchange-alt"></i> Bulk Source Update</h3>
<p>Change source for multiple items at once</p>
<form id="bulk-update-form">
<div class="form-group">
<label>Media Type:</label>
<select id="bulk-media-type" required>
<option value="">Select type...</option>
<option value="movies">Movies</option>
<option value="episodes">Episodes</option>
</select>
</div>
<div class="form-group">
<label>From Source:</label>
<input type="text" id="bulk-old-source" placeholder="e.g., no_valid_date_source" required>
</div>
<div class="form-group">
<label>To Source:</label>
<select id="bulk-new-source" required>
<option value="">Select new source...</option>
<option value="airdate">Air Date</option>
<option value="digital_release">Digital Release</option>
<option value="manual">Manual</option>
<option value="radarr:db.history.import">Radarr Import</option>
<option value="sonarr:history.import">Sonarr Import</option>
</select>
</div>
<button type="submit" class="btn btn-warning">
<i class="fas fa-exchange-alt"></i> Update Sources
</button>
</form>
</div>
<div class="tool-card">
<h3><i class="fas fa-search"></i> Manual Scan</h3>
<p>Scan specific folders or perform full library scans</p>
<form id="manual-scan-form">
<div class="form-group">
<label>Scan Type:</label>
<select id="scan-type" required>
<option value="both">TV Shows & Movies</option>
<option value="tv">TV Shows Only</option>
<option value="movies">Movies Only</option>
</select>
</div>
<div class="form-group">
<label>Scan Mode:</label>
<select id="scan-mode" required>
<option value="smart">Smart (Recommended)</option>
<option value="full">Full Scan</option>
<option value="incomplete">Incomplete Only</option>
</select>
</div>
<div class="form-group">
<label>Specific Path (Optional):</label>
<input type="text" id="scan-path" placeholder="e.g., /mnt/unionfs/Media/TV/Series Name" title="Leave empty for full library scan">
<small>Leave empty to scan entire library</small>
</div>
<button type="submit" class="btn btn-primary">
<i class="fas fa-play"></i> Start Scan
</button>
</form>
<div id="scan-status" class="scan-status" style="display: none;">
<div class="scan-progress">
<div class="progress-bar">
<div class="progress-fill" id="scan-progress-bar"></div>
</div>
<div class="scan-info">
<span id="scan-current-operation">Initializing...</span>
<span id="scan-progress-text">0%</span>
</div>
</div>
<button class="btn btn-secondary btn-sm" onclick="stopScanPolling()">
<i class="fas fa-times"></i> Hide Status
</button>
</div>
</div>
<div class="tool-card">
<h3><i class="fas fa-database"></i> Database Statistics</h3>
<p>View detailed database information</p>
<div class="stats-display" id="detailed-stats">
<p>Click refresh to load detailed statistics</p>
</div>
<button class="btn btn-secondary" onclick="loadDetailedStats()">
<i class="fas fa-sync"></i> Refresh Stats
</button>
</div>
<div class="tool-card">
<h3><i class="fas fa-upload"></i> Populate Database</h3>
<p>Bulk import data from Radarr/Sonarr into NFOGuard database</p>
<form id="populate-form">
<div class="form-group">
<label>Media Type:</label>
<select id="populate-media-type" required>
<option value="both">Movies & TV Shows</option>
<option value="movies">Movies Only</option>
<option value="tv">TV Shows Only</option>
</select>
</div>
<button type="submit" class="btn btn-primary">
<i class="fas fa-play"></i> Start Population
</button>
</form>
<div id="populate-status" class="scan-status" style="display: none;">
<div class="scan-progress">
<div class="progress-bar">
<div class="progress-fill" id="populate-progress-bar"></div>
</div>
<div class="scan-info">
<span id="populate-current-operation">Running...</span>
<span id="populate-progress-text">In Progress</span>
</div>
</div>
<div id="populate-results" class="stats-display" style="margin-top: 10px;"></div>
<button class="btn btn-secondary btn-sm" onclick="stopPopulatePolling()">
<i class="fas fa-times"></i> Hide Status
</button>
</div>
</div>
</div>
</div>
</main>
</div>
<!-- Smart Fix Modal -->
<div class="modal" id="smart-fix-modal">
<div class="modal-content">
<div class="modal-header">
<h3 id="smart-fix-title">Choose Date Source</h3>
<button class="modal-close" onclick="closeSmartFixModal()">&times;</button>
</div>
<div class="modal-body">
<div id="smart-fix-content">
<p>Loading available options...</p>
</div>
<div class="form-actions">
<button type="button" class="btn btn-secondary" onclick="closeSmartFixModal()">Cancel</button>
</div>
</div>
</div>
</div>
<!-- Edit Modal -->
<div class="modal" id="edit-modal">
<div class="modal-content">
<div class="modal-header">
<h3 id="modal-title">Edit Entry</h3>
<button class="modal-close" onclick="closeModal()">&times;</button>
</div>
<div class="modal-body">
<form id="edit-form">
<input type="hidden" id="edit-imdb-id">
<input type="hidden" id="edit-season">
<input type="hidden" id="edit-episode">
<input type="hidden" id="edit-media-type">
<div class="form-group">
<label for="edit-dateadded">Date Added:</label>
<input type="datetime-local" id="edit-dateadded">
<small>Leave empty to clear date</small>
</div>
<div class="form-group">
<label for="edit-source">Source:</label>
<select id="edit-source" required>
<option value="manual">Manual</option>
<option value="airdate">Air Date</option>
<option value="digital_release">Digital Release</option>
<option value="radarr:db.history.import">Radarr Import</option>
<option value="sonarr:history.import">Sonarr Import</option>
<option value="no_valid_date_source">No Valid Source</option>
</select>
</div>
<div class="form-actions">
<button type="button" class="btn btn-secondary" onclick="closeModal()">Cancel</button>
<button type="submit" class="btn btn-primary">Save Changes</button>
</div>
</form>
</div>
</div>
</div>
<!-- Toast Notifications -->
<div class="toast-container" id="toast-container"></div>
<script src="/static/js/app.js?v=manual-scan-ui"></script>
</body>
</html>
static/js/app.js
-1928
View File
File diff suppressed because it is too large Load Diff
utils/async_file_utils.py
-468
View File
@@ -1,468 +0,0 @@
"""
Async file utilities for NFOGuard
High-performance async file operations with concurrent processing
"""
import asyncio
import aiofiles
import aiofiles.os
from pathlib import Path
from typing import List, Dict, Optional, Tuple, Any, Set
import xml.etree.ElementTree as ET
from datetime import datetime
from utils.logging import _log
from utils.exceptions import FileOperationError, NFOCreationError
from utils.file_utils import VIDEO_EXTENSIONS, extract_episode_info, extract_imdb_id_from_path
from utils.nfo_patterns import parse_nfo_with_tolerance, write_nfo_file
async def async_read_text_file(file_path: Path, encoding: str = 'utf-8') -> Optional[str]:
"""
Async read text file with error handling
Args:
file_path: Path to file to read
encoding: Text encoding (default: utf-8)
Returns:
File content as string or None if error
"""
try:
async with aiofiles.open(file_path, 'r', encoding=encoding, errors='ignore') as f:
return await f.read()
except Exception as e:
_log("WARNING", f"Failed to read file {file_path}: {e}")
return None
async def async_write_text_file(file_path: Path, content: str, encoding: str = 'utf-8') -> bool:
"""
Async write text file with error handling
Args:
file_path: Path to file to write
content: Content to write
encoding: Text encoding (default: utf-8)
Returns:
True if successful, False otherwise
"""
try:
# Ensure parent directory exists
await aiofiles.os.makedirs(file_path.parent, exist_ok=True)
async with aiofiles.open(file_path, 'w', encoding=encoding) as f:
await f.write(content)
return True
except Exception as e:
_log("ERROR", f"Failed to write file {file_path}: {e}")
return False
async def async_file_exists(file_path: Path) -> bool:
"""
Async check if file exists
Args:
file_path: Path to check
Returns:
True if file exists, False otherwise
"""
try:
return await aiofiles.os.path.exists(file_path)
except Exception:
return False
async def async_get_file_mtime(file_path: Path) -> Optional[float]:
"""
Async get file modification time
Args:
file_path: Path to file
Returns:
Modification time as timestamp or None if error
"""
try:
stat_result = await aiofiles.os.stat(file_path)
return stat_result.st_mtime
except Exception:
return None
async def async_set_file_mtime(file_path: Path, mtime: float) -> bool:
"""
Async set file modification time
Args:
file_path: Path to file
mtime: New modification time as timestamp
Returns:
True if successful, False otherwise
"""
try:
await aiofiles.os.utime(file_path, (mtime, mtime))
return True
except Exception as e:
_log("WARNING", f"Failed to set mtime for {file_path}: {e}")
return False
async def async_find_video_files(directory: Path, recursive: bool = True) -> List[Path]:
"""
Async find all video files in a directory
Args:
directory: Directory to search
recursive: Whether to search recursively
Returns:
List of video file paths
"""
if not await async_file_exists(directory):
return []
video_files = []
try:
if recursive:
# Use os.walk equivalent for async
async def _walk_directory(path: Path):
try:
entries = await aiofiles.os.listdir(path)
for entry in entries:
entry_path = path / entry
if await aiofiles.os.path.isfile(entry_path):
if entry_path.suffix.lower() in VIDEO_EXTENSIONS:
video_files.append(entry_path)
elif await aiofiles.os.path.isdir(entry_path):
await _walk_directory(entry_path)
except Exception as e:
_log("WARNING", f"Failed to scan directory {path}: {e}")
await _walk_directory(directory)
else:
# Non-recursive scan
try:
entries = await aiofiles.os.listdir(directory)
for entry in entries:
entry_path = directory / entry
if await aiofiles.os.path.isfile(entry_path):
if entry_path.suffix.lower() in VIDEO_EXTENSIONS:
video_files.append(entry_path)
except Exception as e:
_log("WARNING", f"Failed to scan directory {directory}: {e}")
except Exception as e:
_log("ERROR", f"Failed to find video files in {directory}: {e}")
return video_files
async def async_find_episodes_on_disk(series_path: Path) -> Dict[Tuple[int, int], List[Path]]:
"""
Async find all episodes on disk with concurrent processing
Args:
series_path: Path to series directory
Returns:
Dictionary mapping (season, episode) tuples to lists of video files
"""
episodes = {}
if not await async_file_exists(series_path):
return episodes
# Get all video files concurrently
video_files = await async_find_video_files(series_path, recursive=True)
# Process files to extract episode information
for video_file in video_files:
episode_info = extract_episode_info(video_file.name)
if episode_info:
season, episode = episode_info["season"], episode_info["episode"]
key = (season, episode)
if key not in episodes:
episodes[key] = []
episodes[key].append(video_file)
return episodes
async def async_read_nfo_file(nfo_path: Path) -> Optional[ET.Element]:
"""
Async read and parse NFO file
Args:
nfo_path: Path to NFO file
Returns:
XML root element if successful, None otherwise
"""
if not await async_file_exists(nfo_path):
return None
try:
content = await async_read_text_file(nfo_path)
if not content:
return None
# Parse XML content
try:
root = ET.fromstring(content)
return root
except ET.ParseError:
# Try with tolerance (sync operation for now)
return parse_nfo_with_tolerance(nfo_path)
except Exception as e:
_log("ERROR", f"Failed to read NFO file {nfo_path}: {e}")
return None
async def async_write_nfo_file(
nfo_path: Path,
root: ET.Element,
lock_metadata: bool = True
) -> bool:
"""
Async write NFO XML content to file
Args:
nfo_path: Path where to write the NFO file
root: XML root element to write
lock_metadata: Whether to add file locking attributes
Returns:
True if successful, False otherwise
"""
try:
# Ensure parent directory exists
await aiofiles.os.makedirs(nfo_path.parent, exist_ok=True)
# Add file locking if requested
if lock_metadata:
root.set('nfoguard_managed', 'true')
root.set('last_updated', datetime.now().isoformat())
# Create tree and format
tree = ET.ElementTree(root)
ET.indent(tree, space=" ", level=0) # Pretty formatting
# Convert to string
xml_str = ET.tostring(root, encoding='unicode', xml_declaration=False)
xml_content = f'<?xml version="1.0" encoding="utf-8"?>\n{xml_str}'
# Write asynchronously
success = await async_write_text_file(nfo_path, xml_content)
if success:
_log("DEBUG", f"Successfully wrote NFO file: {nfo_path}")
return success
except Exception as e:
_log("ERROR", f"Failed to write NFO file {nfo_path}: {e}")
return False
async def async_batch_process_files(
file_paths: List[Path],
process_func,
max_concurrent: int = 10,
progress_callback: Optional[callable] = None
) -> List[Any]:
"""
Process multiple files concurrently with controlled concurrency
Args:
file_paths: List of file paths to process
process_func: Async function to process each file
max_concurrent: Maximum number of concurrent operations
progress_callback: Optional callback for progress updates
Returns:
List of results from processing each file
"""
semaphore = asyncio.Semaphore(max_concurrent)
results = []
async def _process_with_semaphore(file_path: Path, index: int) -> Any:
async with semaphore:
try:
result = await process_func(file_path)
if progress_callback:
progress_callback(index + 1, len(file_paths), file_path)
return result
except Exception as e:
_log("ERROR", f"Failed to process {file_path}: {e}")
return None
# Create tasks for all files
tasks = [
_process_with_semaphore(file_path, i)
for i, file_path in enumerate(file_paths)
]
# Execute all tasks concurrently with controlled concurrency
results = await asyncio.gather(*tasks, return_exceptions=True)
return results
async def async_batch_nfo_operations(
nfo_operations: List[Dict[str, Any]],
max_concurrent: int = 5
) -> List[bool]:
"""
Batch NFO operations (read/write) with controlled concurrency
Args:
nfo_operations: List of operation dictionaries with 'type', 'path', and other params
max_concurrent: Maximum number of concurrent operations
Returns:
List of success/failure results
"""
async def _execute_nfo_operation(operation: Dict[str, Any]) -> bool:
try:
op_type = operation.get('type')
path = operation.get('path')
if op_type == 'read':
result = await async_read_nfo_file(path)
return result is not None
elif op_type == 'write':
root = operation.get('root')
lock_metadata = operation.get('lock_metadata', True)
return await async_write_nfo_file(path, root, lock_metadata)
else:
_log("ERROR", f"Unknown NFO operation type: {op_type}")
return False
except Exception as e:
_log("ERROR", f"Failed to execute NFO operation: {e}")
return False
return await async_batch_process_files(
[op.get('path') for op in nfo_operations],
lambda path: _execute_nfo_operation(next(op for op in nfo_operations if op.get('path') == path)),
max_concurrent
)
async def async_concurrent_episode_processing(
episodes_data: List[Dict[str, Any]],
process_episode_func,
max_concurrent: int = 3
) -> List[Any]:
"""
Process multiple episodes concurrently
Args:
episodes_data: List of episode data dictionaries
process_episode_func: Async function to process each episode
max_concurrent: Maximum number of concurrent episode processes
Returns:
List of processing results
"""
semaphore = asyncio.Semaphore(max_concurrent)
async def _process_episode_with_semaphore(episode_data: Dict[str, Any]) -> Any:
async with semaphore:
try:
return await process_episode_func(episode_data)
except Exception as e:
_log("ERROR", f"Failed to process episode {episode_data}: {e}")
return None
# Create tasks for all episodes
tasks = [
_process_episode_with_semaphore(episode_data)
for episode_data in episodes_data
]
# Execute all tasks concurrently
results = await asyncio.gather(*tasks, return_exceptions=True)
return results
async def async_directory_scan_with_stats(
directories: List[Path],
file_extensions: Optional[Set[str]] = None
) -> Dict[str, Any]:
"""
Async scan multiple directories and gather statistics
Args:
directories: List of directories to scan
file_extensions: Optional set of file extensions to filter by
Returns:
Dictionary with scan statistics and file lists
"""
if file_extensions is None:
file_extensions = VIDEO_EXTENSIONS
stats = {
'total_files': 0,
'total_directories': len(directories),
'files_by_directory': {},
'scan_errors': [],
'total_size_bytes': 0
}
async def _scan_single_directory(directory: Path) -> Dict[str, Any]:
dir_stats = {
'path': str(directory),
'files': [],
'file_count': 0,
'size_bytes': 0,
'error': None
}
try:
if not await async_file_exists(directory):
dir_stats['error'] = 'Directory does not exist'
return dir_stats
files = await async_find_video_files(directory, recursive=True)
dir_stats['files'] = [str(f) for f in files]
dir_stats['file_count'] = len(files)
# Calculate total size
for file_path in files:
try:
stat_result = await aiofiles.os.stat(file_path)
dir_stats['size_bytes'] += stat_result.st_size
except Exception:
pass # Skip files we can't stat
except Exception as e:
dir_stats['error'] = str(e)
stats['scan_errors'].append(f"{directory}: {e}")
return dir_stats
# Scan all directories concurrently
directory_results = await asyncio.gather(
*[_scan_single_directory(directory) for directory in directories],
return_exceptions=True
)
# Aggregate results
for result in directory_results:
if isinstance(result, dict) and not result.get('error'):
stats['files_by_directory'][result['path']] = result
stats['total_files'] += result['file_count']
stats['total_size_bytes'] += result['size_bytes']
return stats
utils/file_utils.py
+1 -2
View File
@@ -62,8 +62,7 @@ def find_media_path_by_imdb_and_title(
# Search by IMDb ID first (more reliable)
if imdb_id:
# Use proper glob pattern - escape brackets to match literal [imdb-ID]
pattern = str(media_path / f"*\\[imdb-{imdb_id}\\]*")
pattern = str(media_path / f"*[imdb-{imdb_id}]*")
matches = glob.glob(pattern)
if matches:
return Path(matches[0])
utils/imdb_utils.py
-85
View File
@@ -1,85 +0,0 @@
#!/usr/bin/env python3
"""
IMDb ID Extraction Utilities
Parses IMDb IDs from directory/file names (no file I/O)
Phase 3: Replaces NFOManager for IMDb ID extraction only
"""
import re
from pathlib import Path
from typing import Optional
def parse_imdb_from_path(path: Path) -> Optional[str]:
"""
Extract IMDb ID from directory path or filename using regex patterns.
Does NOT read any files - only parses the path string.
Supported patterns:
- [imdb-tt1234567]
- [tt1234567]
- {imdb-tt1234567}
- (imdb-tt1234567)
- -tt1234567 (at end)
- _tt1234567 (at end)
Args:
path: Path object to parse
Returns:
IMDb ID (e.g., "tt1234567") or None if not found
"""
path_str = str(path).lower()
# Try [imdb-ttXXXXXXX] format first (most explicit)
match = re.search(r'\[imdb-?(tt\d+)\]', path_str)
if match:
return match.group(1)
# Try standalone [ttXXXXXXX] format in brackets
match = re.search(r'\[(tt\d+)\]', path_str)
if match:
return match.group(1)
# Try {imdb-ttXXXXXXX} format with curly braces
match = re.search(r'\{imdb-?(tt\d+)\}', path_str)
if match:
return match.group(1)
# Try (imdb-ttXXXXXXX) format with parentheses
match = re.search(r'\(imdb-?(tt\d+)\)', path_str)
if match:
return match.group(1)
# Try ttXXXXXXX at end of filename/dirname (common pattern)
match = re.search(r'[-_\s](tt\d+)$', path_str)
if match:
return match.group(1)
return None
def find_imdb_in_directory(directory: Path) -> Optional[str]:
"""
Find IMDb ID from directory name or filenames within the directory.
Does NOT read file contents - only checks filenames.
Args:
directory: Directory path to search
Returns:
IMDb ID or None if not found
"""
# First try directory name itself
imdb_id = parse_imdb_from_path(directory)
if imdb_id:
return imdb_id
# Try all filenames in the directory
if directory.is_dir():
for file_path in directory.iterdir():
if file_path.is_file():
imdb_id = parse_imdb_from_path(file_path)
if imdb_id:
return imdb_id
return None
utils/logging.py
+8 -82
View File
@@ -10,49 +10,6 @@ from datetime import datetime, timezone
from zoneinfo import ZoneInfo
class SafeRotatingFileHandler(logging.handlers.RotatingFileHandler):
"""A RotatingFileHandler that handles missing backup files gracefully"""
def doRollover(self):
"""
Override doRollover to handle missing backup files gracefully
"""
if self.stream:
self.stream.close()
self.stream = None
if self.backupCount > 0:
# Remove the oldest backup if it exists
oldest_backup = f"{self.baseFilename}.{self.backupCount}"
if os.path.exists(oldest_backup):
try:
os.remove(oldest_backup)
except (OSError, FileNotFoundError):
pass # Ignore if file doesn't exist or can't be removed
# Rename existing backups, skipping missing ones
for i in range(self.backupCount - 1, 0, -1):
sfn = f"{self.baseFilename}.{i}"
dfn = f"{self.baseFilename}.{i + 1}"
if os.path.exists(sfn):
try:
os.rename(sfn, dfn)
except (OSError, FileNotFoundError):
pass # Skip if source doesn't exist or rename fails
# Rename the main log file
dfn = f"{self.baseFilename}.1"
if os.path.exists(self.baseFilename):
try:
os.rename(self.baseFilename, dfn)
except (OSError, FileNotFoundError):
pass # Skip if main file doesn't exist
# Open the new log file
if not self.delay:
self.stream = self._open()
class TimezoneAwareFormatter(logging.Formatter):
"""Formatter that respects the container timezone"""
def __init__(self, *args, **kwargs):
@@ -93,46 +50,15 @@ def _setup_file_logging():
logger = logging.getLogger("NFOGuard")
logger.setLevel(logging.DEBUG)
# Clear any existing handlers to avoid duplicates
logger.handlers.clear()
# Try to set up file logging
file_logging_enabled = False
try:
file_handler = SafeRotatingFileHandler(
log_dir / "nfoguard.log", maxBytes=50*1024*1024, backupCount=3
)
formatter = TimezoneAwareFormatter(
'[%(asctime)s] %(levelname)s: %(message)s'
)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
file_logging_enabled = True
except Exception as e:
# If RotatingFileHandler fails, try regular FileHandler
try:
file_handler = logging.FileHandler(log_dir / "nfoguard.log")
formatter = TimezoneAwareFormatter(
'[%(asctime)s] %(levelname)s: %(message)s'
)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
file_logging_enabled = True
except Exception as e2:
# File logging not available (e.g., read-only filesystem)
# Fall back to console-only logging silently
pass
# If file logging failed, ensure console handler is added
if not file_logging_enabled:
console_handler = logging.StreamHandler()
formatter = TimezoneAwareFormatter(
'[%(asctime)s] %(levelname)s: %(message)s'
)
console_handler.setFormatter(formatter)
logger.addHandler(console_handler)
file_handler = logging.handlers.RotatingFileHandler(
log_dir / "nfoguard.log", maxBytes=50*1024*1024, backupCount=3
)
formatter = TimezoneAwareFormatter(
'[%(asctime)s] %(levelname)s: %(message)s'
)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
return logger
webhooks/webhook_batcher.py
+8 -101
View File
@@ -3,21 +3,18 @@ Webhook Batching System for NFOGuard
Handles batching and processing of webhook events to avoid processing storms
"""
import threading
import time
from pathlib import Path
from typing import Dict, Set
from concurrent.futures import ThreadPoolExecutor
from config.settings import config
from utils.logging import _log
from utils.imdb_utils import find_imdb_in_directory, parse_imdb_from_path # Phase 3: Replaced NFOManager
class WebhookBatcher:
"""Batches webhook events to avoid processing storms"""
def __init__(self, nfo_manager=None):
# nfo_manager parameter kept for backward compatibility but no longer used (Phase 3)
def __init__(self):
self.pending: Dict[str, Dict] = {}
self.timers: Dict[str, threading.Timer] = {}
self.processing: Set[str] = set()
@@ -84,41 +81,14 @@ class WebhookBatcher:
# CRITICAL: Validate that the path contains the expected IMDb ID for movies
if media_type == 'movie':
expected_imdb = key.replace('movie:', '') if key.startswith('movie:') else key
# Use imdb_utils for IMDb detection (Phase 3: no NFO reading)
detected_imdb = find_imdb_in_directory(path_obj)
imdb_match = False
if detected_imdb:
# Compare with and without 'tt' prefix for flexibility
if detected_imdb == expected_imdb or detected_imdb.replace('tt', '') == expected_imdb.replace('tt', ''):
imdb_match = True
if not imdb_match:
_log("ERROR", f"BATCH VALIDATION FAILED: Expected IMDb {expected_imdb} but found {detected_imdb} in {path_str}")
_log("ERROR", f"Detected IMDb: {detected_imdb}, Expected: {expected_imdb}")
if expected_imdb not in path_str.lower():
_log("ERROR", f"BATCH VALIDATION FAILED: Expected IMDb {expected_imdb} not found in path {path_str}")
_log("ERROR", f"This prevents processing wrong movies due to batch corruption")
return
_log("DEBUG", f"Batch validation passed: IMDb {expected_imdb} matches detected {detected_imdb}")
_log("DEBUG", f"Batch validation passed: IMDb {expected_imdb} found in path")
# CRITICAL: Validate that the path contains the expected IMDb ID for TV shows
# Process based on media type
if media_type == 'tv':
expected_imdb = key.replace('tv:', '') if key.startswith('tv:') else key
# Use imdb_utils for IMDb detection (Phase 3: no NFO reading)
detected_imdb = parse_imdb_from_path(path_obj)
imdb_match = False
if detected_imdb:
# Compare with and without 'tt' prefix for flexibility
if detected_imdb == expected_imdb or detected_imdb.replace('tt', '') == expected_imdb.replace('tt', ''):
imdb_match = True
if not imdb_match:
_log("ERROR", f"BATCH VALIDATION FAILED: Expected IMDb {expected_imdb} but found {detected_imdb} in TV {path_str}")
_log("ERROR", f"Detected TV IMDb: {detected_imdb}, Expected: {expected_imdb}")
_log("ERROR", f"This prevents processing wrong TV series due to batch corruption")
return
_log("DEBUG", f"TV batch validation passed: IMDb {expected_imdb} matches detected {detected_imdb}")
if not self.tv_processor:
_log("ERROR", "TV processor not available")
return
@@ -129,13 +99,7 @@ class WebhookBatcher:
if processing_mode == 'targeted' and episodes_data:
_log("INFO", f"Using targeted episode processing for {len(episodes_data)} episodes")
# Handle sequential processing for bulk downloads
if len(episodes_data) > 1 and config.sequential_delay > 0:
_log("INFO", f"Processing {len(episodes_data)} episodes sequentially with {config.sequential_delay}s delay")
self._process_episodes_sequentially(path_obj, episodes_data)
else:
self.tv_processor.process_webhook_episodes(path_obj, episodes_data)
self.tv_processor.process_webhook_episodes(path_obj, episodes_data)
else:
_log("INFO", f"Using series processing mode (fallback or configured)")
self.tv_processor.process_series(path_obj)
@@ -155,29 +119,6 @@ class WebhookBatcher:
with self.lock:
self.processing.discard(key)
def _process_episodes_sequentially(self, path_obj: Path, episodes_data: list):
"""Process episodes one by one with delays to avoid API spam"""
total_episodes = len(episodes_data)
for i, episode in enumerate(episodes_data, 1):
try:
season = episode.get('seasonNumber', '?')
episode_num = episode.get('episodeNumber', '?')
_log("INFO", f"Processing episode {i}/{total_episodes}: S{season:02d}E{episode_num:02d}")
# Process single episode
self.tv_processor.process_webhook_episodes(path_obj, [episode])
# Add delay between episodes (except for the last one)
if i < total_episodes and config.sequential_delay > 0:
_log("INFO", f"Waiting {config.sequential_delay}s before next episode...")
time.sleep(config.sequential_delay)
except Exception as e:
_log("ERROR", f"Error processing episode {i}/{total_episodes}: {e}")
# Continue with next episode even if one fails
_log("INFO", f"Completed sequential processing of {total_episodes} episodes")
def get_status(self) -> Dict:
"""Get batch queue status"""
with self.lock:
@@ -186,38 +127,4 @@ class WebhookBatcher:
"processing_items": list(self.processing),
"pending_count": len(self.pending),
"processing_count": len(self.processing)
}
def shutdown(self):
"""Shutdown the webhook batcher gracefully"""
_log("INFO", "Shutting down webhook batcher...")
with self.lock:
# Cancel all pending timers
for timer in self.timers.values():
try:
timer.cancel()
except Exception as e:
_log("WARNING", f"Error canceling timer: {e}")
self.timers.clear()
# Log any remaining items
if self.pending:
_log("WARNING", f"Shutting down with {len(self.pending)} pending items")
if self.processing:
_log("INFO", f"Waiting for {len(self.processing)} items to finish processing...")
# Shutdown the thread pool executor
try:
# Use timeout parameter only if supported (Python 3.9+)
import sys
if sys.version_info >= (3, 9):
self.executor.shutdown(wait=True, timeout=10) # Wait up to 10 seconds
else:
self.executor.shutdown(wait=True) # No timeout for older Python versions
_log("INFO", "Thread pool executor shut down successfully")
except Exception as e:
_log("WARNING", f"Error shutting down thread pool: {e}")
_log("INFO", "Webhook batcher shutdown complete")
}