cleanup

2025-09-12 15:46:02 -04:00
parent 558a786bdb
commit c6e4e071e4
12 changed files with 83 additions and 590 deletions
@@ -0,0 +1,54 @@
 # NFOGuard Webhook Repository - Final Clean Structure
 After cleanup, the repository contains only essential files for the Docker webhook application.
 ## 📦 Core Application (6 files)
 ```
 nfoguard.py                 # Main webhook server
 requirements.txt            # Python dependencies  
 Dockerfile                  # Container build
 VERSION                     # Version tracking
 LICENSE                     # MIT license
 bulk_update_movies.py       # Utility for bulk library updates
 ```
 ## 🗂️ Source Code Modules (8 files)
 ```
 core/
 ├── database.py            # SQLite database management
 ├── nfo_manager.py         # NFO file creation and management  
 ├── path_mapper.py         # Container path mapping
 └── logging.py             # Logging configuration
 clients/
 ├── external_clients.py    # TMDB, OMDb, TVDB, Jellyseerr APIs
 ├── radarr_client.py       # Radarr API client
 ├── radarr_db_client.py    # Radarr database client
 └── sonarr_client.py       # Sonarr API client
 ```
 ## 🐳 Docker & Configuration (3 files)
 ```
 docker-compose.example.yml  # Public example (industry standard naming)
 .env.template               # Environment configuration template
 .env.secrets.template       # API keys template (includes TVDB_API_KEY)
 ```
 ## 📚 Documentation (6 files) 
 ```
 README.md                   # Main documentation (mentions companion plugin)
 CHANGELOG.md                # Version history
 DEPLOYMENT.md               # Deployment guide
 SETUP.md                    # Setup instructions
 TESTING.md                  # Testing guide  
 SUMMARY.md                  # Project summary
 ```
 ## 🎯 Total: 23 Essential Files
 - **No plugin artifacts** ✅
 - **No CI/development clutter** ✅
 - **No test files** ✅
 - **Industry standard naming** ✅
 - **Clean public webhook application** ✅
 Perfect for end-users who want to deploy the NFOGuard webhook server!
@@ -0,0 +1,29 @@
 # FINAL REPOSITORY CLEANUP
 # Remove these files for a clean public webhook repository
 ## CI/Development Artifacts (REMOVE):
 gitea-registry-troubleshooting.md
 gitea-packages-config.ini
 find-gitea-local-ip.sh
 example-distribution-README.md
 cleanup-github.sh
 check-gitea-registry.sh
 ## Test Files (REMOVE per user request):
 test_movie_scan.py
 test_end_to_end.py
 test_bulk_update.py
 ## Temporary Files Created During Cleanup (REMOVE):
 cleanup-files.sh
 FILES_TO_REMOVE.txt
 REPOSITORY_STRUCTURE.md
 CLEANUP_COMPLETE.md
 ## Summary Files - Keep One (REMOVE duplicate):
 # Keep: SUMMARY.md (shorter, more concise)
 # Remove: projectsummary.md (longer, more detailed)
 projectsummary.md
 ## Command to remove all:
 # rm gitea-registry-troubleshooting.md gitea-packages-config.ini find-gitea-local-ip.sh example-distribution-README.md cleanup-github.sh check-gitea-registry.sh test_movie_scan.py test_end_to_end.py test_bulk_update.py cleanup-files.sh FILES_TO_REMOVE.txt REPOSITORY_STRUCTURE.md CLEANUP_COMPLETE.md projectsummary.md
@@ -1,32 +0,0 @@
 #!/bin/bash
 echo "=== Gitea Registry Diagnostic Script ==="
 echo "Checking Gitea container registry configuration..."
 # Check if registry endpoint exists
 echo "1. Testing registry endpoint..."
 curl -s -o /dev/null -w "HTTP Status: %{http_code}\n" https://gitea.skalas.org/v2/ || echo "Registry endpoint not reachable"
 # Check if packages are enabled in Gitea
 echo "2. Testing packages endpoint..."
 curl -s -o /dev/null -w "HTTP Status: %{http_code}\n" https://gitea.skalas.org/api/packages || echo "Packages endpoint not available"
 # Check Docker registry v2 API
 echo "3. Testing Docker registry v2 API..."
 curl -s https://gitea.skalas.org/v2/_catalog || echo "Registry catalog not available"
 # Check if we can reach the specific repository
 echo "4. Testing repository endpoint..."
 curl -s https://gitea.skalas.org/v2/jskala/nfoguard/tags/list || echo "Repository not available"
 # Test authentication
 echo "5. Testing authentication..."
 echo "Enter your Gitea username:"
 read -r username
 echo "Enter your Gitea password/token:"
 read -rs password
 echo "Testing authenticated access..."
 curl -s -u "$username:$password" https://gitea.skalas.org/v2/_catalog || echo "Authentication failed"
 echo "=== Diagnostic complete ==="
@@ -1,17 +0,0 @@
 #!/bin/bash
 # Script to clean up CI/development files
 echo "Removing CI/development artifacts..."
 rm -f gitea-registry-troubleshooting.md
 rm -f gitea-packages-config.ini
 rm -f find-gitea-local-ip.sh
 rm -f example-multi-registry-workflow.yml
 rm -f example-docker-compose.yml
 rm -f example-distribution-README.md
 rm -f docker-compose.example.yml
 rm -f cleanup-github.sh
 rm -f check-gitea-registry.sh
 echo "Cleanup complete!"
 echo "Removed CI/registry-specific files from webhook repository"
@@ -1,33 +0,0 @@
 #!/bin/bash
 echo "Removing .github directory from repository..."
 # Check if .github directory exists
 if [ -d ".github" ]; then
    echo ".github directory found, removing it..."
    git rm -rf .github/
    git commit -m "Remove GitHub workflows - migrated to Gitea"
    echo ".github directory removed and committed"
    echo "Don't forget to run: git push"
 else
    echo ".github directory not found in current directory"
 fi
 # Check if it's already in .gitignore
 if grep -q "^\.github/" .gitignore; then
    echo ".github/ is already in .gitignore"
 else
    echo "Adding .github/ to .gitignore..."
    echo "" >> .gitignore
    echo "# Ignore GitHub workflows (using Gitea instead)" >> .gitignore
    echo ".github/" >> .gitignore
    git add .gitignore
    git commit -m "Add .github/ to .gitignore"
    echo "Added .github/ to .gitignore"
 fi
 echo ""
 echo "Next steps:"
 echo "1. Run: git push"
 echo "2. Make sure Node.js is installed on your Gitea runner host"
 echo "3. Test your pipeline by making a small commit"
@@ -1,51 +0,0 @@
 # NFOguard - Easy Deployment
 [![Docker Pulls](https://img.shields.io/docker/pulls/jskala/nfoguard)](https://hub.docker.com/r/jskala/nfoguard)
 [![GitHub Release](https://img.shields.io/github/v/release/jskala/NFOguard)](https://github.com/jskala/NFOguard/releases)
 Latest Version: v1.0.0
 ## Quick Start
 ```bash
 # Download the example files
 curl -o docker-compose.yml https://raw.githubusercontent.com/jskala/NFOguard-Deploy/main/docker-compose.yml
 curl -o .env https://raw.githubusercontent.com/jskala/NFOguard-Deploy/main/.env.example
 # Edit your .env file
 nano .env
 # Start the service
 docker-compose up -d
 ```
 ## Available Images
 - **GitHub Container Registry**: `ghcr.io/jskala/nfoguard:latest`
 - **Docker Hub**: `jskala/nfoguard:latest`
 ## Configuration
 See the [.env.example](./.env.example) file for all available configuration options.
 ## Docker Compose
 The [docker-compose.yml](./docker-compose.yml) includes:
 - NFOguard service
 - PostgreSQL database (optional)
 - Volume mounts for data persistence
 - Health checks
 - Restart policies
 ## Updates
 To update to the latest version:
 ```bash
 docker-compose pull
 docker-compose up -d
 ```
 ## Source Code
 The source code is available at: https://github.com/jskala/NFOguard
@@ -1,28 +0,0 @@
 #!/bin/bash
 echo "=== Finding Gitea Local IP ==="
 # Get local IP of gitea-lxc container
 echo "1. Checking local IPs..."
 ip addr show | grep -E "inet.*192\.|inet.*10\.|inet.*172\." || echo "No local IPs found"
 echo "2. Trying to resolve gitea-lxc locally..."
 ping -c 1 gitea-lxc 2>/dev/null | head -1 || echo "gitea-lxc not resolvable"
 echo "3. Checking for Gitea process..."
 ps aux | grep gitea || echo "No gitea process found"
 echo "4. Checking what ports are listening..."
 ss -tulpn | grep :3000 || echo "Port 3000 not listening"
 ss -tulpn | grep :80 || echo "Port 80 not listening"
 ss -tulpn | grep :443 || echo "Port 443 not listening"
 echo "5. Testing local connection..."
 LOCAL_IP=$(ip route get 8.8.8.8 | awk '{print $7; exit}')
 echo "Local IP appears to be: $LOCAL_IP"
 if [ ! -z "$LOCAL_IP" ]; then
    echo "Testing HTTP connection to local IP..."
    curl -s -o /dev/null -w "HTTP Status: %{http_code}\n" "http://$LOCAL_IP:3000/" || echo "Local HTTP not responding"
    curl -s -o /dev/null -w "HTTP Status: %{http_code}\n" "http://$LOCAL_IP/" || echo "Local HTTP port 80 not responding"
 fi
@@ -1,15 +0,0 @@
 # Add this to your Gitea app.ini configuration file
 # Usually located at /etc/gitea/app.ini or /data/gitea/conf/app.ini
 [packages]
 ENABLED = true
 # Optional: Configure storage
 [storage.packages]
 STORAGE_TYPE = local
 SERVE_DIRECT = false
 PATH = /data/packages
 # Optional: Set limits
 [package]
 CHUNKED_UPLOAD_PATH = /tmp/gitea-packages-chunked
@@ -1,49 +0,0 @@
 # Gitea Container Registry Configuration Check
 ## 1. Check Gitea app.ini configuration
 Your Gitea `app.ini` should have:
 ```ini
 [packages]
 ENABLED = true
 [server]
 ; Make sure these are set correctly
 DOMAIN = gitea.skalas.org
 ROOT_URL = https://gitea.skalas.org/
 ```
 ## 2. Check in Gitea Admin Panel
 1. Go to: https://gitea.skalas.org/admin/config
 2. Look for "Packages" section
 3. Should show "Packages: Enabled"
 ## 3. Check Repository Package Settings
 1. Go to your repo: https://gitea.skalas.org/jskala/NFOguard
 2. Check if you see a "Packages" tab
 3. Repository settings should allow packages
 ## 4. Manual Registry Test
 Try manually from your runner host:
 ```bash
 # Test login
 docker login gitea.skalas.org
 # Test push a simple image
 docker pull hello-world
 docker tag hello-world gitea.skalas.org/jskala/test:latest
 docker push gitea.skalas.org/jskala/test:latest
 ```
 ## 5. Common Issues
 - **Registry not enabled**: Check app.ini [packages] ENABLED = true
 - **Wrong URL format**: Should be gitea.skalas.org (not gitea.skalas.org:port)
 - **SSL issues**: Make sure HTTPS is properly configured
 - **Disk space**: Check if Gitea server has enough space
 - **Permissions**: Check file permissions on Gitea data directory
@@ -1,99 +0,0 @@
 #!/usr/bin/env python3
 """
 Test script for bulk_update_movies.py
 This tests the bulk update functionality without modifying data
 """
 import os
 import sys
 from pathlib import Path
 def test_bulk_update_dry_run():
    """Test bulk update script in dry-run mode"""
    print("🧪 Testing Bulk Update Script")
    print("=" * 40)
    # Check if we can import the required modules
    try:
        from clients.radarr_db_client import RadarrDbClient
        from core.database import NFOGuardDatabase
        print("✅ Successfully imported required modules")
    except Exception as e:
        print(f"❌ Import error: {e}")
        return False
    # Test database connections
    print("\n🔌 Testing Database Connections...")
    # Test Radarr database
    try:
        radarr_db = RadarrDbClient.from_env()
        if not radarr_db:
            print("❌ Radarr database connection failed - check environment variables")
            return False
        print("✅ Radarr database connection successful")
    except Exception as e:
        print(f"❌ Radarr database error: {e}")
        return False
    # Test NFOGuard database
    try:
        db_path = Path(os.environ.get("DB_PATH", "/app/data/media_dates.db"))
        nfo_db = NFOGuardDatabase(db_path)
        print(f"✅ NFOGuard database connection successful: {db_path}")
    except Exception as e:
        print(f"❌ NFOGuard database error: {e}")
        return False
    # Test query execution (dry run)
    print("\n📊 Testing Query Execution...")
    try:
        query = """
        SELECT 
            mm."ImdbId" as imdb_id,
            m."Id" as movie_id,
            mm."Title" as title,
            mm."Year" as year,
            m."Path" as path
        FROM "Movies" m
        JOIN "MovieMetadata" mm ON m."MovieMetadataId" = mm."Id"
        WHERE mm."ImdbId" IS NOT NULL AND mm."ImdbId" != ''
        ORDER BY mm."Title"
        LIMIT 5
        """
        with radarr_db._get_connection() as conn:
            if radarr_db.db_type == "postgresql":
                import psycopg2.extras
                cursor = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
            else:
                cursor = conn.cursor()
            cursor.execute(query)
            movies = cursor.fetchall()
        print(f"✅ Successfully queried Radarr database: {len(movies)} sample movies found")
        # Show sample data
        for i, movie in enumerate(movies):
            if radarr_db.db_type == "postgresql":
                imdb_id, title, year = movie['imdb_id'], movie['title'], movie['year']
            else:
                imdb_id, title, year = movie[0], movie[2], movie[3]
            print(f"  Sample {i+1}: {title} ({year}) - {imdb_id}")
    except Exception as e:
        print(f"❌ Query execution failed: {e}")
        return False
    print("\n✅ All tests passed! Bulk update script should work correctly.")
    return True
 if __name__ == "__main__":
    success = test_bulk_update_dry_run()
    if not success:
        print("\n❌ Bulk update test failed!")
        sys.exit(1)
    else:
        print("\n🎉 Bulk update test successful!")
@@ -1,150 +0,0 @@
 #!/usr/bin/env python3
 """
 End-to-end testing script for NFOGuard workflow
 Tests the complete flow: Database -> Manual Scan -> NFO Creation
 """
 import os
 import sys
 import requests
 import time
 from pathlib import Path
 def test_server_health():
    """Test if the server is running and healthy"""
    try:
        response = requests.get("http://localhost:8080/health", timeout=10)
        if response.status_code == 200:
            health_data = response.json()
            print(f"✅ Server is healthy: {health_data['status']}")
            print(f"   Version: {health_data['version']}")
            print(f"   Database: {health_data['database_status']}")
            if health_data.get('radarr_database'):
                radarr_status = health_data['radarr_database']['status']
                print(f"   Radarr DB: {radarr_status}")
            return True
        else:
            print(f"❌ Server health check failed: {response.status_code}")
            return False
    except Exception as e:
        print(f"❌ Cannot connect to server: {e}")
        return False
 def test_database_performance():
    """Test the database performance endpoint"""
    try:
        response = requests.get("http://localhost:8080/debug/movie/tt1674782", timeout=10)
        if response.status_code == 200:
            debug_data = response.json()
            if debug_data.get('detected_import_date'):
                print(f"✅ Database query works: {debug_data['detected_import_date']}")
                print(f"   Source: {debug_data['import_source']}")
                return True
            else:
                print(f"❌ No import date found for test movie")
                return False
        else:
            print(f"❌ Debug endpoint failed: {response.status_code}")
            return False
    except Exception as e:
        print(f"❌ Database test failed: {e}")
        return False
 def test_manual_scan():
    """Test the manual scan functionality"""
    try:
        print("🔍 Starting manual movie scan...")
        response = requests.post(
            "http://localhost:8080/manual/scan?scan_type=movies", 
            timeout=30
        )
        if response.status_code == 200:
            scan_data = response.json()
            print(f"✅ Manual scan started: {scan_data['message']}")
            # Wait a bit for processing
            print("⏳ Waiting for scan to process...")
            time.sleep(10)
            # Check batch status
            batch_response = requests.get("http://localhost:8080/batch/status", timeout=10)
            if batch_response.status_code == 200:
                batch_data = batch_response.json()
                print(f"   Pending items: {batch_data['pending_count']}")
                print(f"   Processing items: {batch_data['processing_count']}")
            return True
        else:
            print(f"❌ Manual scan failed: {response.status_code}")
            return False
    except Exception as e:
        print(f"❌ Manual scan test failed: {e}")
        return False
 def test_stats_endpoint():
    """Test database stats"""
    try:
        response = requests.get("http://localhost:8080/stats", timeout=10)
        if response.status_code == 200:
            stats = response.json()
            print(f"✅ Database stats:")
            print(f"   Movies: {stats.get('movie_count', 0)}")
            print(f"   TV Series: {stats.get('series_count', 0)}")
            print(f"   Episodes: {stats.get('episode_count', 0)}")
            return True
        else:
            print(f"❌ Stats endpoint failed: {response.status_code}")
            return False
    except Exception as e:
        print(f"❌ Stats test failed: {e}")
        return False
 def run_end_to_end_test():
    """Run complete end-to-end test suite"""
    print("🎯 NFOGuard End-to-End Testing")
    print("=" * 50)
    tests = [
        ("Server Health", test_server_health),
        ("Database Performance", test_database_performance),
        ("Manual Scan", test_manual_scan),
        ("Database Stats", test_stats_endpoint)
    ]
    passed = 0
    failed = 0
    for test_name, test_func in tests:
        print(f"\n🧪 Running: {test_name}")
        try:
            if test_func():
                passed += 1
                print(f"✅ {test_name} passed")
            else:
                failed += 1
                print(f"❌ {test_name} failed")
        except Exception as e:
            failed += 1
            print(f"❌ {test_name} crashed: {e}")
    print(f"\n📊 Test Results:")
    print(f"   ✅ Passed: {passed}")
    print(f"   ❌ Failed: {failed}")
    print(f"   📈 Success Rate: {passed}/{passed+failed} ({100*passed/(passed+failed):.1f}%)")
    return failed == 0
 if __name__ == "__main__":
    success = run_end_to_end_test()
    if success:
        print("\n🎉 All end-to-end tests passed!")
        print("\n📋 Next Steps:")
        print("1. Run bulk update: python3 bulk_update_movies.py")
        print("2. Test with real movie: curl http://localhost:8080/debug/movie/tt1674782") 
        print("3. Check NFO file creation in /media/Movies/movies/")
        print("4. Verify file timestamps match import dates")
    else:
        print("\n💥 Some tests failed - check the logs above")
        sys.exit(1)
@@ -1,116 +0,0 @@
 #!/usr/bin/env python3
 """
 Test script to verify movie directory scanning logic
 Simulates what the manual scan would do with correct paths
 """
 import os
 import sys
 from pathlib import Path
 def test_movie_scanning():
    """Test movie directory scanning logic"""
    print("🔍 Testing Movie Directory Scanning")
    print("=" * 50)
    # Test paths - use the corrected paths from the code
    movie_paths = [
        Path("/media/Movies/movies"),
        Path("/media/Movies/movies6")
    ]
    print("📁 Testing paths:")
    for path in movie_paths:
        print(f"  {path}")
    # Import the NFO manager to test the IMDb parsing
    try:
        from core.nfo_manager import NFOManager
        nfo_manager = NFOManager()
        print("✅ Successfully imported NFOManager")
    except Exception as e:
        print(f"❌ Failed to import NFOManager: {e}")
        return False
    # Simulate the scanning logic
    total_movies_found = 0
    for media_path in movie_paths:
        print(f"\n🎬 Scanning: {media_path}")
        # Check if path exists (this will fail on your local machine but show the logic)
        if not media_path.exists():
            print(f"  ⚠️  Path doesn't exist locally (expected on dev machine): {media_path}")
            print(f"  ℹ️  On production server, this path should contain directories like:")
            print(f"     Example Movie [imdb-tt1234567] (2023)/")
            print(f"     Another Movie [imdb-tt7654321] (2022)/")
            continue
        movie_count = 0
        for item in media_path.iterdir():
            if item.is_dir():
                imdb_id = nfo_manager.parse_imdb_from_path(item)
                if imdb_id:
                    movie_count += 1
                    print(f"  ✅ Found movie: {item.name} -> {imdb_id}")
                else:
                    print(f"  ⏭️  Skipped (no IMDb ID): {item.name}")
        print(f"  📊 Found {movie_count} movies with IMDb IDs in {media_path}")
        total_movies_found += movie_count
    print(f"\n🎯 Total movies that would be processed: {total_movies_found}")
    # Test IMDb ID parsing with sample directory names
    print(f"\n🧪 Testing IMDb ID Parsing Logic:")
    test_names = [
        "The Dark Knight [imdb-tt0468569] (2008)",
        "Inception [imdb-tt1375666] (2010)", 
        "Invalid Movie Name (2020)",
        "Movie Without IMDb (2021)",
        "Another Movie [imdb-tt1234567] (2023)"
    ]
    for test_name in test_names:
        test_path = Path(f"/fake/path/{test_name}")
        imdb_id = nfo_manager.parse_imdb_from_path(test_path)
        if imdb_id:
            print(f"  ✅ '{test_name}' -> {imdb_id}")
        else:
            print(f"  ❌ '{test_name}' -> No IMDb ID found")
    return True
 def show_manual_scan_instructions():
    """Show instructions for testing on the production server"""
    print("\n" + "="*60)
    print("📋 NEXT STEPS - Test on Production Server")
    print("="*60)
    print()
    print("1. 🔧 Update your environment variables:")
    print("   export MOVIE_PATHS='/media/Movies/movies,/media/Movies/movies6'")
    print("   export TV_PATHS='/media/TV/tv,/media/TV/tv6'")
    print()
    print("2. 🚀 Test the manual scan:")
    print("   curl -X POST 'http://localhost:8080/manual/scan?scan_type=movies'")
    print()
    print("3. 📊 Check the logs for movie processing:")
    print("   docker logs <container_name> | grep -E '(Processing movie|Completed movie scan)'")
    print()
    print("4. 🧪 Test the bulk update script:")
    print("   python3 test_bulk_update.py")
    print("   python3 bulk_update_movies.py")
    print()
    print("5. ✅ Verify end-to-end workflow:")
    print("   curl http://localhost:8080/debug/movie/tt1674782")
    print()
 if __name__ == "__main__":
    success = test_movie_scanning()
    if success:
        print("\n✅ Local testing completed successfully!")
        show_manual_scan_instructions()
    else:
        print("\n❌ Local testing failed!")
        sys.exit(1)