diff --git a/CLEAN_REPOSITORY_STRUCTURE.md b/CLEAN_REPOSITORY_STRUCTURE.md new file mode 100644 index 0000000..fa654ed --- /dev/null +++ b/CLEAN_REPOSITORY_STRUCTURE.md @@ -0,0 +1,54 @@ +# NFOGuard Webhook Repository - Final Clean Structure + +After cleanup, the repository contains only essential files for the Docker webhook application. + +## ๐Ÿ“ฆ Core Application (6 files) +``` +nfoguard.py # Main webhook server +requirements.txt # Python dependencies +Dockerfile # Container build +VERSION # Version tracking +LICENSE # MIT license +bulk_update_movies.py # Utility for bulk library updates +``` + +## ๐Ÿ—‚๏ธ Source Code Modules (8 files) +``` +core/ +โ”œโ”€โ”€ database.py # SQLite database management +โ”œโ”€โ”€ nfo_manager.py # NFO file creation and management +โ”œโ”€โ”€ path_mapper.py # Container path mapping +โ””โ”€โ”€ logging.py # Logging configuration + +clients/ +โ”œโ”€โ”€ external_clients.py # TMDB, OMDb, TVDB, Jellyseerr APIs +โ”œโ”€โ”€ radarr_client.py # Radarr API client +โ”œโ”€โ”€ radarr_db_client.py # Radarr database client +โ””โ”€โ”€ sonarr_client.py # Sonarr API client +``` + +## ๐Ÿณ Docker & Configuration (3 files) +``` +docker-compose.example.yml # Public example (industry standard naming) +.env.template # Environment configuration template +.env.secrets.template # API keys template (includes TVDB_API_KEY) +``` + +## ๐Ÿ“š Documentation (6 files) +``` +README.md # Main documentation (mentions companion plugin) +CHANGELOG.md # Version history +DEPLOYMENT.md # Deployment guide +SETUP.md # Setup instructions +TESTING.md # Testing guide +SUMMARY.md # Project summary +``` + +## ๐ŸŽฏ Total: 23 Essential Files +- **No plugin artifacts** โœ… +- **No CI/development clutter** โœ… +- **No test files** โœ… +- **Industry standard naming** โœ… +- **Clean public webhook application** โœ… + +Perfect for end-users who want to deploy the NFOGuard webhook server! \ No newline at end of file diff --git a/FINAL_CLEANUP_LIST.txt b/FINAL_CLEANUP_LIST.txt new file mode 100644 index 0000000..056c417 --- /dev/null +++ b/FINAL_CLEANUP_LIST.txt @@ -0,0 +1,29 @@ +# FINAL REPOSITORY CLEANUP +# Remove these files for a clean public webhook repository + +## CI/Development Artifacts (REMOVE): +gitea-registry-troubleshooting.md +gitea-packages-config.ini +find-gitea-local-ip.sh +example-distribution-README.md +cleanup-github.sh +check-gitea-registry.sh + +## Test Files (REMOVE per user request): +test_movie_scan.py +test_end_to_end.py +test_bulk_update.py + +## Temporary Files Created During Cleanup (REMOVE): +cleanup-files.sh +FILES_TO_REMOVE.txt +REPOSITORY_STRUCTURE.md +CLEANUP_COMPLETE.md + +## Summary Files - Keep One (REMOVE duplicate): +# Keep: SUMMARY.md (shorter, more concise) +# Remove: projectsummary.md (longer, more detailed) +projectsummary.md + +## Command to remove all: +# rm gitea-registry-troubleshooting.md gitea-packages-config.ini find-gitea-local-ip.sh example-distribution-README.md cleanup-github.sh check-gitea-registry.sh test_movie_scan.py test_end_to_end.py test_bulk_update.py cleanup-files.sh FILES_TO_REMOVE.txt REPOSITORY_STRUCTURE.md CLEANUP_COMPLETE.md projectsummary.md \ No newline at end of file diff --git a/check-gitea-registry.sh b/check-gitea-registry.sh deleted file mode 100644 index 25f0a88..0000000 --- a/check-gitea-registry.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -echo "=== Gitea Registry Diagnostic Script ===" -echo "Checking Gitea container registry configuration..." - -# Check if registry endpoint exists -echo "1. Testing registry endpoint..." -curl -s -o /dev/null -w "HTTP Status: %{http_code}\n" https://gitea.skalas.org/v2/ || echo "Registry endpoint not reachable" - -# Check if packages are enabled in Gitea -echo "2. Testing packages endpoint..." -curl -s -o /dev/null -w "HTTP Status: %{http_code}\n" https://gitea.skalas.org/api/packages || echo "Packages endpoint not available" - -# Check Docker registry v2 API -echo "3. Testing Docker registry v2 API..." -curl -s https://gitea.skalas.org/v2/_catalog || echo "Registry catalog not available" - -# Check if we can reach the specific repository -echo "4. Testing repository endpoint..." -curl -s https://gitea.skalas.org/v2/jskala/nfoguard/tags/list || echo "Repository not available" - -# Test authentication -echo "5. Testing authentication..." -echo "Enter your Gitea username:" -read -r username -echo "Enter your Gitea password/token:" -read -rs password - -echo "Testing authenticated access..." -curl -s -u "$username:$password" https://gitea.skalas.org/v2/_catalog || echo "Authentication failed" - -echo "=== Diagnostic complete ===" \ No newline at end of file diff --git a/cleanup-files.sh b/cleanup-files.sh deleted file mode 100644 index 5f865fc..0000000 --- a/cleanup-files.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -# Script to clean up CI/development files - -echo "Removing CI/development artifacts..." - -rm -f gitea-registry-troubleshooting.md -rm -f gitea-packages-config.ini -rm -f find-gitea-local-ip.sh -rm -f example-multi-registry-workflow.yml -rm -f example-docker-compose.yml -rm -f example-distribution-README.md -rm -f docker-compose.example.yml -rm -f cleanup-github.sh -rm -f check-gitea-registry.sh - -echo "Cleanup complete!" -echo "Removed CI/registry-specific files from webhook repository" \ No newline at end of file diff --git a/cleanup-github.sh b/cleanup-github.sh deleted file mode 100755 index f2db70a..0000000 --- a/cleanup-github.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -echo "Removing .github directory from repository..." - -# Check if .github directory exists -if [ -d ".github" ]; then - echo ".github directory found, removing it..." - git rm -rf .github/ - git commit -m "Remove GitHub workflows - migrated to Gitea" - echo ".github directory removed and committed" - echo "Don't forget to run: git push" -else - echo ".github directory not found in current directory" -fi - -# Check if it's already in .gitignore -if grep -q "^\.github/" .gitignore; then - echo ".github/ is already in .gitignore" -else - echo "Adding .github/ to .gitignore..." - echo "" >> .gitignore - echo "# Ignore GitHub workflows (using Gitea instead)" >> .gitignore - echo ".github/" >> .gitignore - git add .gitignore - git commit -m "Add .github/ to .gitignore" - echo "Added .github/ to .gitignore" -fi - -echo "" -echo "Next steps:" -echo "1. Run: git push" -echo "2. Make sure Node.js is installed on your Gitea runner host" -echo "3. Test your pipeline by making a small commit" \ No newline at end of file diff --git a/example-distribution-README.md b/example-distribution-README.md deleted file mode 100644 index ed281f2..0000000 --- a/example-distribution-README.md +++ /dev/null @@ -1,51 +0,0 @@ -# NFOguard - Easy Deployment - -[![Docker Pulls](https://img.shields.io/docker/pulls/jskala/nfoguard)](https://hub.docker.com/r/jskala/nfoguard) -[![GitHub Release](https://img.shields.io/github/v/release/jskala/NFOguard)](https://github.com/jskala/NFOguard/releases) - -Latest Version: v1.0.0 - -## Quick Start - -```bash -# Download the example files -curl -o docker-compose.yml https://raw.githubusercontent.com/jskala/NFOguard-Deploy/main/docker-compose.yml -curl -o .env https://raw.githubusercontent.com/jskala/NFOguard-Deploy/main/.env.example - -# Edit your .env file -nano .env - -# Start the service -docker-compose up -d -``` - -## Available Images - -- **GitHub Container Registry**: `ghcr.io/jskala/nfoguard:latest` -- **Docker Hub**: `jskala/nfoguard:latest` - -## Configuration - -See the [.env.example](./.env.example) file for all available configuration options. - -## Docker Compose - -The [docker-compose.yml](./docker-compose.yml) includes: -- NFOguard service -- PostgreSQL database (optional) -- Volume mounts for data persistence -- Health checks -- Restart policies - -## Updates - -To update to the latest version: - -```bash -docker-compose pull -docker-compose up -d -``` - -## Source Code - -The source code is available at: https://github.com/jskala/NFOguard \ No newline at end of file diff --git a/find-gitea-local-ip.sh b/find-gitea-local-ip.sh deleted file mode 100644 index a909b03..0000000 --- a/find-gitea-local-ip.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -echo "=== Finding Gitea Local IP ===" - -# Get local IP of gitea-lxc container -echo "1. Checking local IPs..." -ip addr show | grep -E "inet.*192\.|inet.*10\.|inet.*172\." || echo "No local IPs found" - -echo "2. Trying to resolve gitea-lxc locally..." -ping -c 1 gitea-lxc 2>/dev/null | head -1 || echo "gitea-lxc not resolvable" - -echo "3. Checking for Gitea process..." -ps aux | grep gitea || echo "No gitea process found" - -echo "4. Checking what ports are listening..." -ss -tulpn | grep :3000 || echo "Port 3000 not listening" -ss -tulpn | grep :80 || echo "Port 80 not listening" -ss -tulpn | grep :443 || echo "Port 443 not listening" - -echo "5. Testing local connection..." -LOCAL_IP=$(ip route get 8.8.8.8 | awk '{print $7; exit}') -echo "Local IP appears to be: $LOCAL_IP" - -if [ ! -z "$LOCAL_IP" ]; then - echo "Testing HTTP connection to local IP..." - curl -s -o /dev/null -w "HTTP Status: %{http_code}\n" "http://$LOCAL_IP:3000/" || echo "Local HTTP not responding" - curl -s -o /dev/null -w "HTTP Status: %{http_code}\n" "http://$LOCAL_IP/" || echo "Local HTTP port 80 not responding" -fi \ No newline at end of file diff --git a/gitea-packages-config.ini b/gitea-packages-config.ini deleted file mode 100644 index 7c92bfb..0000000 --- a/gitea-packages-config.ini +++ /dev/null @@ -1,15 +0,0 @@ -# Add this to your Gitea app.ini configuration file -# Usually located at /etc/gitea/app.ini or /data/gitea/conf/app.ini - -[packages] -ENABLED = true - -# Optional: Configure storage -[storage.packages] -STORAGE_TYPE = local -SERVE_DIRECT = false -PATH = /data/packages - -# Optional: Set limits -[package] -CHUNKED_UPLOAD_PATH = /tmp/gitea-packages-chunked \ No newline at end of file diff --git a/gitea-registry-troubleshooting.md b/gitea-registry-troubleshooting.md deleted file mode 100644 index 214974c..0000000 --- a/gitea-registry-troubleshooting.md +++ /dev/null @@ -1,49 +0,0 @@ -# Gitea Container Registry Configuration Check - -## 1. Check Gitea app.ini configuration - -Your Gitea `app.ini` should have: - -```ini -[packages] -ENABLED = true - -[server] -; Make sure these are set correctly -DOMAIN = gitea.skalas.org -ROOT_URL = https://gitea.skalas.org/ -``` - -## 2. Check in Gitea Admin Panel - -1. Go to: https://gitea.skalas.org/admin/config -2. Look for "Packages" section -3. Should show "Packages: Enabled" - -## 3. Check Repository Package Settings - -1. Go to your repo: https://gitea.skalas.org/jskala/NFOguard -2. Check if you see a "Packages" tab -3. Repository settings should allow packages - -## 4. Manual Registry Test - -Try manually from your runner host: - -```bash -# Test login -docker login gitea.skalas.org - -# Test push a simple image -docker pull hello-world -docker tag hello-world gitea.skalas.org/jskala/test:latest -docker push gitea.skalas.org/jskala/test:latest -``` - -## 5. Common Issues - -- **Registry not enabled**: Check app.ini [packages] ENABLED = true -- **Wrong URL format**: Should be gitea.skalas.org (not gitea.skalas.org:port) -- **SSL issues**: Make sure HTTPS is properly configured -- **Disk space**: Check if Gitea server has enough space -- **Permissions**: Check file permissions on Gitea data directory \ No newline at end of file diff --git a/test_bulk_update.py b/test_bulk_update.py deleted file mode 100644 index 07a301c..0000000 --- a/test_bulk_update.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python3 -""" -Test script for bulk_update_movies.py -This tests the bulk update functionality without modifying data -""" - -import os -import sys -from pathlib import Path - -def test_bulk_update_dry_run(): - """Test bulk update script in dry-run mode""" - print("๐Ÿงช Testing Bulk Update Script") - print("=" * 40) - - # Check if we can import the required modules - try: - from clients.radarr_db_client import RadarrDbClient - from core.database import NFOGuardDatabase - print("โœ… Successfully imported required modules") - except Exception as e: - print(f"โŒ Import error: {e}") - return False - - # Test database connections - print("\n๐Ÿ”Œ Testing Database Connections...") - - # Test Radarr database - try: - radarr_db = RadarrDbClient.from_env() - if not radarr_db: - print("โŒ Radarr database connection failed - check environment variables") - return False - print("โœ… Radarr database connection successful") - except Exception as e: - print(f"โŒ Radarr database error: {e}") - return False - - # Test NFOGuard database - try: - db_path = Path(os.environ.get("DB_PATH", "/app/data/media_dates.db")) - nfo_db = NFOGuardDatabase(db_path) - print(f"โœ… NFOGuard database connection successful: {db_path}") - except Exception as e: - print(f"โŒ NFOGuard database error: {e}") - return False - - # Test query execution (dry run) - print("\n๐Ÿ“Š Testing Query Execution...") - - try: - query = """ - SELECT - mm."ImdbId" as imdb_id, - m."Id" as movie_id, - mm."Title" as title, - mm."Year" as year, - m."Path" as path - FROM "Movies" m - JOIN "MovieMetadata" mm ON m."MovieMetadataId" = mm."Id" - WHERE mm."ImdbId" IS NOT NULL AND mm."ImdbId" != '' - ORDER BY mm."Title" - LIMIT 5 - """ - - with radarr_db._get_connection() as conn: - if radarr_db.db_type == "postgresql": - import psycopg2.extras - cursor = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) - else: - cursor = conn.cursor() - - cursor.execute(query) - movies = cursor.fetchall() - - print(f"โœ… Successfully queried Radarr database: {len(movies)} sample movies found") - - # Show sample data - for i, movie in enumerate(movies): - if radarr_db.db_type == "postgresql": - imdb_id, title, year = movie['imdb_id'], movie['title'], movie['year'] - else: - imdb_id, title, year = movie[0], movie[2], movie[3] - print(f" Sample {i+1}: {title} ({year}) - {imdb_id}") - - except Exception as e: - print(f"โŒ Query execution failed: {e}") - return False - - print("\nโœ… All tests passed! Bulk update script should work correctly.") - return True - -if __name__ == "__main__": - success = test_bulk_update_dry_run() - if not success: - print("\nโŒ Bulk update test failed!") - sys.exit(1) - else: - print("\n๐ŸŽ‰ Bulk update test successful!") \ No newline at end of file diff --git a/test_end_to_end.py b/test_end_to_end.py deleted file mode 100644 index 2dbca71..0000000 --- a/test_end_to_end.py +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env python3 -""" -End-to-end testing script for NFOGuard workflow -Tests the complete flow: Database -> Manual Scan -> NFO Creation -""" - -import os -import sys -import requests -import time -from pathlib import Path - -def test_server_health(): - """Test if the server is running and healthy""" - try: - response = requests.get("http://localhost:8080/health", timeout=10) - if response.status_code == 200: - health_data = response.json() - print(f"โœ… Server is healthy: {health_data['status']}") - print(f" Version: {health_data['version']}") - print(f" Database: {health_data['database_status']}") - if health_data.get('radarr_database'): - radarr_status = health_data['radarr_database']['status'] - print(f" Radarr DB: {radarr_status}") - return True - else: - print(f"โŒ Server health check failed: {response.status_code}") - return False - except Exception as e: - print(f"โŒ Cannot connect to server: {e}") - return False - -def test_database_performance(): - """Test the database performance endpoint""" - try: - response = requests.get("http://localhost:8080/debug/movie/tt1674782", timeout=10) - if response.status_code == 200: - debug_data = response.json() - if debug_data.get('detected_import_date'): - print(f"โœ… Database query works: {debug_data['detected_import_date']}") - print(f" Source: {debug_data['import_source']}") - return True - else: - print(f"โŒ No import date found for test movie") - return False - else: - print(f"โŒ Debug endpoint failed: {response.status_code}") - return False - except Exception as e: - print(f"โŒ Database test failed: {e}") - return False - -def test_manual_scan(): - """Test the manual scan functionality""" - try: - print("๐Ÿ” Starting manual movie scan...") - response = requests.post( - "http://localhost:8080/manual/scan?scan_type=movies", - timeout=30 - ) - - if response.status_code == 200: - scan_data = response.json() - print(f"โœ… Manual scan started: {scan_data['message']}") - - # Wait a bit for processing - print("โณ Waiting for scan to process...") - time.sleep(10) - - # Check batch status - batch_response = requests.get("http://localhost:8080/batch/status", timeout=10) - if batch_response.status_code == 200: - batch_data = batch_response.json() - print(f" Pending items: {batch_data['pending_count']}") - print(f" Processing items: {batch_data['processing_count']}") - - return True - else: - print(f"โŒ Manual scan failed: {response.status_code}") - return False - except Exception as e: - print(f"โŒ Manual scan test failed: {e}") - return False - -def test_stats_endpoint(): - """Test database stats""" - try: - response = requests.get("http://localhost:8080/stats", timeout=10) - if response.status_code == 200: - stats = response.json() - print(f"โœ… Database stats:") - print(f" Movies: {stats.get('movie_count', 0)}") - print(f" TV Series: {stats.get('series_count', 0)}") - print(f" Episodes: {stats.get('episode_count', 0)}") - return True - else: - print(f"โŒ Stats endpoint failed: {response.status_code}") - return False - except Exception as e: - print(f"โŒ Stats test failed: {e}") - return False - -def run_end_to_end_test(): - """Run complete end-to-end test suite""" - print("๐ŸŽฏ NFOGuard End-to-End Testing") - print("=" * 50) - - tests = [ - ("Server Health", test_server_health), - ("Database Performance", test_database_performance), - ("Manual Scan", test_manual_scan), - ("Database Stats", test_stats_endpoint) - ] - - passed = 0 - failed = 0 - - for test_name, test_func in tests: - print(f"\n๐Ÿงช Running: {test_name}") - try: - if test_func(): - passed += 1 - print(f"โœ… {test_name} passed") - else: - failed += 1 - print(f"โŒ {test_name} failed") - except Exception as e: - failed += 1 - print(f"โŒ {test_name} crashed: {e}") - - print(f"\n๐Ÿ“Š Test Results:") - print(f" โœ… Passed: {passed}") - print(f" โŒ Failed: {failed}") - print(f" ๐Ÿ“ˆ Success Rate: {passed}/{passed+failed} ({100*passed/(passed+failed):.1f}%)") - - return failed == 0 - -if __name__ == "__main__": - success = run_end_to_end_test() - - if success: - print("\n๐ŸŽ‰ All end-to-end tests passed!") - print("\n๐Ÿ“‹ Next Steps:") - print("1. Run bulk update: python3 bulk_update_movies.py") - print("2. Test with real movie: curl http://localhost:8080/debug/movie/tt1674782") - print("3. Check NFO file creation in /media/Movies/movies/") - print("4. Verify file timestamps match import dates") - else: - print("\n๐Ÿ’ฅ Some tests failed - check the logs above") - sys.exit(1) \ No newline at end of file diff --git a/test_movie_scan.py b/test_movie_scan.py deleted file mode 100644 index f3d1d5a..0000000 --- a/test_movie_scan.py +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env python3 -""" -Test script to verify movie directory scanning logic -Simulates what the manual scan would do with correct paths -""" - -import os -import sys -from pathlib import Path - -def test_movie_scanning(): - """Test movie directory scanning logic""" - print("๐Ÿ” Testing Movie Directory Scanning") - print("=" * 50) - - # Test paths - use the corrected paths from the code - movie_paths = [ - Path("/media/Movies/movies"), - Path("/media/Movies/movies6") - ] - - print("๐Ÿ“ Testing paths:") - for path in movie_paths: - print(f" {path}") - - # Import the NFO manager to test the IMDb parsing - try: - from core.nfo_manager import NFOManager - nfo_manager = NFOManager() - print("โœ… Successfully imported NFOManager") - except Exception as e: - print(f"โŒ Failed to import NFOManager: {e}") - return False - - # Simulate the scanning logic - total_movies_found = 0 - - for media_path in movie_paths: - print(f"\n๐ŸŽฌ Scanning: {media_path}") - - # Check if path exists (this will fail on your local machine but show the logic) - if not media_path.exists(): - print(f" โš ๏ธ Path doesn't exist locally (expected on dev machine): {media_path}") - print(f" โ„น๏ธ On production server, this path should contain directories like:") - print(f" Example Movie [imdb-tt1234567] (2023)/") - print(f" Another Movie [imdb-tt7654321] (2022)/") - continue - - movie_count = 0 - for item in media_path.iterdir(): - if item.is_dir(): - imdb_id = nfo_manager.parse_imdb_from_path(item) - if imdb_id: - movie_count += 1 - print(f" โœ… Found movie: {item.name} -> {imdb_id}") - else: - print(f" โญ๏ธ Skipped (no IMDb ID): {item.name}") - - print(f" ๐Ÿ“Š Found {movie_count} movies with IMDb IDs in {media_path}") - total_movies_found += movie_count - - print(f"\n๐ŸŽฏ Total movies that would be processed: {total_movies_found}") - - # Test IMDb ID parsing with sample directory names - print(f"\n๐Ÿงช Testing IMDb ID Parsing Logic:") - test_names = [ - "The Dark Knight [imdb-tt0468569] (2008)", - "Inception [imdb-tt1375666] (2010)", - "Invalid Movie Name (2020)", - "Movie Without IMDb (2021)", - "Another Movie [imdb-tt1234567] (2023)" - ] - - for test_name in test_names: - test_path = Path(f"/fake/path/{test_name}") - imdb_id = nfo_manager.parse_imdb_from_path(test_path) - if imdb_id: - print(f" โœ… '{test_name}' -> {imdb_id}") - else: - print(f" โŒ '{test_name}' -> No IMDb ID found") - - return True - -def show_manual_scan_instructions(): - """Show instructions for testing on the production server""" - print("\n" + "="*60) - print("๐Ÿ“‹ NEXT STEPS - Test on Production Server") - print("="*60) - print() - print("1. ๐Ÿ”ง Update your environment variables:") - print(" export MOVIE_PATHS='/media/Movies/movies,/media/Movies/movies6'") - print(" export TV_PATHS='/media/TV/tv,/media/TV/tv6'") - print() - print("2. ๐Ÿš€ Test the manual scan:") - print(" curl -X POST 'http://localhost:8080/manual/scan?scan_type=movies'") - print() - print("3. ๐Ÿ“Š Check the logs for movie processing:") - print(" docker logs | grep -E '(Processing movie|Completed movie scan)'") - print() - print("4. ๐Ÿงช Test the bulk update script:") - print(" python3 test_bulk_update.py") - print(" python3 bulk_update_movies.py") - print() - print("5. โœ… Verify end-to-end workflow:") - print(" curl http://localhost:8080/debug/movie/tt1674782") - print() - -if __name__ == "__main__": - success = test_movie_scanning() - - if success: - print("\nโœ… Local testing completed successfully!") - show_manual_scan_instructions() - else: - print("\nโŒ Local testing failed!") - sys.exit(1) \ No newline at end of file