From 4685c970affdeb8c5cd15b1d4433bb4099337f7c Mon Sep 17 00:00:00 2001
From: SBCrumb <sbcrumb@kickthetree.com>
Date: Tue, 9 Sep 2025 10:20:46 -0400
Subject: [PATCH] add api secrets seperate from .env

---
 .env.secrets.template |  28 ++++++++
 .env.template         |  19 +++---
 .gitignore            |  60 +++++++++++++++++
 SETUP.md              | 148 ++++++++++++++++++++++++++++++++++++++++++
 nfoguard.py           |  55 +++++++++++++++-
 requirements.txt      |   1 +
 6 files changed, 298 insertions(+), 13 deletions(-)
 create mode 100644 .env.secrets.template
 create mode 100644 .gitignore
 create mode 100644 SETUP.md

diff --git a/.env.secrets.template b/.env.secrets.template
new file mode 100644
index 0000000..60cf9dd
--- /dev/null
+++ b/.env.secrets.template
@@ -0,0 +1,28 @@
+# NFOGuard Secrets Configuration Template
+# Copy this to .env.secrets and add your actual API keys
+# This file should NOT be committed to version control
+# Add .env.secrets to your .gitignore
+
+# ===========================================
+# RADARR DATABASE CREDENTIALS
+# ===========================================
+# Database password for PostgreSQL connection
+RADARR_DB_PASSWORD=your_radarr_db_password
+
+# ===========================================
+# API KEYS (SENSITIVE DATA)
+# ===========================================
+# TMDB API key for digital release dates
+TMDB_API_KEY=your_tmdb_api_key
+
+# OMDb API key for DVD/digital release dates (optional)
+OMDB_API_KEY=your_omdb_api_key
+
+# Radarr API key (optional - for fallback only)
+RADARR_API_KEY=your_radarr_api_key
+
+# Sonarr API key (optional - for TV processing)
+SONARR_API_KEY=your_sonarr_api_key
+
+# Jellyseerr API key (optional)
+JELLYSEERR_API_KEY=your_jellyseerr_api_key
\ No newline at end of file
diff --git a/.env.template b/.env.template
index 76399d7..a781099 100644
--- a/.env.template
+++ b/.env.template
@@ -19,38 +19,37 @@ DB_PATH=/app/data/media_dates.db
 # RADARR DATABASE CONNECTION (REQUIRED)
 # ===========================================
 # Connection to Radarr's PostgreSQL database
+# NOTE: Database password should be set in .env.secrets file
 RADARR_DB_HOST=radarr-postgres
 RADARR_DB_PORT=5432
 RADARR_DB_NAME=radarr
 RADARR_DB_USER=radarr
-RADARR_DB_PASSWORD=your_radarr_db_password
 
 # ===========================================
 # RADARR API (OPTIONAL - for fallback)
 # ===========================================
 # Only needed if you want API fallback (not recommended for performance)
+# NOTE: API key should be set in .env.secrets file
 RADARR_URL=http://radarr:7878
-RADARR_API_KEY=your_radarr_api_key
 
 # ===========================================
 # SONARR API (OPTIONAL)
 # ===========================================
 # Only needed for TV processing
+# NOTE: API key should be set in .env.secrets file
 SONARR_URL=http://sonarr:8989
-SONARR_API_KEY=your_sonarr_api_key
 
 # ===========================================
 # EXTERNAL APIs FOR DIGITAL RELEASE DATES
 # ===========================================
-# TMDB API for digital release dates (recommended)
-TMDB_API_KEY=your_tmdb_api_key
+# NOTE: All API keys should be set in .env.secrets file
+# See .env.secrets.template for required keys:
+# - TMDB_API_KEY (recommended for digital release dates)
+# - OMDB_API_KEY (optional for DVD/digital release dates)
+# - JELLYSEERR_API_KEY (optional for additional digital release data)
 
-# OMDb API for DVD/digital release dates (optional)
-OMDB_API_KEY=your_omdb_api_key
-
-# Jellyseerr for additional digital release data (optional)
+# Jellyseerr URL (optional)
 JELLYSEERR_URL=http://jellyseerr:5055
-JELLYSEERR_API_KEY=your_jellyseerr_api_key
 
 # ===========================================
 # NFO MANAGEMENT
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..4a230d2
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,60 @@
+# Environment files with sensitive data
+.env
+.env.secrets
+.env.local
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Virtual environments
+venv/
+env/
+ENV/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Logs
+logs/
+*.log
+
+# Database files
+*.db
+*.sqlite
+*.sqlite3
+
+# Temporary files
+tmp/
+temp/
+*.tmp
+
+# Docker
+.dockerignore
\ No newline at end of file
diff --git a/SETUP.md b/SETUP.md
new file mode 100644
index 0000000..fd25416
--- /dev/null
+++ b/SETUP.md
@@ -0,0 +1,148 @@
+# NFOGuard Setup Guide
+
+## 🔐 Secure Configuration Setup
+
+NFOGuard now uses a **two-file configuration system** for better security and easier troubleshooting:
+
+- **`.env`** - Main configuration (safe to share for debugging)
+- **`.env.secrets`** - Sensitive API keys and passwords (never commit to git)
+
+### Step 1: Copy Configuration Templates
+
+```bash
+# Copy main configuration
+cp .env.template .env
+
+# Copy secrets configuration  
+cp .env.secrets.template .env.secrets
+```
+
+### Step 2: Configure Main Settings
+
+Edit your `.env` file with your specific paths and preferences:
+
+```bash
+# Media paths (adjust to your directory structure)
+TV_PATHS=/media/TV/tv,/media/TV/tv6
+MOVIE_PATHS=/media/Movies/movies,/media/Movies/movies6
+
+# Database connection details
+RADARR_DB_HOST=radarr-postgres
+RADARR_DB_PORT=5432
+RADARR_DB_NAME=radarr
+RADARR_DB_USER=radarr
+
+# Processing preferences
+PREFER_RELEASE_DATES_OVER_FILE_DATES=true
+ALLOW_FILE_DATE_FALLBACK=false
+RELEASE_DATE_PRIORITY=digital,physical,theatrical
+```
+
+### Step 3: Configure Secrets
+
+Edit your `.env.secrets` file with your actual API keys and passwords:
+
+```bash
+# Database password
+RADARR_DB_PASSWORD=your_actual_radarr_password
+
+# TMDB API key (required for release date detection)
+TMDB_API_KEY=your_actual_tmdb_api_key
+
+# Optional API keys
+RADARR_API_KEY=your_radarr_api_key
+SONARR_API_KEY=your_sonarr_api_key
+OMDB_API_KEY=your_omdb_api_key
+```
+
+### Step 4: Verify Configuration
+
+Test your setup:
+
+```bash
+# Test database connections
+curl -X POST "http://localhost:8080/test/bulk-update"
+
+# Test movie scanning
+curl -X POST "http://localhost:8080/test/movie-scan"
+
+# Check system health
+curl "http://localhost:8080/health"
+```
+
+## 🔒 Security Features
+
+### API Key Masking
+All API keys and passwords are automatically masked in logs:
+```
+[2025-09-09T12:34:56] INFO: TMDB API call with key=***masked***
+[2025-09-09T12:34:56] INFO: Database connection password=***masked***
+```
+
+### Sensitive Data Separation
+- **Main `.env`**: Paths, preferences, URLs (safe to share)
+- **`.env.secrets`**: API keys, passwords (never commit to version control)
+- **Automatic loading**: Both files loaded automatically at startup
+
+### Git Protection
+The `.gitignore` file prevents accidental commits:
+```
+.env
+.env.secrets
+.env.local
+```
+
+## 🛠 Troubleshooting
+
+### "Environment files not loaded" Warning
+Install python-dotenv:
+```bash
+pip install python-dotenv==1.0.0
+# or
+docker-compose build  # rebuilds with updated requirements.txt
+```
+
+### Sharing Configuration for Help
+You can safely share your `.env` file for debugging since it contains no sensitive data. The `.env.secrets` file should never be shared.
+
+### Migration from Old Setup
+If you have an existing `.env` with API keys:
+1. Move all `*_API_KEY` and `*_PASSWORD` variables to `.env.secrets`
+2. Remove sensitive data from `.env` 
+3. Restart NFOGuard
+
+## 🎯 Docker Compose Example
+
+```yaml
+version: '3.8'
+services:
+  nfoguard:
+    image: ghcr.io/your-org/nfoguard:latest
+    container_name: nfoguard
+    ports:
+      - "8080:8080"
+    volumes:
+      - /path/to/your/media:/media:rw
+      - ./data:/app/data
+      - ./.env:/app/.env:ro          # Main configuration
+      - ./.env.secrets:/app/.env.secrets:ro  # Secrets
+    environment:
+      - PORT=8080
+    depends_on:
+      - radarr-postgres
+```
+
+## 📋 Configuration Reference
+
+### Main Configuration (.env)
+- **Paths**: `TV_PATHS`, `MOVIE_PATHS`, `DB_PATH`
+- **Processing**: `MOVIE_PRIORITY`, `RELEASE_DATE_PRIORITY`
+- **Features**: `MANAGE_NFO`, `FIX_DIR_MTIMES`, `LOCK_METADATA`
+- **URLs**: `RADARR_URL`, `SONARR_URL`, `JELLYSEERR_URL`
+
+### Secrets Configuration (.env.secrets)  
+- **Database**: `RADARR_DB_PASSWORD`
+- **APIs**: `TMDB_API_KEY`, `OMDB_API_KEY`, `RADARR_API_KEY`, `SONARR_API_KEY`
+- **Optional**: `JELLYSEERR_API_KEY`
+
+This setup makes NFOGuard more secure while keeping configuration manageable for troubleshooting and deployment.
\ No newline at end of file
diff --git a/nfoguard.py b/nfoguard.py
index 3f99796..aa9f5fe 100644
--- a/nfoguard.py
+++ b/nfoguard.py
@@ -51,19 +51,68 @@ def _setup_file_logging():
     logger.addHandler(file_handler)
     return logger
 
+def _mask_sensitive_data(msg: str) -> str:
+    """Mask API keys and other sensitive data in log messages"""
+    import re
+    
+    # List of patterns to mask
+    sensitive_patterns = [
+        (r'api_key=([a-zA-Z0-9_\-]+)', r'api_key=***masked***'),
+        (r'password=([^\s&]+)', r'password=***masked***'),
+        (r'token=([a-zA-Z0-9_\-]+)', r'token=***masked***'),
+        (r'key=([a-zA-Z0-9_\-]{8,})', r'key=***masked***'),  # Keys longer than 8 chars
+        (r'([a-zA-Z0-9]{32,})', lambda m: m.group(1)[:8] + '***masked***' if len(m.group(1)) > 16 else m.group(1))  # Long strings likely to be keys
+    ]
+    
+    masked_msg = msg
+    for pattern, replacement in sensitive_patterns:
+        if isinstance(replacement, str):
+            masked_msg = re.sub(pattern, replacement, masked_msg, flags=re.IGNORECASE)
+        else:
+            masked_msg = re.sub(pattern, replacement, masked_msg, flags=re.IGNORECASE)
+    
+    return masked_msg
+
 def _log(level: str, msg: str):
-    """Enhanced logging that writes to both console and file"""
-    print(f"[{datetime.now(timezone.utc).isoformat(timespec='seconds')}] {level}: {msg}")
+    """Enhanced logging that writes to both console and file with sensitive data masking"""
+    masked_msg = _mask_sensitive_data(msg)
+    print(f"[{datetime.now(timezone.utc).isoformat(timespec='seconds')}] {level}: {masked_msg}")
     
     try:
         file_logger = _setup_file_logging()
-        getattr(file_logger, level.lower(), file_logger.info)(msg)
+        getattr(file_logger, level.lower(), file_logger.info)(masked_msg)
     except Exception as e:
         print(f"File logging error: {e}")
 
 # Initialize logging
 _setup_file_logging()
 
+def _load_environment_files():
+    """Load environment variables from .env and optionally .env.secrets"""
+    from pathlib import Path
+    
+    # Try to load from python-dotenv if available
+    try:
+        from dotenv import load_dotenv
+        
+        # Load main .env file
+        env_file = Path(".env")
+        if env_file.exists():
+            load_dotenv(env_file)
+            _log("INFO", f"Loaded environment from {env_file}")
+        
+        # Load secrets file if it exists
+        secrets_file = Path(".env.secrets")
+        if secrets_file.exists():
+            load_dotenv(secrets_file)
+            _log("INFO", f"Loaded secrets from {secrets_file}")
+            
+    except ImportError:
+        _log("WARNING", "python-dotenv not available - environment files not loaded")
+        
+# Load environment files at startup
+_load_environment_files()
+
 def _bool_env(name: str, default: bool) -> bool:
     v = os.environ.get(name)
     if v is None:
diff --git a/requirements.txt b/requirements.txt
index cd6f24f..c15e933 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,3 +2,4 @@ fastapi==0.104.1
 uvicorn[standard]==0.24.0
 pydantic==2.5.0
 psycopg2-binary==2.9.9
+python-dotenv==1.0.0