commit 6ce10f673e3a4559004fbdacb004b86d5ea81861 Author: CaffeineFueled Date: Tue Apr 8 23:41:24 2025 +0200 fresh start diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..53d5862 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,43 @@ +# Git +.git +.gitignore + +# Python cache files +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +.pytest_cache/ +htmlcov/ +.coverage +.coverage.* +.cache/ + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Docker +Dockerfile +docker-compose.yml +.dockerignore + +# Database +*.db +*.sqlite3 +*.json + +# Logs +*.log + +# VS Code +.vscode/ + +# Local data +uniteddomain.csv \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7222a10 --- /dev/null +++ b/.gitignore @@ -0,0 +1,98 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST +*.manifest +*.spec +pip-log.txt +pip-delete-this-directory.txt + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Unit tests / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ +pytestdebug.log + +# Database +*.db +*.sqlite3 +*.sqlite +domains_db.json + +# FastAPI project +domains_db.json +*.csv +!requirements.txt + +# Logs +logs/ +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# IDE specific files +.idea/ +.vscode/ +*.swp +*.swo +*~ +.DS_Store + +# Docker +.docker/ +docker-data/ +data/ + +# Jupyter Notebook +.ipynb_checkpoints + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Node dependencies +node_modules/ + +# Temporary files +tmp/ +temp/ +*.tmp +.temp \ No newline at end of file diff --git a/CONTAINER_INSTRUCTIONS.md b/CONTAINER_INSTRUCTIONS.md new file mode 100644 index 0000000..5b8b89a --- /dev/null +++ b/CONTAINER_INSTRUCTIONS.md @@ -0,0 +1,261 @@ +# Container Instructions for FastAPI Domains Application + +This guide explains how to run the FastAPI Domains application in a secure rootless container with persistent data storage using Podman or Docker. + +## Prerequisites + +- [Podman](https://podman.io/getting-started/installation) (version 3.0 or higher) or [Docker](https://docs.docker.com/get-docker/) (version 20.10 or higher) + +## Security Features + +This deployment includes the following security features: + +1. **Rootless container**: The application runs as a non-root user (UID 1000) +2. **Read-only filesystem**: The container's filesystem is mounted read-only +3. **Dropped capabilities**: All Linux capabilities are dropped +4. **No privilege escalation**: The container cannot gain additional privileges +5. **Minimal base image**: Uses a slim Python image to reduce attack surface +6. **Non-privileged ports**: Uses port 8000 instead of privileged ports (<1024) +7. **Persistent volume**: Data is stored in a volume for persistence + +## Quick Start with Podman + +### Building the Container + +```bash +podman build -t fastapi-domains:latest . +``` + +### Creating a Volume + +```bash +podman volume create domain-data +``` + +### Running the Container + +```bash +podman run --name fastapi-domains \ + -p 8000:8000 \ + -v domain-data:/home/appuser/app/data:Z \ + -e DB_DIR=/home/appuser/app/data \ + --security-opt no-new-privileges:true \ + --read-only \ + --tmpfs /tmp \ + --cap-drop ALL \ + --user 1000:1000 \ + -d fastapi-domains:latest +``` + +### Checking Container Status + +```bash +podman ps +``` + +### Accessing the Application + +Open your browser to: +``` +http://localhost:8000 +``` + +## Quick Start with Docker + +### Building the Container + +```bash +docker build -t fastapi-domains:latest . +``` + +### Creating a Volume + +```bash +docker volume create domain-data +``` + +### Running the Container + +```bash +docker run --name fastapi-domains \ + -p 8000:8000 \ + -v domain-data:/home/appuser/app/data \ + -e DB_DIR=/home/appuser/app/data \ + --security-opt no-new-privileges:true \ + --read-only \ + --tmpfs /tmp \ + --cap-drop ALL \ + --user 1000:1000 \ + -d fastapi-domains:latest +``` + +### Checking Container Status + +```bash +docker ps +``` + +### Accessing the Application + +Open your browser to: +``` +http://localhost:8000 +``` + +## Persistent Data + +The application stores all data in a volume named `domain-data`. This volume persists even when the container is stopped or removed. + +To see information about the volume: + +**Podman:** +```bash +podman volume inspect domain-data +``` + +**Docker:** +```bash +docker volume inspect domain-data +``` + +## Maintenance + +### View Logs + +**Podman:** +```bash +podman logs fastapi-domains +``` + +**Docker:** +```bash +docker logs fastapi-domains +``` + +### Restart the Application + +**Podman:** +```bash +podman restart fastapi-domains +``` + +**Docker:** +```bash +docker restart fastapi-domains +``` + +### Stop the Application + +**Podman:** +```bash +podman stop fastapi-domains +``` + +**Docker:** +```bash +docker stop fastapi-domains +``` + +### Remove the Container + +**Podman:** +```bash +podman rm fastapi-domains +``` + +**Docker:** +```bash +docker rm fastapi-domains +``` + +## Backup and Restore + +### Backup the Database + +**Podman:** +```bash +podman run --rm -v domain-data:/data:Z -v ./:/backup:Z alpine sh -c "cp /data/domains_db.json /backup/domains_backup_$(date +%Y%m%d).json" +``` + +**Docker:** +```bash +docker run --rm -v domain-data:/data -v $(pwd):/backup alpine sh -c "cp /data/domains_db.json /backup/domains_backup_$(date +%Y%m%d).json" +``` + +### Restore from Backup + +**Podman:** +```bash +podman run --rm -v domain-data:/data:Z -v ./:/backup:Z alpine sh -c "cp /backup/domains_backup_YYYYMMDD.json /data/domains_db.json" +``` + +**Docker:** +```bash +docker run --rm -v domain-data:/data -v $(pwd):/backup alpine sh -c "cp /backup/domains_backup_YYYYMMDD.json /data/domains_db.json" +``` + +## Creating a Systemd Service (Podman Only) + +1. Generate a systemd service file: + +```bash +mkdir -p ~/.config/systemd/user +podman generate systemd --name fastapi-domains --files --new +``` + +2. Move the generated file: + +```bash +mv container-fastapi-domains.service ~/.config/systemd/user/ +``` + +3. Enable and start the service: + +```bash +systemctl --user enable container-fastapi-domains.service +systemctl --user start container-fastapi-domains.service +``` + +4. Check service status: + +```bash +systemctl --user status container-fastapi-domains.service +``` + +## Troubleshooting + +### Check Container Status + +**Podman:** +```bash +podman ps -a +``` + +**Docker:** +```bash +docker ps -a +``` + +### Inspect the Container + +**Podman:** +```bash +podman inspect fastapi-domains +``` + +**Docker:** +```bash +docker inspect fastapi-domains +``` + +### Access Container Shell + +**Podman:** +```bash +podman exec -it fastapi-domains bash +``` + +**Docker:** +```bash +docker exec -it fastapi-domains bash +``` \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..20b0cd0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,36 @@ +# Use Python 3.11 slim image for a smaller footprint +FROM python:3.11-slim + +# Set environment variables +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + HOME=/home/appuser \ + APP_HOME=/home/appuser/app + +# Create non-root user and setup directories +RUN groupadd -g 1000 appgroup && \ + useradd -m -u 1000 -g appgroup -s /bin/bash -d ${HOME} appuser && \ + mkdir -p ${APP_HOME} && \ + mkdir -p ${APP_HOME}/data && \ + chown -R appuser:appgroup ${HOME} + +# Set the working directory +WORKDIR ${APP_HOME} + +# Install dependencies +COPY --chown=appuser:appgroup requirements.txt ${APP_HOME}/ +RUN pip install --no-cache-dir -r requirements.txt + +# Copy application code +COPY --chown=appuser:appgroup . ${APP_HOME}/ + +# Switch to non-root user +USER appuser + +# Expose port +EXPOSE 8000 + +# Command to run the application +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..c0577e1 --- /dev/null +++ b/README.md @@ -0,0 +1,107 @@ +# Domain and DNS Management System + +A FastAPI application that allows users to upload CSV files containing domain and DNS records, storing them in a database and displaying them via an interactive UI and API interface. + +## Features + +- File upload system with detailed upload history, including ability to delete uploads +- Main SLD view with filtering by upload and time +- Comprehensive DNS Records view with multi-faceted filtering +- TinyDB database storage with precise timestamps for data persistence +- Deduplicated DNS records (by domain, class, and type) and domains (by FQDN) showing most recent entries +- Option to view only records from a specific upload +- Clean, simplified interface showing only the latest records +- Server-side filtering by upload, record type, class, TLD, and SLD (no JavaScript needed) +- Provides comprehensive RESTful API endpoints for programmatic access +- Responsive design that works on desktop and mobile + +## Containerized Deployment + +The application can be easily deployed in a secure rootless container using Podman or Docker. + +For detailed container instructions, see [CONTAINER_INSTRUCTIONS.md](CONTAINER_INSTRUCTIONS.md). + +Quick start: +```bash +# Make the run script executable +chmod +x run_container.sh + +# Run the container (automatically uses podman if available, falls back to docker) +./run_container.sh +``` + +## Setup with Virtual Environment (Development) + +1. Create a virtual environment: + ``` + python -m venv venv + ``` + +2. Activate the virtual environment: + - On Windows: + ``` + venv\Scripts\activate + ``` + - On macOS/Linux: + ``` + source venv/bin/activate + ``` + +3. Install dependencies: + ``` + pip install -r requirements.txt + ``` + +4. Run the application: + ``` + python main.py + ``` + +5. Access the application: + - Web interface: http://localhost:8000 + - API endpoints: + - http://localhost:8000/api/slds + - http://localhost:8000/api/slds/{sld} + +6. Deactivate the virtual environment when finished: + ``` + deactivate + ``` + +## CSV File Format + +The application accepts CSV files with the following format: + +``` +domain.example.com,3600,IN,A,192.168.1.1 +domain.example.com,3600,IN,MX,10 mail.example.com. +subdomain.example.com,3600,IN,CNAME,domain.example.com. +``` + +Where columns are: +1. Domain name (fully qualified domain name) +2. TTL (Time To Live) in seconds +3. Record Class (usually IN for Internet) +4. Record Type (A, AAAA, MX, CNAME, TXT, etc.) +5. Record Data (IP address, hostname, or other data depending on record type) + +## API Endpoints + +- `/api/uploads` - Get all uploads +- `/api/slds` - Get all SLDs (Second Level Domains) +- `/api/slds/{sld}` - Get domains by SLD +- `/api/dns` - Get all DNS records +- `/api/dns/types` - Get unique values for filters + +### Query Parameters + +You can filter the API results using the following query parameters: + +- `upload_id` - Filter by specific upload +- `record_type` - Filter by DNS record type +- `record_class` - Filter by DNS record class +- `tld` - Filter by Top Level Domain +- `sld` - Filter by Second Level Domain +- `domain` - Search by domain name + +Example: `/api/dns?record_type=A&tld=com&upload_id=upload_20250408120000` diff --git a/main.py b/main.py new file mode 100644 index 0000000..4baf1cb --- /dev/null +++ b/main.py @@ -0,0 +1,465 @@ +import csv +import re +import io +import datetime +from fastapi import FastAPI, Request, HTTPException, Query, UploadFile, File, Form +from fastapi.responses import HTMLResponse, RedirectResponse +from fastapi.staticfiles import StaticFiles +from fastapi.templating import Jinja2Templates +import uvicorn +from pathlib import Path +from typing import List, Dict, Optional, Set +from tinydb import TinyDB, Query as TinyDBQuery + +# Initialize FastAPI app +app = FastAPI(title="Domain Viewer", description="Upload and view domains and DNS records") + +# Setup templates +templates = Jinja2Templates(directory="templates") + +# Setup TinyDB with configurable path +import os +DB_DIR = os.environ.get('DB_DIR', '.') +DB_PATH = Path(f"{DB_DIR}/domains_db.json") +db = TinyDB(DB_PATH) +domains_table = db.table('domains') +dns_records_table = db.table('dns_records') +uploads_table = db.table('uploads') + +# Process a domain entry and return its components +def process_domain_entry(domain_entry): + # Remove trailing dot if present + if domain_entry.endswith('.'): + domain_entry = domain_entry[:-1] + + # Parse domain components + parts = domain_entry.split('.') + if len(parts) > 1: + # For domain.tld format + if len(parts) == 2: + sld = parts[0] # Second Level Domain + tld = parts[1] # Top Level Domain + domain_info = { + "sld": sld, + "tld": tld, + "full_domain": domain_entry + } + # For subdomain.domain.tld format + else: + sld = parts[-2] # Second Level Domain + tld = parts[-1] # Top Level Domain + subdomain = '.'.join(parts[:-2]) # Subdomains + domain_info = { + "sld": sld, + "tld": tld, + "full_domain": domain_entry, + "subdomain": subdomain + } + return domain_info + return None + +# Upload CSV file and store in database +async def process_csv_upload(file_content, upload_id, description=None): + domains_to_insert = [] + dns_records_to_insert = [] + unique_domains = set() + timestamp = datetime.datetime.now().isoformat() + row_count = 0 + + try: + # Read CSV content + file_text = file_content.decode('utf-8') + csv_reader = csv.reader(io.StringIO(file_text)) + + # Print first few lines for debugging + preview_lines = file_text.split('\n')[:5] + print(f"CSV preview (first 5 lines):") + for i, line in enumerate(preview_lines): + print(f" Line {i+1}: {line}") + + for row_num, row in enumerate(csv_reader, 1): + row_count += 1 + if not row: # Skip empty rows + print(f"Skipping empty row at line {row_num}") + continue + + # Extract domain from first column + domain_entry = row[0] + + # Process domain for domains table + domain_info = process_domain_entry(domain_entry) + if domain_info: + # Create a unique key to avoid duplicates within this upload + unique_key = f"{domain_info['sld']}.{domain_info['tld']}" + + if unique_key not in unique_domains: + unique_domains.add(unique_key) + # Add upload metadata + domain_info['upload_id'] = upload_id + domain_info['timestamp'] = timestamp + domains_to_insert.append(domain_info) + else: + print(f"Warning: Could not process domain entry at line {row_num}: {domain_entry}") + + # Process DNS record if we have enough fields + if len(row) >= 5: + domain = row[0] + ttl = row[1] + record_class = row[2] + record_type = row[3] + record_data = ','.join(row[4:]) # Join remaining parts as record data + + # Remove trailing dot from domain if present + if domain.endswith('.'): + domain = domain[:-1] + + # Parse domain components + domain_parts = domain.split('.') + + # Create entry + entry = { + "domain": domain, + "ttl": ttl, + "record_class": record_class, + "record_type": record_type, + "record_data": record_data, + "upload_id": upload_id, + "timestamp": timestamp + } + + # Add domain components + if len(domain_parts) > 1: + if domain_parts[0].startswith('_'): # Service records like _dmarc + entry["service"] = domain_parts[0] + # Adjust domain parts + domain_parts = domain_parts[1:] + + # For domain.tld format + if len(domain_parts) == 2: + entry["sld"] = domain_parts[0] # Second Level Domain + entry["tld"] = domain_parts[1] # Top Level Domain + # For subdomain.domain.tld format + elif len(domain_parts) > 2: + entry["sld"] = domain_parts[-2] # Second Level Domain + entry["tld"] = domain_parts[-1] # Top Level Domain + entry["subdomain"] = '.'.join(domain_parts[:-2]) # Subdomains + + dns_records_to_insert.append(entry) + + print(f"Processed {row_count} rows from CSV") + print(f"Records to insert: {len(domains_to_insert)} domains, {len(dns_records_to_insert)} DNS records") + + # Insert data into tables + if domains_to_insert: + print(f"Inserting {len(domains_to_insert)} domains into database") + domains_table.insert_multiple(domains_to_insert) + else: + print("No domains to insert") + + if dns_records_to_insert: + print(f"Inserting {len(dns_records_to_insert)} DNS records into database") + dns_records_table.insert_multiple(dns_records_to_insert) + else: + print("No DNS records to insert") + + return len(domains_to_insert), len(dns_records_to_insert) + + except Exception as e: + import traceback + print(f"Error processing CSV file: {e}") + print(traceback.format_exc()) + return 0, 0 + +# Load domains from database - deduplicated by full domain name +def load_domains(specific_upload_id: str = None) -> List[Dict]: + try: + domains = domains_table.all() + + # If a specific upload ID is provided, only show domains from that upload + if specific_upload_id: + domains = [d for d in domains if d.get('upload_id') == specific_upload_id] + return domains + + # Sort by timestamp in descending order (newest first) + domains.sort(key=lambda x: x.get('timestamp', ''), reverse=True) + + # Create a dictionary to track unique domains by full domain name + unique_domains = {} + + for domain in domains: + # Create a unique key based on the full domain name + unique_key = domain.get('full_domain', '') + + # Only keep the most recent entry for each unique domain + if unique_key and unique_key not in unique_domains: + domain['is_latest'] = True + unique_domains[unique_key] = domain + + # Return the deduplicated list + return list(unique_domains.values()) + except Exception as e: + print(f"Error loading domains from database: {e}") + return [] + +# Load DNS entries from database - deduplicated by domain, class, and type (no history) +def load_dns_entries(specific_upload_id: str = None) -> List[Dict]: + try: + entries = dns_records_table.all() + + # If a specific upload ID is provided, only show records from that upload + if specific_upload_id: + entries = [e for e in entries if e.get('upload_id') == specific_upload_id] + return entries + + # Sort by timestamp in descending order (newest first) + entries.sort(key=lambda x: x.get('timestamp', ''), reverse=True) + + # Create a dictionary to track unique entries (most recent only) + unique_entries = {} + + for entry in entries: + # Create a unique key based on domain, class, and type + unique_key = f"{entry.get('domain')}:{entry.get('record_class')}:{entry.get('record_type')}" + + # Only keep the most recent entry for each unique combination + if unique_key not in unique_entries: + # Mark as most recent entry + entry['is_latest'] = True + unique_entries[unique_key] = entry + + # Return the deduplicated list with only the most recent entries + return list(unique_entries.values()) + except Exception as e: + print(f"Error loading DNS records from database: {e}") + return [] + +# Get unique values for filter dropdowns +def get_unique_values(entries: List[Dict]) -> Dict[str, Set]: + unique_values = { + "record_type": set(), + "record_class": set(), + "tld": set(), + "sld": set() + } + + for entry in entries: + for key in unique_values.keys(): + if key in entry and entry[key]: + unique_values[key].add(entry[key]) + + # Convert sets to sorted lists + return {k: sorted(list(v)) for k, v in unique_values.items()} + +# Get all uploads +def get_uploads(): + uploads = uploads_table.all() + # Sort uploads by timestamp (newest first) + uploads.sort(key=lambda x: x.get('timestamp', ''), reverse=True) + return uploads + +# Delete an upload and its associated data +def delete_upload(upload_id): + try: + # Remove the upload from uploads table + Upload = TinyDBQuery() + uploads_table.remove(Upload.id == upload_id) + + # Remove associated domain and DNS records + Domain = TinyDBQuery() + domains_table.remove(Domain.upload_id == upload_id) + + DNSRecord = TinyDBQuery() + dns_records_table.remove(DNSRecord.upload_id == upload_id) + + return True + except Exception as e: + print(f"Error deleting upload {upload_id}: {e}") + return False + +# Routes +@app.get("/", response_class=HTMLResponse) +async def home(request: Request, upload_id: Optional[str] = None): + """Home page with upload form and SLD listing""" + domains = load_domains(upload_id) + uploads = get_uploads() + return templates.TemplateResponse( + "index.html", + { + "request": request, + "domains": domains, + "uploads": uploads + } + ) + +@app.get("/delete-upload/{upload_id}", response_class=RedirectResponse) +async def delete_upload_route(upload_id: str): + """Delete an upload and all associated records""" + success = delete_upload(upload_id) + if not success: + raise HTTPException(status_code=500, detail="Failed to delete upload") + + # Redirect back to home page + return RedirectResponse(url="/", status_code=303) + +@app.post("/upload", response_class=RedirectResponse) +async def upload_csv(request: Request, file: UploadFile = File(...), description: str = Form(None)): + """Handle file upload""" + try: + # Read file content + content = await file.read() + + # Ensure content is not empty + if not content or len(content) == 0: + raise ValueError("Uploaded file is empty") + + # Generate a unique ID for this upload with timestamp and a random suffix for extra uniqueness + now = datetime.datetime.now() + import random + random_suffix = ''.join(random.choices('abcdefghijklmnopqrstuvwxyz0123456789', k=6)) + upload_id = f"upload_{now.strftime('%Y%m%d_%H%M%S')}_{random_suffix}" + + print(f"Processing upload: ID={upload_id}, Filename={file.filename}, Content size={len(content)} bytes") + + # Process the CSV content + domains_count, records_count = await process_csv_upload(content, upload_id, description) + + print(f"Upload processed: {domains_count} domains and {records_count} DNS records inserted") + + if domains_count == 0 and records_count == 0: + print("WARNING: No records were inserted. Content may be invalid or empty.") + # Try to decode and print the first few lines for debugging + try: + preview = content.decode('utf-8')[:500] + print(f"File preview: {preview}") + except: + print("Could not decode file content for preview") + + # Store upload information with file hash to identify content changes + import hashlib + content_hash = hashlib.md5(content).hexdigest() + + upload_info = { + "id": upload_id, + "filename": file.filename, + "description": description, + "timestamp": datetime.datetime.now().isoformat(), + "domains_count": domains_count, + "records_count": records_count, + "content_hash": content_hash + } + uploads_table.insert(upload_info) + + # Redirect back to home page + return RedirectResponse(url="/", status_code=303) + except Exception as e: + print(f"Error in upload_csv: {e}") + return {"error": str(e)} + +@app.get("/dns-records", response_class=HTMLResponse) +async def dns_records( + request: Request, + upload_id: Optional[str] = None, + record_type: Optional[str] = None, + record_class: Optional[str] = None, + tld: Optional[str] = None, + sld: Optional[str] = None, + domain: Optional[str] = None +): + """DNS Records page with filtering""" + # Get all entries first, based on upload_id if provided + entries = load_dns_entries(upload_id) + + # Apply additional filters if provided + if record_type: + entries = [e for e in entries if e.get("record_type") == record_type] + if record_class: + entries = [e for e in entries if e.get("record_class") == record_class] + if tld: + entries = [e for e in entries if e.get("tld") == tld] + if sld: + entries = [e for e in entries if e.get("sld") == sld] + if domain: + entries = [e for e in entries if domain.lower() in e.get("domain", "").lower()] + + # Get unique values for filter dropdowns from all entries (not filtered) + all_entries = load_dns_entries(upload_id) + unique_values = get_unique_values(all_entries) + uploads = get_uploads() + + return templates.TemplateResponse( + "dns_records.html", + { + "request": request, + "entries": entries, + "unique_values": unique_values, + "uploads": uploads + } + ) + +# API Routes +@app.get("/api/uploads", response_model=List[Dict]) +async def get_all_uploads(): + """API endpoint that returns all uploads""" + return get_uploads() + +@app.get("/api/slds", response_model=List[Dict]) +async def get_slds(upload_id: Optional[str] = None): + """API endpoint that returns all SLDs with optional filter by upload_id""" + # The load_domains function now handles deduplication and upload_id filtering + domains = load_domains(upload_id) + return domains + +@app.get("/api/slds/{sld}", response_model=List[Dict]) +async def get_domains_by_sld(sld: str, upload_id: Optional[str] = None): + """API endpoint that returns domains for a specific SLD with optional filter by upload_id""" + # Get domains, already deduplicated and optionally filtered by upload_id + all_domains = load_domains(upload_id) + + # Filter by SLD + filtered = [item for item in all_domains if item["sld"].lower() == sld.lower()] + + if not filtered: + raise HTTPException(status_code=404, detail=f"No domains found with SLD: {sld}") + + return filtered + +@app.get("/api/dns", response_model=List[Dict]) +async def get_dns_entries( + record_type: Optional[str] = None, + record_class: Optional[str] = None, + tld: Optional[str] = None, + sld: Optional[str] = None, + domain: Optional[str] = None, + upload_id: Optional[str] = None +): + """API endpoint that returns filtered DNS entries""" + # Get entries - if upload_id is specified, only those entries are returned + entries = load_dns_entries(upload_id) + + # Apply additional filters if provided + if record_type: + entries = [e for e in entries if e.get("record_type") == record_type] + if record_class: + entries = [e for e in entries if e.get("record_class") == record_class] + if tld: + entries = [e for e in entries if e.get("tld") == tld] + if sld: + entries = [e for e in entries if e.get("sld") == sld] + if domain: + entries = [e for e in entries if domain.lower() in e.get("domain", "").lower()] + + return entries + +@app.get("/api/dns/types", response_model=Dict[str, List]) +async def get_unique_filter_values(upload_id: Optional[str] = None): + """API endpoint that returns unique values for filters""" + # Get entries - if upload_id is specified, only those entries are returned + entries = load_dns_entries(upload_id) + + return get_unique_values(entries) + +# Create templates directory and HTML file +Path("templates").mkdir(exist_ok=True) + +if __name__ == "__main__": + uvicorn.run("main:app", host="0.0.0.0", port=8000, reload=True) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7d501a4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +fastapi==0.104.1 +uvicorn==0.23.2 +python-multipart==0.0.6 +jinja2==3.1.2 +pandas==2.1.1 +tinydb==4.8.0 \ No newline at end of file diff --git a/run_container.sh b/run_container.sh new file mode 100755 index 0000000..020bfe3 --- /dev/null +++ b/run_container.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +# Function to check if a command exists +command_exists() { + command -v "$1" >/dev/null 2>&1 +} + +# Determine if we use podman or docker +if command_exists podman; then + CONTAINER_CMD="podman" + VOLUME_FLAG=":Z" + echo "Using Podman for container management." +elif command_exists docker; then + CONTAINER_CMD="docker" + VOLUME_FLAG="" + echo "Using Docker for container management." +else + echo "Error: Neither Podman nor Docker found. Please install one of them first." + exit 1 +fi + +# Stop and remove container if it exists +echo "Checking for existing container..." +if $CONTAINER_CMD ps -a --format '{{.Names}}' | grep -q "^fastapi-domains$"; then + echo "Stopping and removing existing fastapi-domains container..." + $CONTAINER_CMD stop fastapi-domains + $CONTAINER_CMD rm fastapi-domains +fi + +# Create volume if it doesn't exist +echo "Creating volume for persistent data storage..." +$CONTAINER_CMD volume create domain-data + +# Build the container image +echo "Building container image..." +$CONTAINER_CMD build -t fastapi-domains:latest . + +# Run the container +echo "Starting container..." +$CONTAINER_CMD run --name fastapi-domains \ + -p 8000:8000 \ + -v domain-data:/home/appuser/app/data${VOLUME_FLAG} \ + -e DB_DIR=/home/appuser/app/data \ + --security-opt no-new-privileges:true \ + --read-only \ + --tmpfs /tmp \ + --cap-drop ALL \ + --user 1000:1000 \ + -d fastapi-domains:latest + +# Check if container started successfully +if [ $? -eq 0 ]; then + echo "Container started successfully!" + echo "Application is available at: http://localhost:8000" + echo "" + echo "Container logs:" + $CONTAINER_CMD logs fastapi-domains +else + echo "Failed to start container." + exit 1 +fi \ No newline at end of file diff --git a/templates/dns_records.html b/templates/dns_records.html new file mode 100644 index 0000000..f28ec83 --- /dev/null +++ b/templates/dns_records.html @@ -0,0 +1,316 @@ + + + + DNS Records Viewer + + + + + +
+

DNS Entry Viewer

+ +
+ SLD View + DNS Records +
+ +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + Reset +
+
+ +

DNS Records {{ entries|length }}

+ + {% if entries %} + + + + + + + + + + + + + {% for entry in entries %} + + + + + + + + + {% endfor %} + +
DomainTTLClassTypeDataLast Updated
+ {% if entry.get('service') %} + {{ entry.service }} + {% endif %} + {% if entry.get('subdomain') %} + {{ entry.subdomain }}. + {% endif %} + {{ entry.sld }}. + {{ entry.tld }} + {{ entry.ttl }}{{ entry.record_class }}{{ entry.record_type }} + {{ entry.record_data }} + {% if entry.record_type == "SOA" or entry.record_data|length > 50 %} + {{ entry.record_data }} + {% endif %} + + + + {{ entry.timestamp.replace('T', ' ').split('.')[0] }} +
+ {% else %} +

No DNS entries found. Please upload a CSV file to get started.

+ {% endif %} + +
+

API Endpoints

+

Get all DNS entries: /api/dns

+

Get filtered DNS entries: /api/dns?record_type=A&tld=de

+

Filter by upload: /api/dns?upload_id={upload_id}

+

Get unique filter values: /api/dns/types

+
+
+ + + + + \ No newline at end of file diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..6b4f35f --- /dev/null +++ b/templates/index.html @@ -0,0 +1,259 @@ + + + + Domain Viewer + + + + + +
+

Domain Management System

+ +
+ SLD View + DNS Records +
+ +
+

Upload CSV File

+
+
+ + +
+
+ + +
+ +
+
+ + {% if uploads %} +
+

Upload History

+ + + + + + + + + + + + + + {% for upload in uploads %} + + + + + + + + + + {% endfor %} + +
DateFilenameDescriptionDomainsDNS RecordsViewDelete
{{ upload.timestamp.replace('T', ' ').split('.')[0] }}{{ upload.filename }}{{ upload.description or "N/A" }}{{ upload.domains_count }}{{ upload.records_count }} + View + + Delete +
+
+ {% endif %} + +
+

Domain List

+
+ + +
+
+ +
+

API Endpoints

+

Get all uploads: /api/uploads

+

Get all domains: /api/slds

+

Get domains by SLD: /api/slds/{sld}

+

Filter by upload: /api/slds?upload_id={upload_id}

+
+ + {% if domains %} +

Found {{ domains|length }} domains{% if request.query_params.get('upload_id') %} in this upload{% endif %}.

+ + + + + + + + + + + + {% for item in domains %} + + + + + + + + {% endfor %} + +
SLDTLDSubdomainFull DomainUpload Date
{{ item.sld }}{{ item.tld }}{{ item.get('subdomain', 'N/A') }}{{ item.full_domain }}{{ item.timestamp.replace('T', ' ').split('.')[0] if item.get('timestamp') else 'N/A' }}
+ {% else %} +

No domains found. Please upload a CSV file to get started.

+ {% endif %} +
+ + \ No newline at end of file