CaffeineFueled/002-Remote-Logging-Funnel
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

pre api key

Browse source
This commit is contained in:
CaffeineFueled 2025-06-06 11:10:04 +02:00
commit 9569f90aba
8 changed files with 900 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

62
.dockerignore Normal file
View file

@ -0,0 +1,62 @@
# Python
__pycache__/
*.py[cod]
*$py.class
*.so
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
pip-wheel-metadata/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
# Virtual Environment
venv/
env/
ENV/
# IDE
.vscode/
.idea/
*.swp
*.swo
*~
# Git
.git/
.gitignore
# Documentation
*.md
examples.md
# Container files
Dockerfile
.dockerignore
run_container.sh
# OS
.DS_Store
Thumbs.db
# Logs
*.log
# Temporary files
*.tmp
*.temp
# Runtime data (exclude existing input data from image)
input/

153
.gitignore vendored Normal file
View file

@ -0,0 +1,153 @@
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
pip-wheel-metadata/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
.python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# IDE
.vscode/
.idea/
*.swp
*.swo
*~
# OS
.DS_Store
.DS_Store?
._*
.Spotlight-V100
.Trashes
ehthumbs.db
Thumbs.db
# Application specific
input/
*.tmp
*.temp
# Container runtime
.container_id

39
Dockerfile Normal file
View file

@ -0,0 +1,39 @@
FROM python:3.11-slim
# Set working directory
WORKDIR /app
# Create non-root user for security
RUN groupadd -r appuser && useradd -r -g appuser appuser
# Install system dependencies
RUN apt-get update && apt-get install -y \
--no-install-recommends \
&& rm -rf /var/lib/apt/lists/*
# Copy requirements first for better caching
COPY requirements.txt .
# Install Python dependencies
RUN pip install --no-cache-dir --upgrade pip && \
pip install --no-cache-dir -r requirements.txt
# Copy application code
COPY main.py .
# Create input directory and set permissions
RUN mkdir -p /app/input && \
chown -R appuser:appuser /app
# Switch to non-root user
USER appuser
# Expose port
EXPOSE 8000
# Health check
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
CMD python -c "import requests; requests.get('http://localhost:8000/health')" || exit 1
# Run the application
CMD ["python", "-m", "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]

6
README.md Normal file
View file

@ -0,0 +1,6 @@
# TODO
- API Auth Bearer Management
- Access Logs

191
examples.md Normal file
View file

@ -0,0 +1,191 @@
# Simple Data Collector API - Usage Examples
## Starting the Server
### Using Virtual Environment (Recommended)
```bash
# Create virtual environment (if not already created)
python -m venv venv
# Activate virtual environment
# On Linux/Mac:
source venv/bin/activate
# On Windows:
# venv\Scripts\activate
# Install dependencies
pip install -r requirements.txt
# Run the server
python main.py
```
The server will start on `http://localhost:8000`
## Simple Data Collection
### Basic Usage - curl (with Bearer Authentication)
```bash
# Send simple comma-separated data to run1 (requires input token)
curl -X POST http://localhost:8000/api/run1/ \
-H "Authorization: Bearer input_token_123" \
-d "host_a,is ok"
# Send data to different runs
curl -X POST http://localhost:8000/api/run1/ \
-H "Authorization: Bearer input_token_123" \
-d "host_b,is ok"
curl -X POST http://localhost:8000/api/run1/ \
-H "Authorization: Bearer input_token_123" \
-d "host_c,failed"
curl -X POST http://localhost:8000/api/run2/ \
-H "Authorization: Bearer input_token_123" \
-d "server_x,healthy"
```
### PowerShell Examples (with Bearer Authentication)
```powershell
# Send OK status from PowerShell (requires input token)
$hostname = $env:COMPUTERNAME
$status = "is ok"
$headers = @{ "Authorization" = "Bearer input_token_123" }
Invoke-RestMethod -Uri "http://your-server:8000/api/run1/" -Method Post -Body "$hostname,$status" -Headers $headers
# Send error status
$hostname = $env:COMPUTERNAME
$status = "failed - disk full"
$headers = @{ "Authorization" = "Bearer input_token_123" }
Invoke-RestMethod -Uri "http://your-server:8000/api/run1/" -Method Post -Body "$hostname,$status" -Headers $headers
```
### wget Examples (with Bearer Authentication)
```bash
# Send data using wget (requires input token)
wget --post-data="host_d,is ok" \
--header="Authorization: Bearer input_token_123" \
http://localhost:8000/api/run1/
wget --post-data="host_e,network error" \
--header="Authorization: Bearer input_token_123" \
http://localhost:8000/api/run1/
```
### Multiple Endpoints (with Bearer Authentication)
```bash
# Different runs create separate directories and files (requires input token)
curl -X POST http://localhost:8000/api/daily-check/ \
-H "Authorization: Bearer input_token_123" \
-d "server1,ok"
curl -X POST http://localhost:8000/api/backup-status/ \
-H "Authorization: Bearer input_token_123" \
-d "server1,completed"
curl -X POST http://localhost:8000/api/security-scan/ \
-H "Authorization: Bearer input_token_123" \
-d "server1,clean"
```
## Retrieving Results
### View all results for a run (with Bearer Authentication)
```bash
# Get all results for run1 (requires read token)
curl -H "Authorization: Bearer read_token_456" \
http://localhost:8000/results/run1/
# Get results for other runs
curl -H "Authorization: Bearer read_token_456" \
http://localhost:8000/results/daily-check/
curl -H "Authorization: Bearer read_token_456" \
http://localhost:8000/results/backup-status/
```
### PowerShell - Get results (with Bearer Authentication)
```powershell
# Get results from PowerShell (requires read token)
$headers = @{ "Authorization" = "Bearer read_token_456" }
Invoke-RestMethod -Uri "http://your-server:8000/results/run1/" -Method Get -Headers $headers
```
## File Output
Data is saved to text files with timestamps:
### Directory Structure
```
input/
├── run1/
│ └── results.txt
├── daily-check/
│ └── results.txt
└── backup-status/
└── results.txt
```
### Example File Content (input/run1/results.txt)
```
2025-01-06 10:30:15 - host_a,is ok
2025-01-06 10:30:22 - host_b,is ok
2025-01-06 10:30:45 - host_c,failed
2025-01-06 10:31:02 - host_d,is ok
```
## Real-World Intune PowerShell Example (with Bearer Authentication)
```powershell
# Place this in your Intune PowerShell script
try {
# Your script logic here
Write-Host "Script executed successfully"
# Send success status (requires input token)
$hostname = $env:COMPUTERNAME
$headers = @{ "Authorization" = "Bearer input_token_123" }
Invoke-RestMethod -Uri "http://your-api-server:8000/api/intune-deployment/" -Method Post -Body "$hostname,success" -Headers $headers
} catch {
# Send failure status with error
$hostname = $env:COMPUTERNAME
$error = $_.Exception.Message
$headers = @{ "Authorization" = "Bearer input_token_123" }
Invoke-RestMethod -Uri "http://your-api-server:8000/api/intune-deployment/" -Method Post -Body "$hostname,failed - $error" -Headers $headers
}
```
## Health Check
```bash
curl http://localhost:8000/health
```
## API Documentation
Visit `http://localhost:8000/docs` for interactive API documentation.
## Authentication Tokens
### Required Tokens
- **Input Token** (for POST operations): `input_token_123`
- **Read Token** (for GET operations): `read_token_456`
### Security Notes
- POST endpoints require the `input_token_123` bearer token
- GET endpoints require the `read_token_456` bearer token
- Change these tokens in `main.py` lines 22-23 for production use
## Key Features
- **Simple Input**: Just send plain text data (e.g., "host_a,is ok")
- **Bearer Authentication**: Separate tokens for input and read operations
- **Automatic Directories**: Each run name creates its own directory
- **Timestamped Entries**: Every entry gets a timestamp
- **Multiple Runs**: Support multiple concurrent data collection runs
- **Easy Retrieval**: Get all results via REST API
- **Plain Text Output**: Results saved as simple text files

245
main.py Normal file
View file

@ -0,0 +1,245 @@
# Simple FastAPI Data Collector
# Purpose: Accept simple comma-separated input and save to text files
# Usage: curl -X POST http://localhost:8000/api/run1/ -d "host_a,is ok"
from fastapi import FastAPI, HTTPException, Request, Depends
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
from datetime import datetime
from pathlib import Path
import re
import bleach
# Initialize FastAPI application
app = FastAPI(
title="Simple Data Collector",
version="1.0.0",
description="Simple API for collecting comma-separated data"
)
# Define the base directory for input files
INPUT_DIR = Path("input")
INPUT_DIR.mkdir(exist_ok=True) # Create directory if it doesn't exist
# Authentication tokens
INPUT_TOKEN = "input_token_123" # Token for POST endpoints
READ_TOKEN = "read_token_456" # Token for GET endpoints
# Security schemes
input_security = HTTPBearer()
read_security = HTTPBearer()
def verify_input_token(credentials: HTTPAuthorizationCredentials = Depends(input_security)):
"""Verify bearer token for input operations"""
if credentials.credentials != INPUT_TOKEN:
raise HTTPException(
status_code=401,
detail="Invalid authentication token for input operations"
)
return credentials
def verify_read_token(credentials: HTTPAuthorizationCredentials = Depends(read_security)):
"""Verify bearer token for read operations"""
if credentials.credentials != READ_TOKEN:
raise HTTPException(
status_code=401,
detail="Invalid authentication token for read operations"
)
return credentials
def sanitize_input(data: str) -> str:
"""
Sanitize input data using bleach to prevent malicious content.
Args:
data (str): Raw input data
Returns:
str: Sanitized data safe for storage
"""
if not data:
return ""
# Clean with bleach - strip all HTML tags and dangerous content
sanitized = bleach.clean(data, tags=[], attributes={}, strip=True)
# Remove control characters except newlines and tabs
sanitized = re.sub(r'[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]', '', sanitized)
# Normalize whitespace
sanitized = re.sub(r'\s+', ' ', sanitized).strip()
# Limit length to prevent DoS
max_length = 1000
if len(sanitized) > max_length:
sanitized = sanitized[:max_length] + "... [truncated]"
return sanitized
def sanitize_run_name(run_name: str) -> str:
"""
Sanitize run name to ensure it's safe for filesystem operations.
Args:
run_name (str): Raw run name
Returns:
str: Sanitized run name safe for filesystem
"""
if not run_name:
raise HTTPException(status_code=400, detail="Run name cannot be empty")
# Use bleach to clean any potential HTML/script content
sanitized = bleach.clean(run_name, tags=[], attributes={}, strip=True)
# Remove dangerous filesystem characters
sanitized = re.sub(r'[^\w\-.]', '_', sanitized)
# Remove leading/trailing dots and dashes
sanitized = sanitized.strip('.-')
# Prevent reserved names
reserved_names = ['con', 'prn', 'aux', 'nul', 'com1', 'com2', 'com3', 'com4',
'com5', 'com6', 'com7', 'com8', 'com9', 'lpt1', 'lpt2',
'lpt3', 'lpt4', 'lpt5', 'lpt6', 'lpt7', 'lpt8', 'lpt9']
if sanitized.lower() in reserved_names:
sanitized = f"safe_{sanitized}"
# Limit length
if len(sanitized) > 50:
sanitized = sanitized[:50]
if not sanitized:
raise HTTPException(status_code=400, detail="Invalid run name after sanitization")
return sanitized
def save_to_file(run_name: str, data: str):
"""
Save data as a single line to input/{run_name}/results.txt
Args:
run_name (str): Name of the run (creates subdirectory)
data (str): The data to save as a single line
"""
# Create run-specific directory
run_dir = INPUT_DIR / run_name
run_dir.mkdir(exist_ok=True)
# Define the results file path
results_file = run_dir / "results.txt"
# Get current timestamp
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
# Append the data with timestamp to the file
with open(results_file, "a", encoding="utf-8") as f:
f.write(f"{timestamp} - {data.strip()}\n")
@app.get("/")
async def root():
"""Root endpoint providing API information"""
return {
"message": "Simple Data Collector API",
"version": "1.0.0",
"usage": "POST /api/{run_name}/ with data in body",
"example": "curl -X POST http://localhost:8000/api/run1/ -d 'host_a,is ok'",
"output": "Data saved to input/{run_name}/results.txt"
}
@app.post("/api/{run_name}/")
async def collect_data(run_name: str, request: Request, token: HTTPAuthorizationCredentials = Depends(verify_input_token)):
"""
Collect simple data and save to text file.
Args:
run_name (str): Name of the run (creates input/{run_name}/ directory)
request: Raw request body containing the data
Returns:
dict: Confirmation message
Example:
curl -X POST http://localhost:8000/api/run1/ -d "host_a,is ok"
-> Saves to input/run1/results.txt
"""
# Read the raw body data
body = await request.body()
data = body.decode("utf-8")
# Validate that we have some data
if not data.strip():
raise HTTPException(status_code=400, detail="No data provided")
# Sanitize input data and run name
sanitized_data = sanitize_input(data)
sanitized_run_name = sanitize_run_name(run_name)
# Save to file
save_to_file(sanitized_run_name, sanitized_data)
# Return confirmation
return {
"message": "Data saved successfully",
"run_name": sanitized_run_name,
"data": sanitized_data,
"saved_to": f"input/{sanitized_run_name}/results.txt",
"timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
}
@app.get("/results/{run_name}/")
async def get_results(run_name: str, token: HTTPAuthorizationCredentials = Depends(verify_read_token)):
"""
Retrieve all results for a specific run.
Args:
run_name (str): Name of the run to get results for
Returns:
dict: All lines from the results file
"""
# Sanitize run name for safe filesystem access
sanitized_run_name = sanitize_run_name(run_name)
results_file = INPUT_DIR / sanitized_run_name / "results.txt"
if not results_file.exists():
raise HTTPException(
status_code=404,
detail=f"No results found for run '{sanitized_run_name}'"
)
# Read all lines from the file
with open(results_file, "r", encoding="utf-8") as f:
lines = f.readlines()
return {
"run_name": sanitized_run_name,
"total_entries": len(lines),
"results": [line.strip() for line in lines]
}
@app.get("/health")
async def health_check():
"""Health check endpoint"""
return {
"status": "healthy",
"service": "Simple Data Collector",
"timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
}
# Application entry point
if __name__ == "__main__":
import uvicorn
print("Starting Simple Data Collector API...")
print("Server will be available at: http://localhost:8000")
print("API documentation at: http://localhost:8000/docs")
print("Example: curl -X POST http://localhost:8000/api/run1/ -d 'host_a,is ok'")
print("Results saved to: ./input/{run_name}/results.txt")
uvicorn.run(
app,
host="0.0.0.0",
port=8000,
log_level="info"
)

4
requirements.txt Normal file
View file

@ -0,0 +1,4 @@
fastapi==0.104.1
uvicorn[standard]==0.24.0
pydantic==2.5.0
bleach==6.1.0

200
run_container.sh Executable file
View file

@ -0,0 +1,200 @@
#!/bin/bash
# FastAPI Data Collector Container Runner
# Automated script to build and run the data collector in a container
# Supports both Podman and Docker with automatic detection
set -e
# Configuration
CONTAINER_NAME="fastapi-data-collector"
IMAGE_NAME="fastapi-data-collector:latest"
HOST_PORT="8000"
CONTAINER_PORT="8000"
DATA_DIR="./input"
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Function to print colored output
print_status() {
echo -e "${GREEN}[INFO]${NC} $1"
}
print_warning() {
echo -e "${YELLOW}[WARN]${NC} $1"
}
print_error() {
echo -e "${RED}[ERROR]${NC} $1"
}
print_header() {
echo -e "${BLUE}[HEADER]${NC} $1"
}
# Detect container runtime
detect_container_runtime() {
if command -v podman &> /dev/null; then
CONTAINER_CMD="podman"
print_status "Using Podman as container runtime"
elif command -v docker &> /dev/null; then
CONTAINER_CMD="docker"
print_status "Using Docker as container runtime"
else
print_error "Neither Podman nor Docker found. Please install one of them."
exit 1
fi
}
# Stop and remove existing container
cleanup_existing_container() {
print_status "Checking for existing container..."
if $CONTAINER_CMD ps -a --format "table {{.Names}}" | grep -q "^${CONTAINER_NAME}$"; then
print_warning "Stopping existing container: ${CONTAINER_NAME}"
$CONTAINER_CMD stop "${CONTAINER_NAME}" || true
print_warning "Removing existing container: ${CONTAINER_NAME}"
$CONTAINER_CMD rm "${CONTAINER_NAME}" || true
else
print_status "No existing container found"
fi
}
# Build container image
build_image() {
print_status "Building container image: ${IMAGE_NAME}"
if ! $CONTAINER_CMD build -t "${IMAGE_NAME}" .; then
print_error "Failed to build container image"
exit 1
fi
print_status "Container image built successfully"
}
# Create data directory
setup_data_directory() {
if [ ! -d "${DATA_DIR}" ]; then
print_status "Creating data directory: ${DATA_DIR}"
mkdir -p "${DATA_DIR}"
else
print_status "Data directory already exists: ${DATA_DIR}"
fi
}
# Run container
run_container() {
print_status "Starting container: ${CONTAINER_NAME}"
# Convert relative path to absolute path
#DATA_DIR_ABS=$(realpath "${DATA_DIR}")
DATA_DIR_ABS="/home/user/Documents/Dom.Decimal/30-Projects/30-Misc/30-00-ZeroOne/30-00-002-Work-FastAPI-REST-Endpoint/input"
$CONTAINER_CMD run -d \
--name "${CONTAINER_NAME}" \
--publish "${HOST_PORT}:${CONTAINER_PORT}" \
--volume "${DATA_DIR_ABS}:/app/input:Z" \
--restart unless-stopped \
--security-opt no-new-privileges \
--cap-drop ALL \
--cap-add=CHOWN \
--cap-add=SETGID \
--cap-add=SETUID \
--cap-add=DAC_OVERRIDE \
--read-only \
--tmpfs /tmp \
"${IMAGE_NAME}"
if [ $? -eq 0 ]; then
print_status "Container started successfully"
else
print_error "Failed to start container"
exit 1
fi
}
# Verify container is running
verify_container() {
print_status "Verifying container startup..."
# Wait a moment for the container to start
sleep 3
if $CONTAINER_CMD ps --format "table {{.Names}}" | grep -q "^${CONTAINER_NAME}$"; then
print_status "Container is running"
# Show container logs
print_status "Container logs:"
$CONTAINER_CMD logs "${CONTAINER_NAME}" | tail -10
# Test health endpoint
print_status "Testing health endpoint..."
sleep 2
if curl -s "http://localhost:${HOST_PORT}/health" > /dev/null; then
print_status "Health check passed"
else
print_warning "Health check failed - service may still be starting"
fi
else
print_error "Container is not running"
print_error "Container logs:"
$CONTAINER_CMD logs "${CONTAINER_NAME}"
exit 1
fi
}
# Show usage information
show_usage_info() {
print_header "=== FastAPI Data Collector Container Started ==="
echo
print_status "Service URL: http://localhost:${HOST_PORT}"
print_status "API Documentation: http://localhost:${HOST_PORT}/docs"
print_status "Health Check: http://localhost:${HOST_PORT}/health"
echo
print_status "Authentication Tokens:"
echo " - Input Token: input_token_123"
echo " - Read Token: read_token_456"
echo
print_status "Example Usage:"
echo " # Send data:"
echo ' curl -X POST http://localhost:8000/api/run1/ \'
echo ' -H "Authorization: Bearer input_token_123" \'
echo ' -d "host_a,is ok"'
echo
echo " # Read results:"
echo ' curl -H "Authorization: Bearer read_token_456" \'
echo ' http://localhost:8000/results/run1/'
echo
print_status "Data Directory: ${DATA_DIR_ABS}"
echo
print_status "Container Management:"
echo " - View logs: ${CONTAINER_CMD} logs ${CONTAINER_NAME}"
echo " - Stop container: ${CONTAINER_CMD} stop ${CONTAINER_NAME}"
echo " - Restart container: ${CONTAINER_CMD} restart ${CONTAINER_NAME}"
echo " - Remove container: ${CONTAINER_CMD} rm ${CONTAINER_NAME}"
echo
print_header "=============================================="
}
# Main execution
main() {
print_header "FastAPI Data Collector Container Setup"
echo
detect_container_runtime
cleanup_existing_container
build_image
setup_data_directory
run_container
verify_container
show_usage_info
}
# Run main function
main "$@"