CaffeineFueled/domainapi
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

CHANGE SLD+TLD to FQDN, CHANGE dedublication to show all entries

Browse source
This commit is contained in:
CaffeineFueled 2025-04-09 19:38:03 +02:00
parent 6ce10f673e
commit fc72f6f51c
3 changed files with 83 additions and 142 deletions
Download patch file Download diff file Expand all files Collapse all files

136
main.py
View file

@ -32,29 +32,11 @@ def process_domain_entry(domain_entry):
if domain_entry.endswith('.'): if domain_entry.endswith('.'):
domain_entry = domain_entry[:-1] domain_entry = domain_entry[:-1]
# Parse domain components if domain_entry:
parts = domain_entry.split('.') # Store only the full domain name without splitting
if len(parts) > 1: domain_info = {
# For domain.tld format "full_domain": domain_entry
if len(parts) == 2: }
sld = parts[0] # Second Level Domain
tld = parts[1] # Top Level Domain
domain_info = {
"sld": sld,
"tld": tld,
"full_domain": domain_entry
}
# For subdomain.domain.tld format
else:
sld = parts[-2] # Second Level Domain
tld = parts[-1] # Top Level Domain
subdomain = '.'.join(parts[:-2]) # Subdomains
domain_info = {
"sld": sld,
"tld": tld,
"full_domain": domain_entry,
"subdomain": subdomain
}
return domain_info return domain_info
return None return None
@ -90,7 +72,7 @@ async def process_csv_upload(file_content, upload_id, description=None):
domain_info = process_domain_entry(domain_entry) domain_info = process_domain_entry(domain_entry)
if domain_info: if domain_info:
# Create a unique key to avoid duplicates within this upload # Create a unique key to avoid duplicates within this upload
unique_key = f"{domain_info['sld']}.{domain_info['tld']}" unique_key = domain_info['full_domain']
if unique_key not in unique_domains: if unique_key not in unique_domains:
unique_domains.add(unique_key) unique_domains.add(unique_key)
@ -127,22 +109,9 @@ async def process_csv_upload(file_content, upload_id, description=None):
"timestamp": timestamp "timestamp": timestamp
} }
# Add domain components # Add special handling for service records
if len(domain_parts) > 1: if len(domain_parts) > 0 and domain_parts[0].startswith('_'): # Service records like _dmarc
if domain_parts[0].startswith('_'): # Service records like _dmarc entry["service"] = domain_parts[0]
entry["service"] = domain_parts[0]
# Adjust domain parts
domain_parts = domain_parts[1:]
# For domain.tld format
if len(domain_parts) == 2:
entry["sld"] = domain_parts[0] # Second Level Domain
entry["tld"] = domain_parts[1] # Top Level Domain
# For subdomain.domain.tld format
elif len(domain_parts) > 2:
entry["sld"] = domain_parts[-2] # Second Level Domain
entry["tld"] = domain_parts[-1] # Top Level Domain
entry["subdomain"] = '.'.join(domain_parts[:-2]) # Subdomains
dns_records_to_insert.append(entry) dns_records_to_insert.append(entry)
@ -201,34 +170,38 @@ def load_domains(specific_upload_id: str = None) -> List[Dict]:
print(f"Error loading domains from database: {e}") print(f"Error loading domains from database: {e}")
return [] return []
# Load DNS entries from database - deduplicated by domain, class, and type (no history) # Load DNS entries from database - with optional deduplication
def load_dns_entries(specific_upload_id: str = None) -> List[Dict]: def load_dns_entries(specific_upload_id: str = None, deduplicate: bool = False) -> List[Dict]:
try: try:
entries = dns_records_table.all() entries = dns_records_table.all()
# If a specific upload ID is provided, only show records from that upload # If a specific upload ID is provided, only show records from that upload
if specific_upload_id: if specific_upload_id:
entries = [e for e in entries if e.get('upload_id') == specific_upload_id] entries = [e for e in entries if e.get('upload_id') == specific_upload_id]
return entries
# Sort by timestamp in descending order (newest first) # Sort by timestamp in descending order (newest first)
entries.sort(key=lambda x: x.get('timestamp', ''), reverse=True) entries.sort(key=lambda x: x.get('timestamp', ''), reverse=True)
# Create a dictionary to track unique entries (most recent only) # If deduplication is requested, only keep the most recent entry for each unique combination
unique_entries = {} if deduplicate:
# Create a dictionary to track unique entries (most recent only)
unique_entries = {}
for entry in entries: for entry in entries:
# Create a unique key based on domain, class, and type # Create a unique key based on domain, class, type, TTL, and data
unique_key = f"{entry.get('domain')}:{entry.get('record_class')}:{entry.get('record_type')}" unique_key = f"{entry.get('domain')}:{entry.get('record_class')}:{entry.get('record_type')}:{entry.get('ttl')}:{entry.get('record_data')}"
# Only keep the most recent entry for each unique combination # Only keep the most recent entry for each unique combination
if unique_key not in unique_entries: if unique_key not in unique_entries:
# Mark as most recent entry # Mark as most recent entry
entry['is_latest'] = True entry['is_latest'] = True
unique_entries[unique_key] = entry unique_entries[unique_key] = entry
# Return the deduplicated list with only the most recent entries # Return the deduplicated list with only the most recent entries
return list(unique_entries.values()) return list(unique_entries.values())
else:
# No deduplication - return all entries
return entries
except Exception as e: except Exception as e:
print(f"Error loading DNS records from database: {e}") print(f"Error loading DNS records from database: {e}")
return [] return []
@ -237,9 +210,7 @@ def load_dns_entries(specific_upload_id: str = None) -> List[Dict]:
def get_unique_values(entries: List[Dict]) -> Dict[str, Set]: def get_unique_values(entries: List[Dict]) -> Dict[str, Set]:
unique_values = { unique_values = {
"record_type": set(), "record_type": set(),
"record_class": set(), "record_class": set()
"tld": set(),
"sld": set()
} }
for entry in entries: for entry in entries:
@ -361,28 +332,23 @@ async def dns_records(
upload_id: Optional[str] = None, upload_id: Optional[str] = None,
record_type: Optional[str] = None, record_type: Optional[str] = None,
record_class: Optional[str] = None, record_class: Optional[str] = None,
tld: Optional[str] = None, domain: Optional[str] = None,
sld: Optional[str] = None, deduplicate: Optional[bool] = True # Default to showing only unique latest entries
domain: Optional[str] = None
): ):
"""DNS Records page with filtering""" """DNS Records page with filtering"""
# Get all entries first, based on upload_id if provided # Get all entries first, based on upload_id if provided, with deduplication option
entries = load_dns_entries(upload_id) entries = load_dns_entries(upload_id, deduplicate)
# Apply additional filters if provided # Apply additional filters if provided
if record_type: if record_type:
entries = [e for e in entries if e.get("record_type") == record_type] entries = [e for e in entries if e.get("record_type") == record_type]
if record_class: if record_class:
entries = [e for e in entries if e.get("record_class") == record_class] entries = [e for e in entries if e.get("record_class") == record_class]
if tld:
entries = [e for e in entries if e.get("tld") == tld]
if sld:
entries = [e for e in entries if e.get("sld") == sld]
if domain: if domain:
entries = [e for e in entries if domain.lower() in e.get("domain", "").lower()] entries = [e for e in entries if domain.lower() in e.get("domain", "").lower()]
# Get unique values for filter dropdowns from all entries (not filtered) # Get unique values for filter dropdowns from all entries (not filtered)
all_entries = load_dns_entries(upload_id) all_entries = load_dns_entries(upload_id, deduplicate=False)
unique_values = get_unique_values(all_entries) unique_values = get_unique_values(all_entries)
uploads = get_uploads() uploads = get_uploads()
@ -392,7 +358,8 @@ async def dns_records(
"request": request, "request": request,
"entries": entries, "entries": entries,
"unique_values": unique_values, "unique_values": unique_values,
"uploads": uploads "uploads": uploads,
"deduplicate": deduplicate
} }
) )
@ -402,24 +369,24 @@ async def get_all_uploads():
"""API endpoint that returns all uploads""" """API endpoint that returns all uploads"""
return get_uploads() return get_uploads()
@app.get("/api/slds", response_model=List[Dict]) @app.get("/api/domains", response_model=List[Dict])
async def get_slds(upload_id: Optional[str] = None): async def get_domains(upload_id: Optional[str] = None):
"""API endpoint that returns all SLDs with optional filter by upload_id""" """API endpoint that returns all domains with optional filter by upload_id"""
# The load_domains function now handles deduplication and upload_id filtering # The load_domains function now handles deduplication and upload_id filtering
domains = load_domains(upload_id) domains = load_domains(upload_id)
return domains return domains
@app.get("/api/slds/{sld}", response_model=List[Dict]) @app.get("/api/domains/{domain}", response_model=List[Dict])
async def get_domains_by_sld(sld: str, upload_id: Optional[str] = None): async def get_domains_by_name(domain: str, upload_id: Optional[str] = None):
"""API endpoint that returns domains for a specific SLD with optional filter by upload_id""" """API endpoint that returns domains matching a specific domain name with optional filter by upload_id"""
# Get domains, already deduplicated and optionally filtered by upload_id # Get domains, already deduplicated and optionally filtered by upload_id
all_domains = load_domains(upload_id) all_domains = load_domains(upload_id)
# Filter by SLD # Filter by domain name
filtered = [item for item in all_domains if item["sld"].lower() == sld.lower()] filtered = [item for item in all_domains if domain.lower() in item["full_domain"].lower()]
if not filtered: if not filtered:
raise HTTPException(status_code=404, detail=f"No domains found with SLD: {sld}") raise HTTPException(status_code=404, detail=f"No domains found matching: {domain}")
return filtered return filtered
@ -427,24 +394,19 @@ async def get_domains_by_sld(sld: str, upload_id: Optional[str] = None):
async def get_dns_entries( async def get_dns_entries(
record_type: Optional[str] = None, record_type: Optional[str] = None,
record_class: Optional[str] = None, record_class: Optional[str] = None,
tld: Optional[str] = None,
sld: Optional[str] = None,
domain: Optional[str] = None, domain: Optional[str] = None,
upload_id: Optional[str] = None upload_id: Optional[str] = None,
deduplicate: Optional[bool] = True
): ):
"""API endpoint that returns filtered DNS entries""" """API endpoint that returns filtered DNS entries with optional deduplication"""
# Get entries - if upload_id is specified, only those entries are returned # Get entries - if upload_id is specified, only those entries are returned
entries = load_dns_entries(upload_id) entries = load_dns_entries(upload_id, deduplicate)
# Apply additional filters if provided # Apply additional filters if provided
if record_type: if record_type:
entries = [e for e in entries if e.get("record_type") == record_type] entries = [e for e in entries if e.get("record_type") == record_type]
if record_class: if record_class:
entries = [e for e in entries if e.get("record_class") == record_class] entries = [e for e in entries if e.get("record_class") == record_class]
if tld:
entries = [e for e in entries if e.get("tld") == tld]
if sld:
entries = [e for e in entries if e.get("sld") == sld]
if domain: if domain:
entries = [e for e in entries if domain.lower() in e.get("domain", "").lower()] entries = [e for e in entries if domain.lower() in e.get("domain", "").lower()]

57
templates/dns_records.html
View file

@ -115,14 +115,10 @@
background-color: #e0e0e0; background-color: #e0e0e0;
color: #333; color: #333;
} }
.sld-badge { .domain-badge {
background-color: #d1e7dd; background-color: #d1e7dd;
color: #0f5132; color: #0f5132;
} }
.tld-badge {
background-color: #cfe2ff;
color: #0a58ca;
}
.service-badge { .service-badge {
background-color: #fff3cd; background-color: #fff3cd;
color: #664d03; color: #664d03;
@ -149,6 +145,13 @@
margin-left: 10px; margin-left: 10px;
font-weight: normal; font-weight: normal;
} }
.dedup-note {
display: block;
font-size: 0.7em;
color: #666;
font-weight: normal;
margin-top: 5px;
}
.tooltip { .tooltip {
position: relative; position: relative;
display: inline-block; display: inline-block;
@ -185,10 +188,10 @@
</head> </head>
<body> <body>
<div class="container"> <div class="container">
<h1>DNS Entry Viewer</h1> <h1>DNS Records Viewer {% if deduplicate %}(Deduplicated){% else %}(All Records){% endif %}</h1>
<div class="nav"> <div class="nav">
<a href="/" class="nav-link">SLD View</a> <a href="/" class="nav-link">Domains</a>
<a href="/dns-records" class="nav-link">DNS Records</a> <a href="/dns-records" class="nav-link">DNS Records</a>
</div> </div>
@ -222,35 +225,28 @@
{% endfor %} {% endfor %}
</select> </select>
</div> </div>
<div class="filter-group">
<label for="tld">TLD:</label>
<select id="tld" name="tld">
<option value="">All TLDs</option>
{% for tld in unique_values.tld %}
<option value="{{ tld }}" {% if request.query_params.get('tld') == tld %}selected{% endif %}>{{ tld }}</option>
{% endfor %}
</select>
</div>
<div class="filter-group">
<label for="sld">SLD:</label>
<select id="sld" name="sld">
<option value="">All SLDs</option>
{% for sld in unique_values.sld %}
<option value="{{ sld }}" {% if request.query_params.get('sld') == sld %}selected{% endif %}>{{ sld }}</option>
{% endfor %}
</select>
</div>
<div class="filter-group"> <div class="filter-group">
<label for="domain-search">Domain Search:</label> <label for="domain-search">Domain Search:</label>
<input type="text" id="domain-search" name="domain" placeholder="Enter domain name..." value="{{ request.query_params.get('domain', '') }}"> <input type="text" id="domain-search" name="domain" placeholder="Enter domain name..." value="{{ request.query_params.get('domain', '') }}">
</div> </div>
<div class="filter-group">
<label for="deduplicate">Deduplicate Records:</label>
<select id="deduplicate" name="deduplicate" class="filter-select">
<option value="true" {% if request.query_params.get('deduplicate', 'true') == 'true' %}selected{% endif %}>Yes (Deduplicate Identical Records)</option>
<option value="false" {% if request.query_params.get('deduplicate') == 'false' %}selected{% endif %}>No (Show All)</option>
</select>
</div>
<div class="filter-buttons"> <div class="filter-buttons">
<button type="submit">Apply Filters</button> <button type="submit">Apply Filters</button>
<a href="/dns-records" class="reset-button" style="padding: 8px 16px; background-color: #f44336; color: white; text-decoration: none; border-radius: 4px; font-weight: bold; display: inline-block;">Reset</a> <a href="/dns-records" class="reset-button" style="padding: 8px 16px; background-color: #f44336; color: white; text-decoration: none; border-radius: 4px; font-weight: bold; display: inline-block;">Reset</a>
</div> </div>
</form> </form>
<h2>DNS Records <span class="count-badge">{{ entries|length }}</span></h2> <h2>DNS Records <span class="count-badge">{{ entries|length }}</span>
{% if deduplicate %}
<small class="dedup-note">(Showing most recent entries for each Domain+Class+Type+TTL+Data combination)</small>
{% endif %}
</h2>
{% if entries %} {% if entries %}
<table id="dns-table"> <table id="dns-table">
@ -271,11 +267,7 @@
{% if entry.get('service') %} {% if entry.get('service') %}
<span class="badge service-badge">{{ entry.service }}</span> <span class="badge service-badge">{{ entry.service }}</span>
{% endif %} {% endif %}
{% if entry.get('subdomain') %} <span class="badge domain-badge">{{ entry.domain }}</span>
{{ entry.subdomain }}.
{% endif %}
<span class="badge sld-badge">{{ entry.sld }}</span>.
<span class="badge tld-badge">{{ entry.tld }}</span>
</td> </td>
<td>{{ entry.ttl }}</td> <td>{{ entry.ttl }}</td>
<td>{{ entry.record_class }}</td> <td>{{ entry.record_class }}</td>
@ -302,8 +294,9 @@
<div class="api-section"> <div class="api-section">
<h2>API Endpoints</h2> <h2>API Endpoints</h2>
<p>Get all DNS entries: <code>/api/dns</code></p> <p>Get all DNS entries: <code>/api/dns</code></p>
<p>Get filtered DNS entries: <code>/api/dns?record_type=A&tld=de</code></p> <p>Get filtered DNS entries: <code>/api/dns?record_type=A&domain=example.com</code></p>
<p>Filter by upload: <code>/api/dns?upload_id={upload_id}</code></p> <p>Filter by upload: <code>/api/dns?upload_id={upload_id}</code></p>
<p>Show all records (no deduplication): <code>/api/dns?deduplicate=false</code></p>
<p>Get unique filter values: <code>/api/dns/types</code></p> <p>Get unique filter values: <code>/api/dns/types</code></p>
</div> </div>
</div> </div>

28
templates/index.html
View file

@ -53,7 +53,7 @@
tr:hover { tr:hover {
background-color: #f5f5f5; background-color: #f5f5f5;
} }
.sld-badge { .domain-badge {
display: inline-block; display: inline-block;
padding: 3px 7px; padding: 3px 7px;
background-color: #d1e7dd; background-color: #d1e7dd;
@ -61,14 +61,6 @@
font-size: 0.9em; font-size: 0.9em;
color: #0f5132; color: #0f5132;
} }
.tld-badge {
display: inline-block;
padding: 3px 7px;
background-color: #cfe2ff;
border-radius: 4px;
font-size: 0.9em;
color: #0a58ca;
}
.api-section { .api-section {
margin-top: 30px; margin-top: 30px;
padding: 15px; padding: 15px;
@ -149,7 +141,7 @@
<h1>Domain Management System</h1> <h1>Domain Management System</h1>
<div class="nav"> <div class="nav">
<a href="/" class="nav-link">SLD View</a> <a href="/" class="nav-link">Domains</a>
<a href="/dns-records" class="nav-link">DNS Records</a> <a href="/dns-records" class="nav-link">DNS Records</a>
</div> </div>
@ -222,9 +214,9 @@
<div class="api-section"> <div class="api-section">
<h3>API Endpoints</h3> <h3>API Endpoints</h3>
<p>Get all uploads: <code>/api/uploads</code></p> <p>Get all uploads: <code>/api/uploads</code></p>
<p>Get all domains: <code>/api/slds</code></p> <p>Get all domains: <code>/api/domains</code></p>
<p>Get domains by SLD: <code>/api/slds/{sld}</code></p> <p>Get domains by name: <code>/api/domains/{domain}</code></p>
<p>Filter by upload: <code>/api/slds?upload_id={upload_id}</code></p> <p>Filter by upload: <code>/api/domains?upload_id={upload_id}</code></p>
</div> </div>
{% if domains %} {% if domains %}
@ -232,20 +224,14 @@
<table> <table>
<thead> <thead>
<tr> <tr>
<th>SLD</th> <th>Domain</th>
<th>TLD</th>
<th>Subdomain</th>
<th>Full Domain</th>
<th>Upload Date</th> <th>Upload Date</th>
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
{% for item in domains %} {% for item in domains %}
<tr> <tr>
<td><span class="sld-badge">{{ item.sld }}</span></td> <td><span class="domain-badge">{{ item.full_domain }}</span></td>
<td><span class="tld-badge">{{ item.tld }}</span></td>
<td>{{ item.get('subdomain', 'N/A') }}</td>
<td>{{ item.full_domain }}</td>
<td>{{ item.timestamp.replace('T', ' ').split('.')[0] if item.get('timestamp') else 'N/A' }}</td> <td>{{ item.timestamp.replace('T', ' ').split('.')[0] if item.get('timestamp') else 'N/A' }}</td>
</tr> </tr>
{% endfor %} {% endfor %}