405 lines
16 KiB
Python
405 lines
16 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
Генерирует HTML-дашборд по access-логам Nginx Proxy Manager.
|
||
- Сводка по доменам (2 суток) + по IP (топ-5 по каждому домену + остальные) с геолокацией.
|
||
- Лента запросов с пагинацией и фильтром по домену (клиентский JS).
|
||
Геолокация: ip-api.com с кэшем (локальная сеть для 10.x, 192.168.x, 127.x).
|
||
"""
|
||
import base64
|
||
import json
|
||
import re
|
||
import sys
|
||
import time
|
||
import urllib.request
|
||
from collections import defaultdict
|
||
from datetime import datetime, timedelta, timezone
|
||
from pathlib import Path
|
||
|
||
LOG_DIR = Path("/opt/docker/nginx-proxy/data/logs")
|
||
CACHE_FILE = Path("/opt/docker/log-dashboard/ip_cache.json")
|
||
FEED_MAX = 2_000 # макс. записей в ленте (встроено в HTML, ~300 KB)
|
||
GEO_MAX_NEW_PER_RUN = 40 # ip-api.com limit 45/min
|
||
API_URL = "http://ip-api.com/json/{ip}?fields=status,country,city,isp"
|
||
|
||
LINE_RE = re.compile(
|
||
r'\[(\d{2}/\w{3}/\d{4}:\d{2}:\d{2}:\d{2}\s+\+\d{4})\]\s+-\s+(\d+)\s+\d+\s+-\s+(\w+)\s+\w+\s+([^\s]+)\s+"([^"]*)"\s+\[Client\s+([^\]]+)\]'
|
||
)
|
||
|
||
|
||
def parse_date(s: str) -> datetime | None:
|
||
try:
|
||
return datetime.strptime(s.strip(), "%d/%b/%Y:%H:%M:%S %z")
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def parse_line(line: str) -> dict | None:
|
||
m = LINE_RE.search(line)
|
||
if not m:
|
||
return None
|
||
date_s, status, method, domain, path, client = m.groups()
|
||
dt = parse_date(date_s)
|
||
if not dt:
|
||
return None
|
||
return {
|
||
"time": dt,
|
||
"date_s": date_s,
|
||
"status": status,
|
||
"method": method,
|
||
"domain": domain,
|
||
"path": (path or "/")[:80],
|
||
"client": client.strip(),
|
||
}
|
||
|
||
|
||
def is_private_ip(ip: str) -> bool:
|
||
if not ip or ip == "-":
|
||
return True
|
||
parts = ip.split(".")
|
||
if len(parts) != 4:
|
||
return True
|
||
try:
|
||
a, b, c, d = (int(x) for x in parts)
|
||
if a == 10:
|
||
return True
|
||
if a == 172 and 16 <= b <= 31:
|
||
return True
|
||
if a == 192 and b == 168:
|
||
return True
|
||
if a == 127:
|
||
return True
|
||
except ValueError:
|
||
return True
|
||
return False
|
||
|
||
|
||
def load_geo_cache() -> dict:
|
||
if CACHE_FILE.exists():
|
||
try:
|
||
with open(CACHE_FILE, "r", encoding="utf-8") as f:
|
||
return json.load(f)
|
||
except Exception:
|
||
pass
|
||
return {}
|
||
|
||
|
||
def save_geo_cache(cache: dict) -> None:
|
||
try:
|
||
CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||
with open(CACHE_FILE, "w", encoding="utf-8") as f:
|
||
json.dump(cache, f, ensure_ascii=False, indent=0)
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def fetch_geo(ip: str) -> str:
|
||
if is_private_ip(ip):
|
||
return "Локальная сеть"
|
||
try:
|
||
req = urllib.request.Request(API_URL.format(ip=ip), headers={"User-Agent": "NPM-Log-Dashboard/1"})
|
||
with urllib.request.urlopen(req, timeout=3) as r:
|
||
data = json.loads(r.read().decode())
|
||
if data.get("status") != "success":
|
||
return "—"
|
||
parts = [data.get("country") or "", data.get("city") or ""]
|
||
loc = ", ".join(p for p in parts if p).strip() or "—"
|
||
isp = (data.get("isp") or "").strip()
|
||
if isp:
|
||
loc = f"{loc} ({isp})" if loc != "—" else isp
|
||
return loc or "—"
|
||
except Exception:
|
||
return "—"
|
||
|
||
|
||
def ensure_geo_for_ips(cache: dict, ips: list[str], max_new: int) -> None:
|
||
to_fetch = [ip for ip in ips if ip and not is_private_ip(ip) and ip not in cache]
|
||
to_fetch = to_fetch[:max_new]
|
||
for ip in to_fetch:
|
||
cache[ip] = fetch_geo(ip)
|
||
time.sleep(1.35) # ~44/min to stay under 45
|
||
|
||
|
||
def main():
|
||
out_path = sys.argv[1] if len(sys.argv) > 1 else None
|
||
try:
|
||
now = datetime.now(timezone.utc)
|
||
except Exception:
|
||
now = datetime.utcnow()
|
||
two_days_ago = now - timedelta(days=2)
|
||
try:
|
||
t_cut = two_days_ago.timestamp()
|
||
except Exception:
|
||
t_cut = (two_days_ago - datetime(1970, 1, 1)).total_seconds()
|
||
|
||
counts_by_domain: dict[str, int] = defaultdict(int)
|
||
counts_by_ip: dict[str, int] = defaultdict(int)
|
||
counts_by_domain_ip: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int))
|
||
all_entries: list[dict] = []
|
||
|
||
log_files = sorted(LOG_DIR.glob("proxy-host-*_access.log"))
|
||
for log_path in log_files:
|
||
try:
|
||
with open(log_path, "r", encoding="utf-8", errors="ignore") as f:
|
||
lines = f.readlines()
|
||
except Exception:
|
||
continue
|
||
for line in lines:
|
||
entry = parse_line(line)
|
||
if not entry:
|
||
continue
|
||
domain = entry["domain"]
|
||
client = entry["client"]
|
||
try:
|
||
et = entry["time"].timestamp()
|
||
except Exception:
|
||
try:
|
||
et = entry["time"].replace(tzinfo=timezone.utc).timestamp()
|
||
except Exception:
|
||
et = 0
|
||
if et >= t_cut:
|
||
counts_by_domain[domain] += 1
|
||
counts_by_ip[client] += 1
|
||
counts_by_domain_ip[domain][client] += 1
|
||
all_entries.append(entry)
|
||
|
||
# Лента = все запросы за последние 2 суток (с лимитом FEED_MAX для размера HTML)
|
||
def entry_ts(e: dict) -> float:
|
||
try:
|
||
return e["time"].timestamp()
|
||
except Exception:
|
||
try:
|
||
return e["time"].replace(tzinfo=timezone.utc).timestamp()
|
||
except Exception:
|
||
return 0.0
|
||
feed_2d = [e for e in all_entries if entry_ts(e) >= t_cut]
|
||
feed_2d.sort(key=lambda x: x["time"], reverse=True)
|
||
total_2d = len(feed_2d)
|
||
feed = feed_2d[:FEED_MAX]
|
||
feed_capped = total_2d > FEED_MAX
|
||
feed_serial = [
|
||
{"t": e["date_s"][:20], "d": e["domain"], "m": e["method"], "p": e["path"], "s": e["status"], "c": e["client"]}
|
||
for e in feed
|
||
]
|
||
|
||
# Порядок доменов как в сводке (по убыванию запросов)
|
||
domain_order = sorted(counts_by_domain.keys(), key=lambda d: -counts_by_domain[d])
|
||
# Геокэш: топ-5 IP по каждому домену + часть уникальных из ленты
|
||
geo_cache = load_geo_cache()
|
||
ips_for_geo = []
|
||
for d in domain_order:
|
||
top5_for_d = [ip for ip, _ in sorted(counts_by_domain_ip[d].items(), key=lambda x: -x[1])[:5]]
|
||
ips_for_geo.extend(top5_for_d)
|
||
ips_for_geo = list(dict.fromkeys(ips_for_geo))
|
||
feed_ips = list(dict.fromkeys(e["client"] for e in feed[:500]))
|
||
ensure_geo_for_ips(geo_cache, ips_for_geo + feed_ips, GEO_MAX_NEW_PER_RUN)
|
||
save_geo_cache(geo_cache)
|
||
|
||
def geo_label(ip: str) -> str:
|
||
if is_private_ip(ip):
|
||
return "Локальная сеть"
|
||
return geo_cache.get(ip, "—")
|
||
|
||
# Сводка по доменам (таблица)
|
||
summary_domain_rows = []
|
||
for domain in domain_order:
|
||
summary_domain_rows.append(f"<tr><td>{domain}</td><td>{counts_by_domain[domain]}</td></tr>")
|
||
summary_domain_table = "\n".join(summary_domain_rows)
|
||
|
||
# Сводка по IP: топ-5 по каждому домену + остальные
|
||
summary_ip_parts = []
|
||
for domain in domain_order:
|
||
sorted_ips_d = sorted(counts_by_domain_ip[domain].items(), key=lambda x: -x[1])
|
||
top5_d = sorted_ips_d[:5]
|
||
rest_d = sum(c for _, c in sorted_ips_d[5:])
|
||
rows = []
|
||
for ip, cnt in top5_d:
|
||
geo = geo_label(ip)
|
||
link_2ip = f'<a href="https://2ip.ru/whois/?ip={ip}" target="_blank" rel="noopener">{ip}</a>'
|
||
rows.append(f"<tr><td>{link_2ip}</td><td>{cnt}</td><td>{geo}</td></tr>")
|
||
if rest_d:
|
||
rows.append(f'<tr><td><em>Остальные IP</em></td><td>{rest_d}</td><td>—</td></tr>')
|
||
summary_ip_parts.append(
|
||
f'<tr class="domain-header"><td colspan="3"><strong>{domain}</strong></td></tr>\n' + "\n".join(rows)
|
||
)
|
||
summary_ip_table = "\n".join(summary_ip_parts)
|
||
|
||
# Гео для всех IP в кэше (для вывода в ленте в JS)
|
||
geo_map = {ip: geo_label(ip) for ip in set(e["client"] for e in feed)}
|
||
geo_map_json = json.dumps(geo_map, ensure_ascii=False)
|
||
|
||
domains_list = domain_order
|
||
feed_note = f" (показаны последние {FEED_MAX:,} из {total_2d:,})" if feed_capped else ""
|
||
feed_count = len(feed_serial)
|
||
|
||
_gen_time = datetime.now().strftime("%d.%m.%Y %H:%M")
|
||
# Favicon: мини-терминал с логами (тёмный фон, cyan строки)
|
||
favicon_svg = """<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32">
|
||
<rect width="32" height="32" rx="5" fill="#1a1a2e"/>
|
||
<rect x="4" y="6" width="24" height="20" rx="2" fill="#0f0f1a" stroke="#7fdbff" stroke-width="1"/>
|
||
<line x1="8" y1="11" x2="22" y2="11" stroke="#7fdbff" stroke-width="1"/>
|
||
<line x1="8" y1="16" x2="18" y2="16" stroke="#7fdbff" stroke-width="1"/>
|
||
<line x1="8" y1="21" x2="24" y2="21" stroke="#7fdbff" stroke-width="1"/>
|
||
<circle cx="10" cy="11" r="1" fill="#7fdbff"/>
|
||
</svg>"""
|
||
favicon_data_url = "data:image/svg+xml;base64," + base64.b64encode(favicon_svg.encode("utf-8")).decode("ascii")
|
||
html_start = f"""<!DOCTYPE html>
|
||
<html lang="ru">
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<meta http-equiv="Cache-Control" content="no-cache, no-store, must-revalidate">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||
<title>Обращения к внешним URL (NPM)</title>
|
||
<link rel="icon" type="image/svg+xml" href="{favicon_data_url}">
|
||
<style>
|
||
body {{ font-family: system-ui, sans-serif; margin: 1rem; background: #1a1a2e; color: #eee; }}
|
||
h1 {{ font-size: 1.2rem; }}
|
||
h2 {{ font-size: 1rem; margin-top: 1.5rem; }}
|
||
table {{ border-collapse: collapse; width: 100%; font-size: 0.85rem; }}
|
||
th, td {{ border: 1px solid #444; padding: 0.35rem 0.5rem; text-align: left; }}
|
||
th {{ background: #16213e; }}
|
||
tr:nth-child(even) {{ background: #0f0f1a; }}
|
||
tr.domain-header td {{ background: #16213e; padding: 0.4rem 0.5rem; }}
|
||
.meta {{ color: #888; font-size: 0.8rem; margin-bottom: 1rem; }}
|
||
.controls {{ display: flex; align-items: center; gap: 1rem; flex-wrap: wrap; margin-bottom: 0.5rem; }}
|
||
.controls label {{ display: flex; align-items: center; gap: 0.35rem; }}
|
||
.controls select {{ background: #0f0f1a; color: #eee; border: 1px solid #444; padding: 0.25rem 0.5rem; }}
|
||
.pagination {{ margin-top: 0.5rem; display: flex; align-items: center; gap: 0.5rem; flex-wrap: wrap; }}
|
||
.pagination button {{ background: #16213e; color: #eee; border: 1px solid #444; padding: 0.35rem 0.75rem; cursor: pointer; }}
|
||
.pagination button:disabled {{ opacity: 0.5; cursor: not-allowed; }}
|
||
.geo {{ font-size: 0.8rem; color: #7fdbff; }}
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<h1>Обращения к внешним URL (Nginx Proxy Manager)</h1>
|
||
<p class="meta">Сводка за 2 суток; лента — последние {feed_count:,} запросов. Расчёт по крону раз в 15 мин. <em>Сгенерировано: {_gen_time}</em></p>
|
||
|
||
<h2>Запросы по доменам (2 суток)</h2>
|
||
<table>
|
||
<thead><tr><th>Домен</th><th>Запросов</th></tr></thead>
|
||
<tbody>{summary_domain_table}</tbody>
|
||
</table>
|
||
|
||
<h2>Запросы по IP (топ-5 по каждому домену + остальные)</h2>
|
||
<table>
|
||
<thead><tr><th>IP</th><th>Запросов</th><th>Местоположение</th></tr></thead>
|
||
<tbody>{summary_ip_table}</tbody>
|
||
</table>
|
||
|
||
<h2>Лента запросов (последние {feed_count:,})</h2>
|
||
<div class="controls">
|
||
<label>Домен: <select id="filterDomain">
|
||
<option value="">Все</option>
|
||
</select></label>
|
||
<label>На странице: <select id="pageSize">
|
||
<option value="50">50</option>
|
||
<option value="100" selected>100</option>
|
||
<option value="200">200</option>
|
||
</select></label>
|
||
</div>
|
||
<table id="feedTable">
|
||
<thead><tr><th>Время</th><th>Домен</th><th>Метод</th><th>Путь</th><th>Статус</th><th>Client</th><th>Местоположение</th></tr></thead>
|
||
<tbody id="feedBody"></tbody>
|
||
</table>
|
||
<div class="pagination">
|
||
<button type="button" id="prevPage">← Назад</button>
|
||
<span id="pageInfo">—</span>
|
||
<button type="button" id="nextPage">Вперёд →</button>
|
||
</div>
|
||
|
||
<script type="application/json" id="feed-data"></script>
|
||
<script>
|
||
(function() {{
|
||
var el = document.getElementById("feed-data");
|
||
var data = JSON.parse(el.textContent);
|
||
var feedEntries = data.feed;
|
||
var geoMap = data.geo;
|
||
var domainsList = data.domains;
|
||
|
||
var currentPage = 1;
|
||
var pageSize = parseInt(document.getElementById("pageSize").value, 10);
|
||
var filterDomain = "";
|
||
|
||
var filterSelect = document.getElementById("filterDomain");
|
||
for (var i = 0; i < domainsList.length; i++) {{
|
||
var opt = document.createElement("option");
|
||
opt.value = domainsList[i];
|
||
opt.textContent = domainsList[i];
|
||
filterSelect.appendChild(opt);
|
||
}}
|
||
|
||
document.getElementById("pageSize").onchange = function() {{
|
||
pageSize = parseInt(this.value, 10);
|
||
currentPage = 1;
|
||
render();
|
||
}};
|
||
filterSelect.onchange = function() {{
|
||
filterDomain = this.value;
|
||
currentPage = 1;
|
||
render();
|
||
}};
|
||
document.getElementById("prevPage").onclick = function() {{
|
||
currentPage = Math.max(1, currentPage - 1);
|
||
render();
|
||
}};
|
||
document.getElementById("nextPage").onclick = function() {{
|
||
currentPage++;
|
||
render();
|
||
}};
|
||
|
||
function getFiltered() {{
|
||
if (!filterDomain) return feedEntries;
|
||
var out = [];
|
||
for (var i = 0; i < feedEntries.length; i++)
|
||
if (feedEntries[i].d === filterDomain) out.push(feedEntries[i]);
|
||
return out;
|
||
}}
|
||
|
||
function render() {{
|
||
var list = getFiltered();
|
||
var totalPages = Math.max(1, Math.ceil(list.length / pageSize));
|
||
if (currentPage > totalPages) currentPage = totalPages;
|
||
var start = (currentPage - 1) * pageSize;
|
||
var page = list.slice(start, start + pageSize);
|
||
|
||
var tbody = document.getElementById("feedBody");
|
||
var html = "";
|
||
for (var i = 0; i < page.length; i++) {{
|
||
var e = page[i];
|
||
var geo = geoMap[e.c] || "—";
|
||
var ipLink = e.c ? "<a href=\\"https://2ip.ru/whois/?ip=" + e.c + "\\" target=\\"_blank\\" rel=\\"noopener\\">" + e.c + "</a>" : e.c;
|
||
html += "<tr><td>" + e.t + "</td><td>" + e.d + "</td><td>" + e.m + "</td><td>" + e.p + "</td><td>" + e.s + "</td><td>" + ipLink + "</td><td class=\\"geo\\">" + geo + "</td></tr>";
|
||
}}
|
||
tbody.innerHTML = html || "<tr><td colspan=\\"7\\">Нет записей</td></tr>";
|
||
|
||
document.getElementById("pageInfo").textContent = "Страница " + currentPage + " из " + totalPages + " (всего " + list.length + ")";
|
||
document.getElementById("prevPage").disabled = currentPage <= 1;
|
||
document.getElementById("nextPage").disabled = currentPage >= totalPages;
|
||
}}
|
||
|
||
render();
|
||
}})();
|
||
</script>
|
||
</body>
|
||
</html>
|
||
"""
|
||
# JSON для ленты: экранируем </script> чтобы не закрыть тег
|
||
payload = {"feed": feed_serial, "geo": geo_map, "domains": domains_list}
|
||
feed_json_raw = json.dumps(payload, ensure_ascii=False)
|
||
feed_json_safe = feed_json_raw.replace("<", "\\u003c").replace(">", "\\u003e")
|
||
|
||
if out_path:
|
||
out_dir = Path(out_path).parent
|
||
out_dir.mkdir(parents=True, exist_ok=True)
|
||
html_full = html_start.replace('<script type="application/json" id="feed-data"></script>',
|
||
'<script type="application/json" id="feed-data">' + feed_json_safe + '</script>')
|
||
with open(out_path, "w", encoding="utf-8") as f:
|
||
f.write(html_full)
|
||
else:
|
||
html_full = html_start.replace('<script type="application/json" id="feed-data"></script>',
|
||
'<script type="application/json" id="feed-data">' + feed_json_safe + '</script>')
|
||
print(html_full)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|