- Integrated metrics scheduler start and stop functionality in `run_helper.py` for better resource management. - Improved logging for metrics server operations in `server_prometheus.py`, ensuring clearer error reporting and status updates. - Updated metrics middleware to collect comprehensive metrics for all event types, enhancing monitoring capabilities. - Added active user metrics tracking in `admin_handlers.py` to provide insights on user engagement. - Refactored command and callback handling in `metrics_middleware.py` to improve clarity and error handling.
207 lines
7.2 KiB
Python
207 lines
7.2 KiB
Python
"""
|
|
Metrics Scheduler for periodic updates of bot metrics.
|
|
Automatically updates active users, system metrics, and other periodic data.
|
|
"""
|
|
|
|
import asyncio
|
|
import logging
|
|
from datetime import datetime, timezone, timedelta
|
|
from typing import Optional
|
|
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
|
from apscheduler.triggers.cron import CronTrigger
|
|
from apscheduler.triggers.interval import IntervalTrigger
|
|
|
|
from .metrics import metrics
|
|
from .base_dependency_factory import get_global_instance
|
|
|
|
|
|
class MetricsScheduler:
|
|
"""Scheduler for periodic metrics updates."""
|
|
|
|
def __init__(self):
|
|
self.scheduler = AsyncIOScheduler()
|
|
self.logger = logging.getLogger(__name__)
|
|
self.bdf = None
|
|
self.bot_db = None
|
|
self.is_running = False
|
|
|
|
async def initialize(self):
|
|
"""Initialize scheduler with database connection."""
|
|
try:
|
|
self.bdf = get_global_instance()
|
|
self.bot_db = self.bdf.get_db()
|
|
self.logger.info("✅ Metrics scheduler initialized successfully")
|
|
except Exception as e:
|
|
self.logger.error(f"❌ Failed to initialize metrics scheduler: {e}")
|
|
|
|
def start_scheduler(self):
|
|
"""Start the metrics scheduler."""
|
|
if self.is_running:
|
|
self.logger.warning("⚠️ Metrics scheduler is already running")
|
|
return
|
|
|
|
try:
|
|
# Update active users every 5 minutes
|
|
self.scheduler.add_job(
|
|
self._update_active_users_metric,
|
|
IntervalTrigger(minutes=5),
|
|
id='update_active_users',
|
|
name='Update Active Users Metric',
|
|
replace_existing=True
|
|
)
|
|
|
|
# Update system metrics every minute
|
|
self.scheduler.add_job(
|
|
self._update_system_metrics,
|
|
IntervalTrigger(minutes=1),
|
|
id='update_system_metrics',
|
|
name='Update System Metrics',
|
|
replace_existing=True
|
|
)
|
|
|
|
# Daily metrics reset at midnight
|
|
self.scheduler.add_job(
|
|
self._daily_metrics_reset,
|
|
CronTrigger(hour=0, minute=0),
|
|
id='daily_metrics_reset',
|
|
name='Daily Metrics Reset',
|
|
replace_existing=True
|
|
)
|
|
|
|
# Start scheduler
|
|
self.scheduler.start()
|
|
self.is_running = True
|
|
self.logger.info("✅ Metrics scheduler started successfully")
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"❌ Failed to start metrics scheduler: {e}")
|
|
|
|
def stop_scheduler(self):
|
|
"""Stop the metrics scheduler."""
|
|
if not self.is_running:
|
|
self.logger.warning("⚠️ Metrics scheduler is not running")
|
|
return
|
|
|
|
try:
|
|
self.scheduler.shutdown()
|
|
self.is_running = False
|
|
self.logger.info("✅ Metrics scheduler stopped successfully")
|
|
except Exception as e:
|
|
self.logger.error(f"❌ Failed to stop metrics scheduler: {e}")
|
|
|
|
async def _update_active_users_metric(self):
|
|
"""Update active users metric from database."""
|
|
try:
|
|
if not self.bot_db:
|
|
await self.initialize()
|
|
if not self.bot_db:
|
|
self.logger.warning("⚠️ Cannot update active users: no database connection")
|
|
return
|
|
|
|
self.logger.debug("📊 Updating active users metric...")
|
|
|
|
# Count active users (last 24 hours)
|
|
import time
|
|
current_timestamp = int(time.time())
|
|
one_day_ago = current_timestamp - (24 * 60 * 60)
|
|
|
|
active_users_query = """
|
|
SELECT COUNT(DISTINCT user_id) as active_users
|
|
FROM our_users
|
|
WHERE date_changed > ?
|
|
"""
|
|
|
|
await self.bot_db.connect()
|
|
await self.bot_db.cursor.execute(active_users_query, (one_day_ago,))
|
|
result = await self.bot_db.cursor.fetchone()
|
|
active_users = result[0] if result else 0
|
|
await self.bot_db.close()
|
|
|
|
# Update metrics
|
|
metrics.set_active_users(active_users, "daily")
|
|
metrics.set_active_users(active_users, "total")
|
|
|
|
self.logger.debug(f"📊 Active users updated: {active_users}")
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"❌ Failed to update active users metric: {e}")
|
|
# Set fallback value
|
|
metrics.set_active_users(0, "daily")
|
|
metrics.set_active_users(0, "total")
|
|
|
|
async def _update_system_metrics(self):
|
|
"""Update system-related metrics."""
|
|
try:
|
|
# You can add system metrics here (CPU, memory, etc.)
|
|
# For now, we'll just log that the job is running
|
|
self.logger.debug("📊 System metrics update job running...")
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"❌ Failed to update system metrics: {e}")
|
|
|
|
async def _daily_metrics_reset(self):
|
|
"""Reset daily metrics at midnight."""
|
|
try:
|
|
self.logger.info("🔄 Daily metrics reset job running...")
|
|
|
|
# You can add daily metrics reset logic here
|
|
# For example, reset daily counters, update retention metrics, etc.
|
|
|
|
self.logger.info("✅ Daily metrics reset completed")
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"❌ Failed to reset daily metrics: {e}")
|
|
|
|
async def force_update_active_users(self):
|
|
"""Force update of active users metric (for testing)."""
|
|
await self._update_active_users_metric()
|
|
|
|
def get_scheduler_status(self) -> dict:
|
|
"""Get current scheduler status."""
|
|
if not self.is_running:
|
|
return {"status": "stopped", "jobs": 0}
|
|
|
|
jobs = self.scheduler.get_jobs()
|
|
return {
|
|
"status": "running",
|
|
"jobs": len(jobs),
|
|
"job_details": [
|
|
{
|
|
"id": job.id,
|
|
"name": job.name,
|
|
"next_run": job.next_run_time.isoformat() if job.next_run_time else None
|
|
}
|
|
for job in jobs
|
|
]
|
|
}
|
|
|
|
|
|
# Global metrics scheduler instance
|
|
metrics_scheduler: Optional[MetricsScheduler] = None
|
|
|
|
|
|
def get_metrics_scheduler() -> MetricsScheduler:
|
|
"""Get global metrics scheduler instance."""
|
|
global metrics_scheduler
|
|
if metrics_scheduler is None:
|
|
metrics_scheduler = MetricsScheduler()
|
|
return metrics_scheduler
|
|
|
|
|
|
async def start_metrics_scheduler() -> MetricsScheduler:
|
|
"""Start metrics scheduler and return instance."""
|
|
global metrics_scheduler
|
|
if metrics_scheduler is None:
|
|
metrics_scheduler = MetricsScheduler()
|
|
|
|
await metrics_scheduler.initialize()
|
|
metrics_scheduler.start_scheduler()
|
|
return metrics_scheduler
|
|
|
|
|
|
def stop_metrics_scheduler():
|
|
"""Stop metrics scheduler if running."""
|
|
global metrics_scheduler
|
|
if metrics_scheduler:
|
|
metrics_scheduler.stop_scheduler()
|