houseassassin/remnawave-bedolaga-telegram-bot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add configurable retry attempts for maintenance checks

Browse Source
This commit is contained in:
Egor
2025-10-31 22:15:05 +03:00
parent 10d36a829d
commit e02123c1ed
8 changed files with 201 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+1
View File
@@ -396,6 +396,7 @@ MAINTENANCE_MODE=false
MAINTENANCE_CHECK_INTERVAL=30
MAINTENANCE_AUTO_ENABLE=true
MAINTENANCE_MONITORING_ENABLED=true
MAINTENANCE_RETRY_ATTEMPTS=1
MAINTENANCE_MESSAGE=Ведутся технические работы. Сервис временно недоступен. Попробуйте позже.
# ===== ЛОКАЛИЗАЦИЯ =====
README.md
+2
View File
@@ -473,6 +473,7 @@ MAINTENANCE_MODE=false
MAINTENANCE_AUTO_ENABLE=true
MAINTENANCE_MONITORING_ENABLED=true
MAINTENANCE_CHECK_INTERVAL=30
MAINTENANCE_RETRY_ATTEMPTS=1
# Интервал проверки состояния панели (секунды)
MONITORING_INTERVAL=60
@@ -665,6 +666,7 @@ MAINTENANCE_MODE=false
MAINTENANCE_CHECK_INTERVAL=30
MAINTENANCE_AUTO_ENABLE=true
MAINTENANCE_MONITORING_ENABLED=true
MAINTENANCE_RETRY_ATTEMPTS=1
# ===== ЛОКАЛИЗАЦИЯ =====
DEFAULT_LANGUAGE=ru
app/config.py
+8
View File
@@ -159,6 +159,7 @@ class Settings(BaseSettings):
MAINTENANCE_CHECK_INTERVAL: int = 30
MAINTENANCE_AUTO_ENABLE: bool = True
MAINTENANCE_MONITORING_ENABLED: bool = True
MAINTENANCE_RETRY_ATTEMPTS: int = 1
MAINTENANCE_MESSAGE: str = "🔧 Ведутся технические работы. Сервис временно недоступен. Попробуйте позже."
TELEGRAM_STARS_ENABLED: bool = True
@@ -1011,6 +1012,13 @@ class Settings(BaseSettings):
def get_maintenance_check_interval(self) -> int:
return self.MAINTENANCE_CHECK_INTERVAL
def get_maintenance_retry_attempts(self) -> int:
try:
attempts = int(self.MAINTENANCE_RETRY_ATTEMPTS)
except (TypeError, ValueError):
attempts = 1
return max(1, attempts)
def is_base_promo_group_period_discount_enabled(self) -> bool:
return self.BASE_PROMO_GROUP_PERIOD_DISCOUNTS_ENABLED
app/handlers/admin/maintenance.py
+5 -1
View File
@@ -234,7 +234,11 @@ async def check_panel_status(
f"👥 Пользователей онлайн: {status_data.get('users_online', 0)}",
f"🖥️ Нод онлайн: {status_data.get('nodes_online', 0)}/{status_data.get('total_nodes', 0)}"
]
attempts_used = status_data.get("attempts_used")
if attempts_used:
message_parts.append(f"🔁 Попыток проверки: {attempts_used}")
if status_data.get("api_error"):
message_parts.append(f"❌ Ошибка: {status_data['api_error'][:100]}")
app/services/maintenance_service.py
+64 -38
View File
@@ -223,13 +223,18 @@ class MaintenanceService:
await self._load_status_from_cache()
self._check_task = asyncio.create_task(self._monitoring_loop())
logger.info(f"🔄 Запущен мониторинг API Remnawave (интервал: {settings.get_maintenance_check_interval()}с)")
logger.info(
"🔄 Запущен мониторинг API Remnawave (интервал: %sс, попыток: %s)",
settings.get_maintenance_check_interval(),
settings.get_maintenance_retry_attempts(),
)
await self._notify_admins(f"""Мониторинг технических работ запущен
🔄 <b>Интервал проверки:</b> {settings.get_maintenance_check_interval()} секунд
🤖 <b>Автовключение:</b> {'Включено' if settings.is_maintenance_auto_enable() else 'Отключено'}
🎯 <b>Порог ошибок:</b> {self._max_consecutive_failures}
🔁 <b>Повторных попыток:</b> {settings.get_maintenance_retry_attempts()}
Система будет следить за доступностью API.""", "info")
@@ -260,10 +265,10 @@ class MaintenanceService:
try:
if self._is_checking:
return self._status.api_status
self._is_checking = True
self._status.last_check = datetime.utcnow()
auth_params = settings.get_remnawave_auth_params()
api = RemnaWaveAPI(
base_url=auth_params["base_url"],
@@ -272,53 +277,74 @@ class MaintenanceService:
username=auth_params["username"],
password=auth_params["password"]
)
attempts = settings.get_maintenance_retry_attempts()
async with api:
is_connected = await test_api_connection(api)
if is_connected:
if not self._status.api_status:
await self._notify_admins(f"""API Remnawave восстановлено!
for attempt in range(1, attempts + 1):
is_connected = await test_api_connection(api)
if is_connected:
if attempt > 1:
logger.info(
"API Remnawave ответило с %s попытки", attempt
)
if not self._status.api_status:
await self._notify_admins(f"""API Remnawave восстановлено!
✅ <b>Статус:</b> Доступно
🕐 <b>Время восстановления:</b> {self._status.last_check.strftime('%H:%M:%S')}
🔄 <b>Неудачных попыток было:</b> {self._status.consecutive_failures}
API снова отвечает на запросы.""", "success")
self._status.api_status = True
self._status.consecutive_failures = 0
if self._status.is_active and self._status.auto_enabled:
await self.disable_maintenance()
logger.info("✅ API восстановился, режим техработ автоматически отключен")
return True
else:
was_available = self._status.api_status
self._status.api_status = False
self._status.consecutive_failures += 1
if was_available:
await self._notify_admins(f"""API Remnawave недоступно!
self._status.api_status = True
self._status.consecutive_failures = 0
if self._status.is_active and self._status.auto_enabled:
await self.disable_maintenance()
logger.info("✅ API восстановился, режим техработ автоматически отключен")
return True
if attempt < attempts:
logger.warning(
"API Remnawave недоступно (попытка %s/%s)",
attempt,
attempts,
)
await asyncio.sleep(1)
was_available = self._status.api_status
self._status.api_status = False
self._status.consecutive_failures += 1
if was_available:
await self._notify_admins(f"""API Remnawave недоступно!
❌ <b>Статус:</b> Недоступно
🕐 <b>Время обнаружения:</b> {self._status.last_check.strftime('%H:%M:%S')}
🔄 <b>Попытка:</b> {self._status.consecutive_failures}
Началась серия неудачных проверок API.""", "error")
if (self._status.consecutive_failures >= self._max_consecutive_failures and
not self._status.is_active and
settings.is_maintenance_auto_enable()):
await self.enable_maintenance(
reason=f"Автоматическое включение после {self._status.consecutive_failures} неудачных проверок API",
auto=True
)
return False
if (
self._status.consecutive_failures >= self._max_consecutive_failures
and not self._status.is_active
and settings.is_maintenance_auto_enable()
):
await self.enable_maintenance(
reason=(
f"Автоматическое включение после {self._status.consecutive_failures} "
"неудачных проверок API"
),
auto=True
)
return False
except Exception as e:
logger.error(f"Ошибка проверки API: {e}")
app/services/remnawave_service.py
+106 -62
View File
@@ -1,3 +1,4 @@
import asyncio
import logging
import os
import re
@@ -1998,67 +1999,110 @@ class RemnaWaveService:
}
async def check_panel_health(self) -> Dict[str, Any]:
try:
start_time = datetime.utcnow()
async with self.get_api_client() as api:
try:
system_stats = await api.get_system_stats()
api_available = True
api_error = None
except Exception as e:
api_available = False
api_error = str(e)
system_stats = {}
try:
nodes = await api.get_all_nodes()
nodes_online = sum(1 for node in nodes if node.is_connected and node.is_node_online)
total_nodes = len(nodes)
nodes_health = "healthy" if nodes_online > 0 else "unhealthy"
except Exception:
nodes_online = 0
total_nodes = 0
nodes_health = "unknown"
end_time = datetime.utcnow()
response_time = (end_time - start_time).total_seconds()
if not api_available:
status = "offline"
elif response_time > 10:
status = "degraded"
elif nodes_health == "unhealthy":
status = "degraded"
else:
status = "online"
return {
"status": status,
"api_available": api_available,
"api_error": api_error,
"response_time": round(response_time, 2),
"nodes_online": nodes_online,
"total_nodes": total_nodes,
"nodes_health": nodes_health,
"users_online": system_stats.get('onlineStats', {}).get('onlineNow', 0),
"total_users": system_stats.get('users', {}).get('totalUsers', 0),
"last_check": end_time,
"api_url": settings.REMNAWAVE_API_URL
}
except Exception as e:
logger.error(f"Ошибка проверки здоровья панели: {e}")
return {
"status": "offline",
"api_available": False,
"api_error": str(e),
"response_time": 0,
"nodes_online": 0,
"total_nodes": 0,
"nodes_health": "unknown",
"last_check": datetime.utcnow(),
"api_url": settings.REMNAWAVE_API_URL
}
attempts = settings.get_maintenance_retry_attempts()
attempts = max(1, attempts)
last_result: Optional[Dict[str, Any]] = None
last_error: Optional[Exception] = None
for attempt in range(1, attempts + 1):
try:
start_time = datetime.utcnow()
async with self.get_api_client() as api:
try:
system_stats = await api.get_system_stats()
api_available = True
api_error = None
except Exception as e:
api_available = False
api_error = str(e)
system_stats = {}
try:
nodes = await api.get_all_nodes()
nodes_online = sum(
1 for node in nodes if node.is_connected and node.is_node_online
)
total_nodes = len(nodes)
nodes_health = "healthy" if nodes_online > 0 else "unhealthy"
except Exception:
nodes_online = 0
total_nodes = 0
nodes_health = "unknown"
end_time = datetime.utcnow()
response_time = (end_time - start_time).total_seconds()
if not api_available:
status = "offline"
elif response_time > 10:
status = "degraded"
elif nodes_health == "unhealthy":
status = "degraded"
else:
status = "online"
result = {
"status": status,
"api_available": api_available,
"api_error": api_error,
"response_time": round(response_time, 2),
"nodes_online": nodes_online,
"total_nodes": total_nodes,
"nodes_health": nodes_health,
"users_online": system_stats.get('onlineStats', {}).get('onlineNow', 0),
"total_users": system_stats.get('users', {}).get('totalUsers', 0),
"last_check": end_time,
"api_url": settings.REMNAWAVE_API_URL,
"attempts_used": attempt,
}
if result["api_available"]:
if attempt > 1:
logger.info("Панель Remnawave ответила с %s попытки", attempt)
return result
last_result = result
if attempt < attempts:
logger.warning(
"Панель Remnawave недоступна (попытка %s/%s): %s",
attempt,
attempts,
result.get("api_error") or "неизвестная ошибка",
)
await asyncio.sleep(1)
except Exception as error:
last_error = error
if attempt < attempts:
logger.warning(
"Ошибка проверки здоровья панели (попытка %s/%s): %s",
attempt,
attempts,
error,
)
await asyncio.sleep(1)
continue
logger.error(f"Ошибка проверки здоровья панели: {error}")
if last_result is not None:
return last_result
error_message = str(last_error) if last_error else "Неизвестная ошибка"
return {
"status": "offline",
"api_available": False,
"api_error": error_message,
"response_time": 0,
"nodes_online": 0,
"total_nodes": 0,
"nodes_health": "unknown",
"last_check": datetime.utcnow(),
"api_url": settings.REMNAWAVE_API_URL,
"attempts_used": attempts,
}
app/services/system_settings_service.py
+12
View File
@@ -262,6 +262,7 @@ class BotConfigurationService:
"MAINTENANCE_MESSAGE": "MAINTENANCE",
"MAINTENANCE_CHECK_INTERVAL": "MAINTENANCE",
"MAINTENANCE_AUTO_ENABLE": "MAINTENANCE",
"MAINTENANCE_RETRY_ATTEMPTS": "MAINTENANCE",
"WEBHOOK_URL": "WEBHOOK",
"WEBHOOK_SECRET": "WEBHOOK",
"VERSION_CHECK_ENABLED": "VERSION",
@@ -527,6 +528,17 @@ class BotConfigurationService:
),
"dependencies": "MAINTENANCE_CHECK_INTERVAL",
},
"MAINTENANCE_RETRY_ATTEMPTS": {
"description": (
"Сколько раз повторять проверку панели Remnawave перед фиксацией недоступности."
),
"format": "Целое число не меньше 1.",
"example": "3",
"warning": (
"Большие значения увеличивают время реакции на реальные сбои, но помогают избежать ложных срабатываний."
),
"dependencies": "MAINTENANCE_CHECK_INTERVAL",
},
"DISPLAY_NAME_BANNED_KEYWORDS": {
"description": (
"Список слов и фрагментов, при наличии которых в отображаемом имени "
main.py
+3
View File
@@ -409,6 +409,9 @@ async def main():
elif not maintenance_service._check_task or maintenance_service._check_task.done():
maintenance_task = asyncio.create_task(maintenance_service.start_monitoring())
stage.log(f"Интервал проверки: {settings.MAINTENANCE_CHECK_INTERVAL}с")
stage.log(
f"Повторных попыток проверки: {settings.get_maintenance_retry_attempts()}"
)
else:
maintenance_task = None
stage.skip("Служба техработ уже активна")