diff --git a/.env.example b/.env.example
index c9a3b5e4..829c6830 100644
--- a/.env.example
+++ b/.env.example
@@ -396,6 +396,7 @@ MAINTENANCE_MODE=false
MAINTENANCE_CHECK_INTERVAL=30
MAINTENANCE_AUTO_ENABLE=true
MAINTENANCE_MONITORING_ENABLED=true
+MAINTENANCE_RETRY_ATTEMPTS=1
MAINTENANCE_MESSAGE=Ведутся технические работы. Сервис временно недоступен. Попробуйте позже.
# ===== ЛОКАЛИЗАЦИЯ =====
diff --git a/README.md b/README.md
index 3e159fdb..fce7a360 100644
--- a/README.md
+++ b/README.md
@@ -473,6 +473,7 @@ MAINTENANCE_MODE=false
MAINTENANCE_AUTO_ENABLE=true
MAINTENANCE_MONITORING_ENABLED=true
MAINTENANCE_CHECK_INTERVAL=30
+MAINTENANCE_RETRY_ATTEMPTS=1
# Интервал проверки состояния панели (секунды)
MONITORING_INTERVAL=60
@@ -665,6 +666,7 @@ MAINTENANCE_MODE=false
MAINTENANCE_CHECK_INTERVAL=30
MAINTENANCE_AUTO_ENABLE=true
MAINTENANCE_MONITORING_ENABLED=true
+MAINTENANCE_RETRY_ATTEMPTS=1
# ===== ЛОКАЛИЗАЦИЯ =====
DEFAULT_LANGUAGE=ru
diff --git a/app/config.py b/app/config.py
index 3ecdb6b5..6da1388c 100644
--- a/app/config.py
+++ b/app/config.py
@@ -159,6 +159,7 @@ class Settings(BaseSettings):
MAINTENANCE_CHECK_INTERVAL: int = 30
MAINTENANCE_AUTO_ENABLE: bool = True
MAINTENANCE_MONITORING_ENABLED: bool = True
+ MAINTENANCE_RETRY_ATTEMPTS: int = 1
MAINTENANCE_MESSAGE: str = "🔧 Ведутся технические работы. Сервис временно недоступен. Попробуйте позже."
TELEGRAM_STARS_ENABLED: bool = True
@@ -1011,6 +1012,13 @@ class Settings(BaseSettings):
def get_maintenance_check_interval(self) -> int:
return self.MAINTENANCE_CHECK_INTERVAL
+ def get_maintenance_retry_attempts(self) -> int:
+ try:
+ attempts = int(self.MAINTENANCE_RETRY_ATTEMPTS)
+ except (TypeError, ValueError):
+ attempts = 1
+ return max(1, attempts)
+
def is_base_promo_group_period_discount_enabled(self) -> bool:
return self.BASE_PROMO_GROUP_PERIOD_DISCOUNTS_ENABLED
diff --git a/app/handlers/admin/maintenance.py b/app/handlers/admin/maintenance.py
index 09bfe6d4..82b67ced 100644
--- a/app/handlers/admin/maintenance.py
+++ b/app/handlers/admin/maintenance.py
@@ -234,7 +234,11 @@ async def check_panel_status(
f"👥 Пользователей онлайн: {status_data.get('users_online', 0)}",
f"🖥️ Нод онлайн: {status_data.get('nodes_online', 0)}/{status_data.get('total_nodes', 0)}"
]
-
+
+ attempts_used = status_data.get("attempts_used")
+ if attempts_used:
+ message_parts.append(f"🔁 Попыток проверки: {attempts_used}")
+
if status_data.get("api_error"):
message_parts.append(f"❌ Ошибка: {status_data['api_error'][:100]}")
diff --git a/app/services/maintenance_service.py b/app/services/maintenance_service.py
index 11509f6a..df183d0e 100644
--- a/app/services/maintenance_service.py
+++ b/app/services/maintenance_service.py
@@ -223,13 +223,18 @@ class MaintenanceService:
await self._load_status_from_cache()
self._check_task = asyncio.create_task(self._monitoring_loop())
- logger.info(f"🔄 Запущен мониторинг API Remnawave (интервал: {settings.get_maintenance_check_interval()}с)")
-
+ logger.info(
+ "🔄 Запущен мониторинг API Remnawave (интервал: %sс, попыток: %s)",
+ settings.get_maintenance_check_interval(),
+ settings.get_maintenance_retry_attempts(),
+ )
+
await self._notify_admins(f"""Мониторинг технических работ запущен
🔄 Интервал проверки: {settings.get_maintenance_check_interval()} секунд
🤖 Автовключение: {'Включено' if settings.is_maintenance_auto_enable() else 'Отключено'}
🎯 Порог ошибок: {self._max_consecutive_failures}
+🔁 Повторных попыток: {settings.get_maintenance_retry_attempts()}
Система будет следить за доступностью API.""", "info")
@@ -260,10 +265,10 @@ class MaintenanceService:
try:
if self._is_checking:
return self._status.api_status
-
+
self._is_checking = True
self._status.last_check = datetime.utcnow()
-
+
auth_params = settings.get_remnawave_auth_params()
api = RemnaWaveAPI(
base_url=auth_params["base_url"],
@@ -272,53 +277,74 @@ class MaintenanceService:
username=auth_params["username"],
password=auth_params["password"]
)
-
+
+ attempts = settings.get_maintenance_retry_attempts()
+
async with api:
- is_connected = await test_api_connection(api)
-
- if is_connected:
- if not self._status.api_status:
- await self._notify_admins(f"""API Remnawave восстановлено!
+ for attempt in range(1, attempts + 1):
+ is_connected = await test_api_connection(api)
+
+ if is_connected:
+ if attempt > 1:
+ logger.info(
+ "API Remnawave ответило с %s попытки", attempt
+ )
+
+ if not self._status.api_status:
+ await self._notify_admins(f"""API Remnawave восстановлено!
✅ Статус: Доступно
🕐 Время восстановления: {self._status.last_check.strftime('%H:%M:%S')}
🔄 Неудачных попыток было: {self._status.consecutive_failures}
API снова отвечает на запросы.""", "success")
-
- self._status.api_status = True
- self._status.consecutive_failures = 0
-
- if self._status.is_active and self._status.auto_enabled:
- await self.disable_maintenance()
- logger.info("✅ API восстановился, режим техработ автоматически отключен")
-
- return True
- else:
- was_available = self._status.api_status
- self._status.api_status = False
- self._status.consecutive_failures += 1
-
- if was_available:
- await self._notify_admins(f"""API Remnawave недоступно!
+
+ self._status.api_status = True
+ self._status.consecutive_failures = 0
+
+ if self._status.is_active and self._status.auto_enabled:
+ await self.disable_maintenance()
+ logger.info("✅ API восстановился, режим техработ автоматически отключен")
+
+ return True
+
+ if attempt < attempts:
+ logger.warning(
+ "API Remnawave недоступно (попытка %s/%s)",
+ attempt,
+ attempts,
+ )
+ await asyncio.sleep(1)
+
+ was_available = self._status.api_status
+ self._status.api_status = False
+ self._status.consecutive_failures += 1
+
+ if was_available:
+ await self._notify_admins(f"""API Remnawave недоступно!
❌ Статус: Недоступно
🕐 Время обнаружения: {self._status.last_check.strftime('%H:%M:%S')}
🔄 Попытка: {self._status.consecutive_failures}
Началась серия неудачных проверок API.""", "error")
-
- if (self._status.consecutive_failures >= self._max_consecutive_failures and
- not self._status.is_active and
- settings.is_maintenance_auto_enable()):
-
- await self.enable_maintenance(
- reason=f"Автоматическое включение после {self._status.consecutive_failures} неудачных проверок API",
- auto=True
- )
-
- return False
-
+
+ if (
+ self._status.consecutive_failures >= self._max_consecutive_failures
+ and not self._status.is_active
+ and settings.is_maintenance_auto_enable()
+ ):
+
+ await self.enable_maintenance(
+ reason=(
+ f"Автоматическое включение после {self._status.consecutive_failures} "
+ "неудачных проверок API"
+ ),
+ auto=True
+ )
+
+ return False
+
except Exception as e:
logger.error(f"Ошибка проверки API: {e}")
diff --git a/app/services/remnawave_service.py b/app/services/remnawave_service.py
index 34309614..72607382 100644
--- a/app/services/remnawave_service.py
+++ b/app/services/remnawave_service.py
@@ -1,3 +1,4 @@
+import asyncio
import logging
import os
import re
@@ -1998,67 +1999,110 @@ class RemnaWaveService:
}
async def check_panel_health(self) -> Dict[str, Any]:
- try:
- start_time = datetime.utcnow()
-
- async with self.get_api_client() as api:
- try:
- system_stats = await api.get_system_stats()
- api_available = True
- api_error = None
- except Exception as e:
- api_available = False
- api_error = str(e)
- system_stats = {}
-
- try:
- nodes = await api.get_all_nodes()
- nodes_online = sum(1 for node in nodes if node.is_connected and node.is_node_online)
- total_nodes = len(nodes)
- nodes_health = "healthy" if nodes_online > 0 else "unhealthy"
- except Exception:
- nodes_online = 0
- total_nodes = 0
- nodes_health = "unknown"
-
- end_time = datetime.utcnow()
- response_time = (end_time - start_time).total_seconds()
-
- if not api_available:
- status = "offline"
- elif response_time > 10:
- status = "degraded"
- elif nodes_health == "unhealthy":
- status = "degraded"
- else:
- status = "online"
-
- return {
- "status": status,
- "api_available": api_available,
- "api_error": api_error,
- "response_time": round(response_time, 2),
- "nodes_online": nodes_online,
- "total_nodes": total_nodes,
- "nodes_health": nodes_health,
- "users_online": system_stats.get('onlineStats', {}).get('onlineNow', 0),
- "total_users": system_stats.get('users', {}).get('totalUsers', 0),
- "last_check": end_time,
- "api_url": settings.REMNAWAVE_API_URL
- }
-
- except Exception as e:
- logger.error(f"Ошибка проверки здоровья панели: {e}")
- return {
- "status": "offline",
- "api_available": False,
- "api_error": str(e),
- "response_time": 0,
- "nodes_online": 0,
- "total_nodes": 0,
- "nodes_health": "unknown",
- "last_check": datetime.utcnow(),
- "api_url": settings.REMNAWAVE_API_URL
- }
+ attempts = settings.get_maintenance_retry_attempts()
+ attempts = max(1, attempts)
+
+ last_result: Optional[Dict[str, Any]] = None
+ last_error: Optional[Exception] = None
+
+ for attempt in range(1, attempts + 1):
+ try:
+ start_time = datetime.utcnow()
+
+ async with self.get_api_client() as api:
+ try:
+ system_stats = await api.get_system_stats()
+ api_available = True
+ api_error = None
+ except Exception as e:
+ api_available = False
+ api_error = str(e)
+ system_stats = {}
+
+ try:
+ nodes = await api.get_all_nodes()
+ nodes_online = sum(
+ 1 for node in nodes if node.is_connected and node.is_node_online
+ )
+ total_nodes = len(nodes)
+ nodes_health = "healthy" if nodes_online > 0 else "unhealthy"
+ except Exception:
+ nodes_online = 0
+ total_nodes = 0
+ nodes_health = "unknown"
+
+ end_time = datetime.utcnow()
+ response_time = (end_time - start_time).total_seconds()
+
+ if not api_available:
+ status = "offline"
+ elif response_time > 10:
+ status = "degraded"
+ elif nodes_health == "unhealthy":
+ status = "degraded"
+ else:
+ status = "online"
+
+ result = {
+ "status": status,
+ "api_available": api_available,
+ "api_error": api_error,
+ "response_time": round(response_time, 2),
+ "nodes_online": nodes_online,
+ "total_nodes": total_nodes,
+ "nodes_health": nodes_health,
+ "users_online": system_stats.get('onlineStats', {}).get('onlineNow', 0),
+ "total_users": system_stats.get('users', {}).get('totalUsers', 0),
+ "last_check": end_time,
+ "api_url": settings.REMNAWAVE_API_URL,
+ "attempts_used": attempt,
+ }
+
+ if result["api_available"]:
+ if attempt > 1:
+ logger.info("Панель Remnawave ответила с %s попытки", attempt)
+ return result
+
+ last_result = result
+
+ if attempt < attempts:
+ logger.warning(
+ "Панель Remnawave недоступна (попытка %s/%s): %s",
+ attempt,
+ attempts,
+ result.get("api_error") or "неизвестная ошибка",
+ )
+ await asyncio.sleep(1)
+
+ except Exception as error:
+ last_error = error
+ if attempt < attempts:
+ logger.warning(
+ "Ошибка проверки здоровья панели (попытка %s/%s): %s",
+ attempt,
+ attempts,
+ error,
+ )
+ await asyncio.sleep(1)
+ continue
+
+ logger.error(f"Ошибка проверки здоровья панели: {error}")
+
+ if last_result is not None:
+ return last_result
+
+ error_message = str(last_error) if last_error else "Неизвестная ошибка"
+ return {
+ "status": "offline",
+ "api_available": False,
+ "api_error": error_message,
+ "response_time": 0,
+ "nodes_online": 0,
+ "total_nodes": 0,
+ "nodes_health": "unknown",
+ "last_check": datetime.utcnow(),
+ "api_url": settings.REMNAWAVE_API_URL,
+ "attempts_used": attempts,
+ }
diff --git a/app/services/system_settings_service.py b/app/services/system_settings_service.py
index e6615804..34a74bae 100644
--- a/app/services/system_settings_service.py
+++ b/app/services/system_settings_service.py
@@ -262,6 +262,7 @@ class BotConfigurationService:
"MAINTENANCE_MESSAGE": "MAINTENANCE",
"MAINTENANCE_CHECK_INTERVAL": "MAINTENANCE",
"MAINTENANCE_AUTO_ENABLE": "MAINTENANCE",
+ "MAINTENANCE_RETRY_ATTEMPTS": "MAINTENANCE",
"WEBHOOK_URL": "WEBHOOK",
"WEBHOOK_SECRET": "WEBHOOK",
"VERSION_CHECK_ENABLED": "VERSION",
@@ -527,6 +528,17 @@ class BotConfigurationService:
),
"dependencies": "MAINTENANCE_CHECK_INTERVAL",
},
+ "MAINTENANCE_RETRY_ATTEMPTS": {
+ "description": (
+ "Сколько раз повторять проверку панели Remnawave перед фиксацией недоступности."
+ ),
+ "format": "Целое число не меньше 1.",
+ "example": "3",
+ "warning": (
+ "Большие значения увеличивают время реакции на реальные сбои, но помогают избежать ложных срабатываний."
+ ),
+ "dependencies": "MAINTENANCE_CHECK_INTERVAL",
+ },
"DISPLAY_NAME_BANNED_KEYWORDS": {
"description": (
"Список слов и фрагментов, при наличии которых в отображаемом имени "
diff --git a/main.py b/main.py
index 2ae6780b..d50e4689 100644
--- a/main.py
+++ b/main.py
@@ -409,6 +409,9 @@ async def main():
elif not maintenance_service._check_task or maintenance_service._check_task.done():
maintenance_task = asyncio.create_task(maintenance_service.start_monitoring())
stage.log(f"Интервал проверки: {settings.MAINTENANCE_CHECK_INTERVAL}с")
+ stage.log(
+ f"Повторных попыток проверки: {settings.get_maintenance_retry_attempts()}"
+ )
else:
maintenance_task = None
stage.skip("Служба техработ уже активна")