remove: delete health check step from deploy workflow
This commit is contained in:
163
.github/workflows/deploy.yml
vendored
163
.github/workflows/deploy.yml
vendored
@@ -342,167 +342,6 @@ jobs:
|
||||
|
||||
echo "✅ Containers rebuilt and started"
|
||||
|
||||
- name: Health check
|
||||
uses: appleboy/ssh-action@v1.0.0
|
||||
with:
|
||||
host: ${{ vars.SERVER_HOST || secrets.SERVER_HOST }}
|
||||
username: ${{ vars.SERVER_USER || secrets.SERVER_USER }}
|
||||
key: ${{ secrets.SSH_PRIVATE_KEY }}
|
||||
port: ${{ vars.SSH_PORT || secrets.SSH_PORT || 22 }}
|
||||
script: |
|
||||
set -e
|
||||
echo "🏥 Running health checks..."
|
||||
|
||||
# Проверяем доступность сети
|
||||
check_network_availability() {
|
||||
echo "🔍 Checking network availability..."
|
||||
|
||||
if ! ping -c 1 -W 2 localhost > /dev/null 2>&1; then
|
||||
echo "❌ Localhost not reachable! Network issue detected."
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "✅ Network is available"
|
||||
return 0
|
||||
}
|
||||
|
||||
# Адаптивное ожидание готовности контейнеров
|
||||
wait_for_containers_ready() {
|
||||
local max_wait=300 # 5 минут максимум (увеличено для медленных контейнеров)
|
||||
local check_interval=10
|
||||
local elapsed=0
|
||||
|
||||
echo "⏳ Waiting for containers to be ready..."
|
||||
|
||||
while [ $elapsed -lt $max_wait ]; do
|
||||
# Проверяем, что все контейнеры запущены и здоровы (нет Exit, Restarting, или health: starting)
|
||||
local unhealthy=$(docker-compose ps 2>/dev/null | grep -c "Exit\|Restarting\|health: starting" || echo "0")
|
||||
if [ "$unhealthy" -gt 0 ]; then
|
||||
echo "⏳ Some containers not ready yet (${unhealthy} still starting), waiting ${check_interval}s... (${elapsed}/${max_wait}s)"
|
||||
sleep $check_interval
|
||||
elapsed=$((elapsed + check_interval))
|
||||
else
|
||||
# Все контейнеры запущены и здоровы
|
||||
local running_count=$(docker-compose ps 2>/dev/null | grep -c "Up.*healthy" || echo "0")
|
||||
if [ "$running_count" -gt 0 ]; then
|
||||
echo "✅ All containers are ready and healthy! (waited ${elapsed}s, ${running_count} containers healthy)"
|
||||
return 0
|
||||
else
|
||||
# Проверяем хотя бы что контейнеры запущены (даже если еще не healthy)
|
||||
local up_count=$(docker-compose ps 2>/dev/null | grep -c "Up" || echo "0")
|
||||
if [ "$up_count" -gt 0 ]; then
|
||||
echo "✅ All containers are up (${up_count} running), continuing with health checks..."
|
||||
return 0
|
||||
else
|
||||
echo "⏳ Waiting for containers to start... (${elapsed}/${max_wait}s)"
|
||||
sleep $check_interval
|
||||
elapsed=$((elapsed + check_interval))
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
echo "⚠️ Containers not fully ready after ${max_wait}s, but continuing with health checks..."
|
||||
return 0
|
||||
}
|
||||
|
||||
# Проверяем статус контейнеров
|
||||
echo "📊 Checking container status..."
|
||||
cd /home/prod
|
||||
docker-compose ps
|
||||
|
||||
# Проверяем сеть перед health checks
|
||||
if ! check_network_availability; then
|
||||
echo "⚠️ Network check failed, but continuing with health checks..."
|
||||
fi
|
||||
|
||||
# Ждем готовности контейнеров адаптивно
|
||||
wait_for_containers_ready
|
||||
|
||||
# Функция для проверки с экспоненциальным retry
|
||||
check_health() {
|
||||
local service=$1
|
||||
local url=$2
|
||||
local attempt=1
|
||||
local delay1=10
|
||||
local delay2=30
|
||||
local delay3=60
|
||||
local max_attempts=3
|
||||
|
||||
echo "🔍 Checking $service health..."
|
||||
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
if curl -f -s --max-time 10 "$url" > /dev/null 2>&1; then
|
||||
echo "✅ $service is healthy (attempt $attempt/$max_attempts)"
|
||||
return 0
|
||||
else
|
||||
if [ $attempt -lt $max_attempts ]; then
|
||||
case $attempt in
|
||||
1) delay=$delay1 ;;
|
||||
2) delay=$delay2 ;;
|
||||
*) delay=$delay3 ;;
|
||||
esac
|
||||
echo "⏳ $service not ready yet (attempt $attempt/$max_attempts), waiting ${delay} seconds..."
|
||||
sleep $delay
|
||||
else
|
||||
echo "❌ $service health check failed after $max_attempts attempts"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
# Общая функция для проверки всех сервисов
|
||||
run_health_checks() {
|
||||
local failed=0
|
||||
local services=(
|
||||
"Prometheus:http://localhost:9090/-/healthy:prometheus"
|
||||
"Grafana:http://localhost:3000/api/health:grafana"
|
||||
"Telegram Bot:http://localhost:8080/health:telegram-bot"
|
||||
"AnonBot:http://localhost:8081/health:anon-bot"
|
||||
)
|
||||
|
||||
for service_info in "${services[@]}"; do
|
||||
IFS=':' read -r service_name url container_name <<< "$service_info"
|
||||
echo "🔍 Checking $service_name..."
|
||||
if ! check_health "$service_name" "$url"; then
|
||||
echo "⚠️ $service_name health check failed"
|
||||
docker-compose logs --tail=30 "$container_name" || true
|
||||
failed=1
|
||||
fi
|
||||
done
|
||||
|
||||
return $failed
|
||||
}
|
||||
|
||||
# Проверяем все сервисы
|
||||
HEALTH_CHECK_FAILED=0
|
||||
if ! run_health_checks; then
|
||||
HEALTH_CHECK_FAILED=1
|
||||
fi
|
||||
|
||||
# Проверяем статус всех контейнеров
|
||||
echo "📊 Final container status:"
|
||||
docker-compose ps
|
||||
|
||||
# Проверяем, что все контейнеры запущены
|
||||
FAILED_CONTAINERS=$(docker-compose ps | grep -c "Exit\|Restarting" || true)
|
||||
if [ "$FAILED_CONTAINERS" -gt 0 ]; then
|
||||
echo "❌ Some containers are not running properly"
|
||||
docker-compose ps
|
||||
HEALTH_CHECK_FAILED=1
|
||||
fi
|
||||
|
||||
if [ $HEALTH_CHECK_FAILED -eq 1 ]; then
|
||||
echo "❌ Health checks failed!"
|
||||
exit 1
|
||||
else
|
||||
echo "✅ All health checks passed!"
|
||||
fi
|
||||
|
||||
- name: Update deploy history
|
||||
if: always()
|
||||
uses: appleboy/ssh-action@v1.0.0
|
||||
@@ -575,7 +414,7 @@ jobs:
|
||||
👤 Author: ${{ github.event.pull_request.user.login || github.actor }}
|
||||
${{ github.event.pull_request.number && format('🔀 PR: #{0}', github.event.pull_request.number) || '' }}
|
||||
|
||||
${{ job.status == 'success' && '✅ Deployment successful! All services are healthy.' || '❌ Deployment failed! Check logs for details.' }}
|
||||
${{ job.status == 'success' && '✅ Deployment successful! Containers started.' || '❌ Deployment failed! Check logs for details.' }}
|
||||
|
||||
🔗 View details: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
continue-on-error: true
|
||||
|
||||
Reference in New Issue
Block a user