Files
prod/.github/workflows/pipeline.yml
Andrey 0cdc40cd21 chore: enhance deployment workflow with improved health checks and manual trigger
- Updated the deployment job to allow manual triggering via workflow_dispatch.
- Implemented a retry mechanism for health checks on Prometheus and Grafana to improve reliability.
- Increased wait time for services to start before health checks are performed.
- Modified health check messages for better clarity and added logging for failed checks.
2026-01-25 16:58:16 +03:00

210 lines
7.6 KiB
YAML
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
name: CI & CD pipeline
on:
push:
branches: [ main, 'develop', 'dev-*', 'feature/**' ]
pull_request:
branches: [ main, develop ]
jobs:
test:
runs-on: ubuntu-latest
name: Test
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up Python 3.11
uses: actions/setup-python@v5
with:
python-version: '3.11'
cache: 'pip'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r tests/infra/requirements-test.txt
- name: Run infrastructure tests
run: |
python -m pytest tests/infra/ -v --tb=short
- name: Validate Prometheus config
run: |
python -m pytest tests/infra/test_prometheus_config.py -v
- name: Upload test results
if: always()
uses: actions/upload-artifact@v4
with:
name: test-results
path: |
.pytest_cache/
htmlcov/
retention-days: 7
- name: Send test results notification
if: always()
uses: appleboy/telegram-action@v1.0.0
with:
to: ${{ secrets.TELEGRAM_CHAT_ID }}
token: ${{ secrets.TELEGRAM_BOT_TOKEN }}
message: |
🧪 CI Tests ${{ job.status }}
Repository: prod
Branch: ${{ github.ref_name }}
Commit: ${{ github.sha }}
Author: ${{ github.actor }}
${{ job.status == 'success' && '✅ All tests passed! Ready for deployment.' || '❌ Tests failed! Deployment blocked.' }}
View details: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
continue-on-error: true
deploy:
runs-on: ubuntu-latest
name: Deploy
needs: test
if: github.event_name == 'workflow_dispatch' # Только ручной запуск через кнопку
environment:
name: production
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Deploy to server
uses: appleboy/ssh-action@v1.0.0
with:
host: ${{ vars.SERVER_HOST || secrets.SERVER_HOST }}
username: ${{ vars.SERVER_USER || secrets.SERVER_USER }}
key: ${{ secrets.SSH_PRIVATE_KEY }}
port: ${{ vars.SSH_PORT || secrets.SSH_PORT || 22 }}
script: |
set -e
echo "🚀 Starting deployment..."
# Переходим в директорию проекта
cd /home/prod
# Сохраняем текущий коммит для отката
CURRENT_COMMIT=$(git rev-parse HEAD)
echo "Current commit: $CURRENT_COMMIT" > /tmp/last_deploy_commit.txt
# Обновляем код
echo "📥 Pulling latest changes..."
# Исправляем права на файлы перед обновлением
sudo chown -R deploy:deploy /home/prod || true
git fetch origin main
git reset --hard origin/main
# Устанавливаем правильные права после обновления
sudo chown -R deploy:deploy /home/prod || true
# Проверяем, что изменения есть
NEW_COMMIT=$(git rev-parse HEAD)
if [ "$CURRENT_COMMIT" = "$NEW_COMMIT" ]; then
echo " No new changes to deploy"
else
echo "✅ Code updated: $CURRENT_COMMIT → $NEW_COMMIT"
fi
# Перезапускаем сервисы
echo "🔄 Restarting services..."
if command -v make &> /dev/null; then
make restart || docker-compose restart
else
cd /home/prod
docker-compose down
docker-compose up -d --build
fi
echo "✅ Deployment completed"
- name: Health check
uses: appleboy/ssh-action@v1.0.0
with:
host: ${{ vars.SERVER_HOST || secrets.SERVER_HOST }}
username: ${{ vars.SERVER_USER || secrets.SERVER_USER }}
key: ${{ secrets.SSH_PRIVATE_KEY }}
port: ${{ vars.SSH_PORT || secrets.SSH_PORT || 22 }}
script: |
echo "🏥 Running health checks..."
# Проверяем статус контейнеров сначала
echo "📊 Checking container status..."
cd /home/prod
docker-compose ps || docker ps --filter "name=bots_"
# Ждем запуска сервисов (увеличено время)
echo "⏳ Waiting for services to start (30 seconds)..."
sleep 30
# Функция для проверки с повторными попытками
check_health() {
local service=$1
local url=$2
local max_attempts=5
local attempt=1
echo "🔍 Checking $service health..."
while [ $attempt -le $max_attempts ]; do
if curl -f -s --max-time 5 "$url" > /dev/null 2>&1; then
echo "✅ $service is healthy (attempt $attempt/$max_attempts)"
return 0
else
echo "⏳ $service not ready yet (attempt $attempt/$max_attempts), waiting 10 seconds..."
sleep 10
attempt=$((attempt + 1))
fi
done
echo "❌ $service health check failed after $max_attempts attempts"
return 1
}
# Проверяем Prometheus с повторными попытками
if ! check_health "Prometheus" "http://localhost:9090/-/healthy"; then
echo "⚠️ Prometheus health check failed, but continuing..."
echo "📊 Checking Prometheus logs:"
docker-compose logs --tail=20 prometheus || true
fi
# Проверяем Grafana с повторными попытками
if ! check_health "Grafana" "http://localhost:3000/api/health"; then
echo "⚠️ Grafana health check failed, but continuing..."
echo "📊 Checking Grafana logs:"
docker-compose logs --tail=20 grafana || true
fi
# Проверяем статус контейнеров
echo "📊 Container status:"
cd /home/prod
docker-compose ps || docker ps --filter "name=bots_"
echo "✅ Health checks completed"
- name: Send deployment notification
if: always()
uses: appleboy/telegram-action@v1.0.0
with:
to: ${{ secrets.TELEGRAM_CHAT_ID }}
token: ${{ secrets.TELEGRAM_BOT_TOKEN }}
message: |
🚀 Deployment ${{ job.status }}
Repository: prod
Branch: ${{ github.ref_name }}
Commit: ${{ github.sha }}
Author: ${{ github.actor }}
${{ job.status == 'success' && '✅ Deployment successful!' || '❌ Deployment failed!' }}
View details: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
continue-on-error: true