- Updated the deployment job to allow manual triggering via workflow_dispatch. - Implemented a retry mechanism for health checks on Prometheus and Grafana to improve reliability. - Increased wait time for services to start before health checks are performed. - Modified health check messages for better clarity and added logging for failed checks.
210 lines
7.6 KiB
YAML
210 lines
7.6 KiB
YAML
name: CI & CD pipeline
|
||
|
||
on:
|
||
push:
|
||
branches: [ main, 'develop', 'dev-*', 'feature/**' ]
|
||
pull_request:
|
||
branches: [ main, develop ]
|
||
|
||
jobs:
|
||
test:
|
||
runs-on: ubuntu-latest
|
||
name: Test
|
||
|
||
steps:
|
||
- name: Checkout code
|
||
uses: actions/checkout@v4
|
||
|
||
- name: Set up Python 3.11
|
||
uses: actions/setup-python@v5
|
||
with:
|
||
python-version: '3.11'
|
||
cache: 'pip'
|
||
|
||
- name: Install dependencies
|
||
run: |
|
||
python -m pip install --upgrade pip
|
||
pip install -r tests/infra/requirements-test.txt
|
||
|
||
- name: Run infrastructure tests
|
||
run: |
|
||
python -m pytest tests/infra/ -v --tb=short
|
||
|
||
- name: Validate Prometheus config
|
||
run: |
|
||
python -m pytest tests/infra/test_prometheus_config.py -v
|
||
|
||
- name: Upload test results
|
||
if: always()
|
||
uses: actions/upload-artifact@v4
|
||
with:
|
||
name: test-results
|
||
path: |
|
||
.pytest_cache/
|
||
htmlcov/
|
||
retention-days: 7
|
||
|
||
- name: Send test results notification
|
||
if: always()
|
||
uses: appleboy/telegram-action@v1.0.0
|
||
with:
|
||
to: ${{ secrets.TELEGRAM_CHAT_ID }}
|
||
token: ${{ secrets.TELEGRAM_BOT_TOKEN }}
|
||
message: |
|
||
🧪 CI Tests ${{ job.status }}
|
||
|
||
Repository: prod
|
||
Branch: ${{ github.ref_name }}
|
||
Commit: ${{ github.sha }}
|
||
Author: ${{ github.actor }}
|
||
|
||
${{ job.status == 'success' && '✅ All tests passed! Ready for deployment.' || '❌ Tests failed! Deployment blocked.' }}
|
||
|
||
View details: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||
continue-on-error: true
|
||
|
||
deploy:
|
||
runs-on: ubuntu-latest
|
||
name: Deploy
|
||
needs: test
|
||
if: github.event_name == 'workflow_dispatch' # Только ручной запуск через кнопку
|
||
environment:
|
||
name: production
|
||
|
||
steps:
|
||
- name: Checkout code
|
||
uses: actions/checkout@v4
|
||
|
||
- name: Deploy to server
|
||
uses: appleboy/ssh-action@v1.0.0
|
||
with:
|
||
host: ${{ vars.SERVER_HOST || secrets.SERVER_HOST }}
|
||
username: ${{ vars.SERVER_USER || secrets.SERVER_USER }}
|
||
key: ${{ secrets.SSH_PRIVATE_KEY }}
|
||
port: ${{ vars.SSH_PORT || secrets.SSH_PORT || 22 }}
|
||
script: |
|
||
set -e
|
||
echo "🚀 Starting deployment..."
|
||
|
||
# Переходим в директорию проекта
|
||
cd /home/prod
|
||
|
||
# Сохраняем текущий коммит для отката
|
||
CURRENT_COMMIT=$(git rev-parse HEAD)
|
||
echo "Current commit: $CURRENT_COMMIT" > /tmp/last_deploy_commit.txt
|
||
|
||
# Обновляем код
|
||
echo "📥 Pulling latest changes..."
|
||
|
||
# Исправляем права на файлы перед обновлением
|
||
sudo chown -R deploy:deploy /home/prod || true
|
||
|
||
git fetch origin main
|
||
git reset --hard origin/main
|
||
|
||
# Устанавливаем правильные права после обновления
|
||
sudo chown -R deploy:deploy /home/prod || true
|
||
|
||
# Проверяем, что изменения есть
|
||
NEW_COMMIT=$(git rev-parse HEAD)
|
||
if [ "$CURRENT_COMMIT" = "$NEW_COMMIT" ]; then
|
||
echo "ℹ️ No new changes to deploy"
|
||
else
|
||
echo "✅ Code updated: $CURRENT_COMMIT → $NEW_COMMIT"
|
||
fi
|
||
|
||
# Перезапускаем сервисы
|
||
echo "🔄 Restarting services..."
|
||
if command -v make &> /dev/null; then
|
||
make restart || docker-compose restart
|
||
else
|
||
cd /home/prod
|
||
docker-compose down
|
||
docker-compose up -d --build
|
||
fi
|
||
|
||
echo "✅ Deployment completed"
|
||
|
||
- name: Health check
|
||
uses: appleboy/ssh-action@v1.0.0
|
||
with:
|
||
host: ${{ vars.SERVER_HOST || secrets.SERVER_HOST }}
|
||
username: ${{ vars.SERVER_USER || secrets.SERVER_USER }}
|
||
key: ${{ secrets.SSH_PRIVATE_KEY }}
|
||
port: ${{ vars.SSH_PORT || secrets.SSH_PORT || 22 }}
|
||
script: |
|
||
echo "🏥 Running health checks..."
|
||
|
||
# Проверяем статус контейнеров сначала
|
||
echo "📊 Checking container status..."
|
||
cd /home/prod
|
||
docker-compose ps || docker ps --filter "name=bots_"
|
||
|
||
# Ждем запуска сервисов (увеличено время)
|
||
echo "⏳ Waiting for services to start (30 seconds)..."
|
||
sleep 30
|
||
|
||
# Функция для проверки с повторными попытками
|
||
check_health() {
|
||
local service=$1
|
||
local url=$2
|
||
local max_attempts=5
|
||
local attempt=1
|
||
|
||
echo "🔍 Checking $service health..."
|
||
|
||
while [ $attempt -le $max_attempts ]; do
|
||
if curl -f -s --max-time 5 "$url" > /dev/null 2>&1; then
|
||
echo "✅ $service is healthy (attempt $attempt/$max_attempts)"
|
||
return 0
|
||
else
|
||
echo "⏳ $service not ready yet (attempt $attempt/$max_attempts), waiting 10 seconds..."
|
||
sleep 10
|
||
attempt=$((attempt + 1))
|
||
fi
|
||
done
|
||
|
||
echo "❌ $service health check failed after $max_attempts attempts"
|
||
return 1
|
||
}
|
||
|
||
# Проверяем Prometheus с повторными попытками
|
||
if ! check_health "Prometheus" "http://localhost:9090/-/healthy"; then
|
||
echo "⚠️ Prometheus health check failed, but continuing..."
|
||
echo "📊 Checking Prometheus logs:"
|
||
docker-compose logs --tail=20 prometheus || true
|
||
fi
|
||
|
||
# Проверяем Grafana с повторными попытками
|
||
if ! check_health "Grafana" "http://localhost:3000/api/health"; then
|
||
echo "⚠️ Grafana health check failed, but continuing..."
|
||
echo "📊 Checking Grafana logs:"
|
||
docker-compose logs --tail=20 grafana || true
|
||
fi
|
||
|
||
# Проверяем статус контейнеров
|
||
echo "📊 Container status:"
|
||
cd /home/prod
|
||
docker-compose ps || docker ps --filter "name=bots_"
|
||
|
||
echo "✅ Health checks completed"
|
||
|
||
- name: Send deployment notification
|
||
if: always()
|
||
uses: appleboy/telegram-action@v1.0.0
|
||
with:
|
||
to: ${{ secrets.TELEGRAM_CHAT_ID }}
|
||
token: ${{ secrets.TELEGRAM_BOT_TOKEN }}
|
||
message: |
|
||
🚀 Deployment ${{ job.status }}
|
||
|
||
Repository: prod
|
||
Branch: ${{ github.ref_name }}
|
||
Commit: ${{ github.sha }}
|
||
Author: ${{ github.actor }}
|
||
|
||
${{ job.status == 'success' && '✅ Deployment successful!' || '❌ Deployment failed!' }}
|
||
|
||
View details: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||
continue-on-error: true
|