chore: Обновление Docker Compose и CI/CD пайплайна
- Docker Compose теперь использует GitHub Secrets для токенов ботов (приоритет над .env) - Добавлена функция ручного отката с указанием коммита - Реализованы проверки работоспособности с экспоненциальной задержкой - Улучшены уведомления об откате
This commit is contained in:
1039
.github/workflows/deploy.yml
vendored
Normal file
1039
.github/workflows/deploy.yml
vendored
Normal file
File diff suppressed because it is too large
Load Diff
386
.github/workflows/pipeline.yml
vendored
386
.github/workflows/pipeline.yml
vendored
@@ -5,11 +5,23 @@ on:
|
||||
branches: [ main, 'develop', 'dev-*', 'feature/**' ]
|
||||
pull_request:
|
||||
branches: [ main, develop ]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
action:
|
||||
description: 'Action to perform'
|
||||
required: true
|
||||
type: choice
|
||||
options:
|
||||
- rollback
|
||||
rollback_to_commit:
|
||||
description: 'Commit hash to rollback to (optional, uses last deploy if empty)'
|
||||
required: false
|
||||
type: string
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
name: Test
|
||||
name: Test & Code Quality
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -25,6 +37,28 @@ jobs:
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r tests/infra/requirements-test.txt
|
||||
pip install flake8 black isort mypy || true
|
||||
|
||||
- name: Code formatting check (Black)
|
||||
run: |
|
||||
echo "🔍 Checking code formatting with Black..."
|
||||
black --check . || (echo "❌ Code formatting issues found. Run 'black .' to fix." && exit 1)
|
||||
|
||||
- name: Import sorting check (isort)
|
||||
run: |
|
||||
echo "🔍 Checking import sorting with isort..."
|
||||
isort --check-only . || (echo "❌ Import sorting issues found. Run 'isort .' to fix." && exit 1)
|
||||
|
||||
- name: Linting (flake8) - Critical errors
|
||||
run: |
|
||||
echo "🔍 Running flake8 linter (critical errors only)..."
|
||||
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
|
||||
|
||||
- name: Linting (flake8) - Warnings
|
||||
run: |
|
||||
echo "🔍 Running flake8 linter (warnings)..."
|
||||
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics || true
|
||||
continue-on-error: true
|
||||
|
||||
- name: Run infrastructure tests
|
||||
run: |
|
||||
@@ -34,6 +68,19 @@ jobs:
|
||||
run: |
|
||||
python -m pytest tests/infra/test_prometheus_config.py -v
|
||||
|
||||
- name: Check for merge conflicts
|
||||
if: github.event_name == 'pull_request'
|
||||
run: |
|
||||
echo "🔍 Checking for merge conflicts..."
|
||||
git fetch origin main
|
||||
if ! git merge --no-commit --no-ff origin/main; then
|
||||
echo "❌ Merge conflicts detected!"
|
||||
git merge --abort || true
|
||||
exit 1
|
||||
fi
|
||||
git merge --abort || true
|
||||
echo "✅ No merge conflicts detected"
|
||||
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
@@ -51,31 +98,181 @@ jobs:
|
||||
to: ${{ secrets.TELEGRAM_CHAT_ID }}
|
||||
token: ${{ secrets.TELEGRAM_BOT_TOKEN }}
|
||||
message: |
|
||||
🧪 CI Tests ${{ job.status }}
|
||||
${{ job.status == 'success' && '✅' || '❌' }} CI Tests: ${{ job.status }}
|
||||
|
||||
Repository: prod
|
||||
Branch: ${{ github.ref_name }}
|
||||
Commit: ${{ github.sha }}
|
||||
Author: ${{ github.actor }}
|
||||
📦 Repository: prod
|
||||
🌿 Branch: ${{ github.ref_name }}
|
||||
📝 Commit: ${{ github.sha }}
|
||||
👤 Author: ${{ github.actor }}
|
||||
|
||||
${{ job.status == 'success' && '✅ All tests passed! Ready for deployment.' || '❌ Tests failed! Deployment blocked.' }}
|
||||
${{ job.status == 'success' && '✅ All tests passed! Pull request will be created.' || '❌ Tests failed! Deployment blocked.' }}
|
||||
|
||||
View details: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
🔗 View details: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
continue-on-error: true
|
||||
|
||||
deploy:
|
||||
create-pr:
|
||||
runs-on: ubuntu-latest
|
||||
name: Deploy
|
||||
name: Create Pull Request
|
||||
needs: test
|
||||
if: github.event_name == 'workflow_dispatch' # Только ручной запуск через кнопку
|
||||
if: |
|
||||
github.event_name == 'push' &&
|
||||
needs.test.result == 'success' &&
|
||||
github.ref_name != 'main' &&
|
||||
github.ref_name != 'develop' &&
|
||||
(startsWith(github.ref_name, 'dev-') || startsWith(github.ref_name, 'feature/'))
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Check if PR already exists
|
||||
id: check-pr
|
||||
uses: actions/github-script@v6
|
||||
with:
|
||||
script: |
|
||||
const branchName = context.ref.replace('refs/heads/', '');
|
||||
const { data: prs } = await github.rest.pulls.list({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
head: `${context.repo.owner}:${branchName}`,
|
||||
base: 'main',
|
||||
state: 'open'
|
||||
});
|
||||
|
||||
if (prs.length > 0) {
|
||||
core.setOutput('exists', 'true');
|
||||
core.setOutput('number', prs[0].number);
|
||||
core.setOutput('url', prs[0].html_url);
|
||||
} else {
|
||||
core.setOutput('exists', 'false');
|
||||
}
|
||||
|
||||
- name: Update existing PR
|
||||
if: steps.check-pr.outputs.exists == 'true'
|
||||
uses: actions/github-script@v6
|
||||
with:
|
||||
script: |
|
||||
const prNumber = parseInt('${{ steps.check-pr.outputs.number }}');
|
||||
const branchName = context.ref.replace('refs/heads/', '');
|
||||
|
||||
await github.rest.pulls.update({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
pull_number: prNumber,
|
||||
title: `Merge ${branchName} into main`,
|
||||
body: `## Updated Changes\n\nPR updated with new commits after successful CI tests.\n\n- Latest commit: ${{ github.sha }}\n- Branch: \`${branchName}\`\n- Author: @${{ github.actor }}\n\n## Test Results\n\n✅ All tests passed successfully!\n\nPlease review the changes and merge when ready.`
|
||||
});
|
||||
|
||||
console.log(`✅ PR #${prNumber} updated successfully`);
|
||||
|
||||
- name: Create Pull Request
|
||||
if: steps.check-pr.outputs.exists == 'false'
|
||||
id: create-pr
|
||||
uses: peter-evans/create-pull-request@v5
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
branch: ${{ github.ref_name }}
|
||||
base: main
|
||||
title: "Merge ${{ github.ref_name }} into main"
|
||||
body: |
|
||||
## Changes
|
||||
|
||||
This PR was automatically created after successful CI tests.
|
||||
|
||||
- Branch: `${{ github.ref_name }}`
|
||||
- Commit: `${{ github.sha }}`
|
||||
- Author: @${{ github.actor }}
|
||||
|
||||
## Test Results
|
||||
|
||||
✅ All tests passed successfully!
|
||||
|
||||
Please review the changes and merge when ready.
|
||||
labels: |
|
||||
automated
|
||||
ready-for-review
|
||||
draft: false
|
||||
|
||||
- name: Get created PR number
|
||||
if: steps.check-pr.outputs.exists == 'false' && steps.create-pr.outcome == 'success'
|
||||
id: get-pr-number
|
||||
uses: actions/github-script@v6
|
||||
with:
|
||||
script: |
|
||||
const branchName = context.ref.replace('refs/heads/', '');
|
||||
const { data: prs } = await github.rest.pulls.list({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
head: `${context.repo.owner}:${branchName}`,
|
||||
base: 'main',
|
||||
state: 'open'
|
||||
});
|
||||
|
||||
if (prs.length > 0) {
|
||||
core.setOutput('number', prs[0].number);
|
||||
core.setOutput('url', prs[0].html_url);
|
||||
}
|
||||
|
||||
- name: Send PR notification - PR exists
|
||||
if: steps.check-pr.outputs.exists == 'true'
|
||||
uses: appleboy/telegram-action@v1.0.0
|
||||
with:
|
||||
to: ${{ secrets.TELEGRAM_CHAT_ID }}
|
||||
token: ${{ secrets.TELEGRAM_BOT_TOKEN }}
|
||||
message: |
|
||||
ℹ️ Pull Request Updated
|
||||
|
||||
📦 Repository: prod
|
||||
🌿 Branch: ${{ github.ref_name }} → main
|
||||
📝 Commit: ${{ github.sha }}
|
||||
👤 Author: ${{ github.actor }}
|
||||
|
||||
✅ All tests passed! PR #${{ steps.check-pr.outputs.number }} already exists and has been updated.
|
||||
|
||||
🔗 View PR: ${{ steps.check-pr.outputs.url }}
|
||||
continue-on-error: true
|
||||
|
||||
- name: Send PR notification - PR created
|
||||
if: steps.check-pr.outputs.exists == 'false' && steps.create-pr.outcome == 'success'
|
||||
uses: appleboy/telegram-action@v1.0.0
|
||||
with:
|
||||
to: ${{ secrets.TELEGRAM_CHAT_ID }}
|
||||
token: ${{ secrets.TELEGRAM_BOT_TOKEN }}
|
||||
message: |
|
||||
📝 Pull Request Created
|
||||
|
||||
📦 Repository: prod
|
||||
🌿 Branch: ${{ github.ref_name }} → main
|
||||
📝 Commit: ${{ github.sha }}
|
||||
👤 Author: ${{ github.actor }}
|
||||
|
||||
✅ All tests passed! Pull request #${{ steps.get-pr-number.outputs.number }} has been created and is ready for review.
|
||||
|
||||
🔗 View PR: ${{ steps.get-pr-number.outputs.url }}
|
||||
continue-on-error: true
|
||||
|
||||
rollback:
|
||||
runs-on: ubuntu-latest
|
||||
name: Manual Rollback
|
||||
if: |
|
||||
github.event_name == 'workflow_dispatch' &&
|
||||
github.event.inputs.action == 'rollback'
|
||||
environment:
|
||||
name: production
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: main
|
||||
|
||||
- name: Deploy to server
|
||||
- name: Manual Rollback
|
||||
uses: appleboy/ssh-action@v1.0.0
|
||||
with:
|
||||
host: ${{ vars.SERVER_HOST || secrets.SERVER_HOST }}
|
||||
@@ -84,126 +281,135 @@ jobs:
|
||||
port: ${{ vars.SSH_PORT || secrets.SSH_PORT || 22 }}
|
||||
script: |
|
||||
set -e
|
||||
echo "🚀 Starting deployment..."
|
||||
echo "🔄 Starting manual rollback..."
|
||||
|
||||
# Переходим в директорию проекта
|
||||
cd /home/prod
|
||||
DEPLOY_HISTORY="/home/prod/.deploy_history.txt"
|
||||
DEPLOY_HISTORY_SIZE="${DEPLOY_HISTORY_SIZE:-10}"
|
||||
|
||||
# Сохраняем текущий коммит для отката
|
||||
CURRENT_COMMIT=$(git rev-parse HEAD)
|
||||
echo "Current commit: $CURRENT_COMMIT" > /tmp/last_deploy_commit.txt
|
||||
|
||||
# Обновляем код
|
||||
echo "📥 Pulling latest changes..."
|
||||
|
||||
# Исправляем права на файлы перед обновлением
|
||||
sudo chown -R deploy:deploy /home/prod || true
|
||||
# Определяем коммит для отката
|
||||
if [ -n "${{ github.event.inputs.rollback_to_commit }}" ]; then
|
||||
ROLLBACK_COMMIT="${{ github.event.inputs.rollback_to_commit }}"
|
||||
echo "📝 Using specified commit: $ROLLBACK_COMMIT"
|
||||
else
|
||||
# Используем последний успешный деплой из истории
|
||||
ROLLBACK_COMMIT=$(grep "|success" "$DEPLOY_HISTORY" 2>/dev/null | tail -1 | cut -d'|' -f2 || echo "")
|
||||
|
||||
# Если нет в истории, используем сохраненный коммит
|
||||
if [ -z "$ROLLBACK_COMMIT" ]; then
|
||||
if [ -f "/tmp/last_deploy_commit.txt" ]; then
|
||||
ROLLBACK_COMMIT=$(cat /tmp/last_deploy_commit.txt)
|
||||
echo "📝 Using saved commit from /tmp/last_deploy_commit.txt: $ROLLBACK_COMMIT"
|
||||
else
|
||||
echo "❌ No commit specified and no previous deploy found!"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "📝 Using last successful deploy from history: $ROLLBACK_COMMIT"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Проверяем что коммит существует
|
||||
git fetch origin main
|
||||
git reset --hard origin/main
|
||||
if ! git rev-parse --verify "$ROLLBACK_COMMIT" > /dev/null 2>&1; then
|
||||
echo "❌ Commit $ROLLBACK_COMMIT not found!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Устанавливаем правильные права после обновления
|
||||
# Откатываем код
|
||||
echo "🔄 Rolling back to commit: $ROLLBACK_COMMIT"
|
||||
|
||||
# Исправляем права на файлы
|
||||
sudo chown -R deploy:deploy /home/prod || true
|
||||
|
||||
# Проверяем, что изменения есть
|
||||
NEW_COMMIT=$(git rev-parse HEAD)
|
||||
if [ "$CURRENT_COMMIT" = "$NEW_COMMIT" ]; then
|
||||
echo "ℹ️ No new changes to deploy"
|
||||
else
|
||||
echo "✅ Code updated: $CURRENT_COMMIT → $NEW_COMMIT"
|
||||
fi
|
||||
git reset --hard "$ROLLBACK_COMMIT"
|
||||
|
||||
# Перезапускаем сервисы
|
||||
echo "🔄 Restarting services..."
|
||||
if command -v make &> /dev/null; then
|
||||
make restart || docker-compose restart
|
||||
else
|
||||
cd /home/prod
|
||||
docker-compose down
|
||||
docker-compose up -d --build
|
||||
fi
|
||||
# Устанавливаем правильные права после отката
|
||||
sudo chown -R deploy:deploy /home/prod || true
|
||||
|
||||
echo "✅ Deployment completed"
|
||||
|
||||
- name: Health check
|
||||
uses: appleboy/ssh-action@v1.0.0
|
||||
with:
|
||||
host: ${{ vars.SERVER_HOST || secrets.SERVER_HOST }}
|
||||
username: ${{ vars.SERVER_USER || secrets.SERVER_USER }}
|
||||
key: ${{ secrets.SSH_PRIVATE_KEY }}
|
||||
port: ${{ vars.SSH_PORT || secrets.SSH_PORT || 22 }}
|
||||
script: |
|
||||
echo "🏥 Running health checks..."
|
||||
echo "✅ Code rolled back to: $ROLLBACK_COMMIT"
|
||||
|
||||
# Проверяем статус контейнеров сначала
|
||||
echo "📊 Checking container status..."
|
||||
cd /home/prod
|
||||
docker-compose ps || docker ps --filter "name=bots_"
|
||||
# Пересобираем все контейнеры с обновлением базовых образов и кешированием
|
||||
echo "🔨 Rebuilding all containers with --pull (updating base images, using cache)..."
|
||||
docker-compose down || true
|
||||
docker-compose build --pull
|
||||
docker-compose up -d
|
||||
|
||||
# Ждем запуска сервисов (увеличено время)
|
||||
echo "⏳ Waiting for services to start (30 seconds)..."
|
||||
sleep 30
|
||||
echo "✅ Containers rebuilt and started"
|
||||
|
||||
# Функция для проверки с повторными попытками
|
||||
# Ждем запуска сервисов
|
||||
echo "⏳ Waiting for services to start (45 seconds)..."
|
||||
sleep 45
|
||||
|
||||
# Health checks с экспоненциальным retry
|
||||
check_health() {
|
||||
local service=$1
|
||||
local url=$2
|
||||
local max_attempts=5
|
||||
local attempt=1
|
||||
local delays=(5 15 45)
|
||||
local max_attempts=${#delays[@]}
|
||||
|
||||
echo "🔍 Checking $service health..."
|
||||
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
if curl -f -s --max-time 5 "$url" > /dev/null 2>&1; then
|
||||
if curl -f -s --max-time 10 "$url" > /dev/null 2>&1; then
|
||||
echo "✅ $service is healthy (attempt $attempt/$max_attempts)"
|
||||
return 0
|
||||
else
|
||||
echo "⏳ $service not ready yet (attempt $attempt/$max_attempts), waiting 10 seconds..."
|
||||
sleep 10
|
||||
attempt=$((attempt + 1))
|
||||
if [ $attempt -lt $max_attempts ]; then
|
||||
delay=${delays[$((attempt - 1))]}
|
||||
echo "⏳ $service not ready yet (attempt $attempt/$max_attempts), waiting ${delay} seconds..."
|
||||
sleep $delay
|
||||
else
|
||||
echo "❌ $service health check failed after $max_attempts attempts"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
|
||||
echo "❌ $service health check failed after $max_attempts attempts"
|
||||
return 1
|
||||
}
|
||||
|
||||
# Проверяем Prometheus с повторными попытками
|
||||
if ! check_health "Prometheus" "http://localhost:9090/-/healthy"; then
|
||||
echo "⚠️ Prometheus health check failed, but continuing..."
|
||||
echo "📊 Checking Prometheus logs:"
|
||||
docker-compose logs --tail=20 prometheus || true
|
||||
HEALTH_CHECK_FAILED=0
|
||||
|
||||
check_health "Prometheus" "http://localhost:9090/-/healthy" || HEALTH_CHECK_FAILED=1
|
||||
check_health "Grafana" "http://localhost:3000/api/health" || HEALTH_CHECK_FAILED=1
|
||||
check_health "Telegram Bot" "http://localhost:8080/health" || HEALTH_CHECK_FAILED=1
|
||||
check_health "AnonBot" "http://localhost:8081/health" || HEALTH_CHECK_FAILED=1
|
||||
|
||||
if [ $HEALTH_CHECK_FAILED -eq 1 ]; then
|
||||
echo "⚠️ Some health checks failed, but rollback completed"
|
||||
else
|
||||
echo "✅ All health checks passed after rollback!"
|
||||
fi
|
||||
|
||||
# Проверяем Grafana с повторными попытками
|
||||
if ! check_health "Grafana" "http://localhost:3000/api/health"; then
|
||||
echo "⚠️ Grafana health check failed, but continuing..."
|
||||
echo "📊 Checking Grafana logs:"
|
||||
docker-compose logs --tail=20 grafana || true
|
||||
fi
|
||||
# Обновляем историю
|
||||
TIMESTAMP=$(date +"%Y-%m-%d %H:%M:%S")
|
||||
COMMIT_MESSAGE=$(git log -1 --pretty=format:"%s" "$ROLLBACK_COMMIT" 2>/dev/null || echo "Manual rollback")
|
||||
COMMIT_AUTHOR="${{ github.actor }}"
|
||||
echo "${TIMESTAMP}|${ROLLBACK_COMMIT}|${COMMIT_MESSAGE}|${COMMIT_AUTHOR}|rolled_back" >> "$DEPLOY_HISTORY"
|
||||
|
||||
# Проверяем статус контейнеров
|
||||
echo "📊 Container status:"
|
||||
cd /home/prod
|
||||
docker-compose ps || docker ps --filter "name=bots_"
|
||||
# Оставляем только последние N записей
|
||||
tail -n "$DEPLOY_HISTORY_SIZE" "$DEPLOY_HISTORY" > "${DEPLOY_HISTORY}.tmp" && mv "${DEPLOY_HISTORY}.tmp" "$DEPLOY_HISTORY"
|
||||
|
||||
echo "✅ Health checks completed"
|
||||
echo "✅ Rollback completed successfully"
|
||||
|
||||
- name: Send deployment notification
|
||||
- name: Send rollback notification
|
||||
if: always()
|
||||
uses: appleboy/telegram-action@v1.0.0
|
||||
with:
|
||||
to: ${{ secrets.TELEGRAM_CHAT_ID }}
|
||||
token: ${{ secrets.TELEGRAM_BOT_TOKEN }}
|
||||
message: |
|
||||
🚀 Deployment ${{ job.status }}
|
||||
🔄 Manual Rollback: ${{ job.status }}
|
||||
|
||||
Repository: prod
|
||||
Branch: ${{ github.ref_name }}
|
||||
Commit: ${{ github.sha }}
|
||||
Author: ${{ github.actor }}
|
||||
📦 Repository: prod
|
||||
🌿 Branch: main
|
||||
📝 Commit: ${{ github.event.inputs.rollback_to_commit || 'Previous successful deploy' }}
|
||||
👤 Author: ${{ github.actor }}
|
||||
|
||||
${{ job.status == 'success' && '✅ Deployment successful!' || '❌ Deployment failed!' }}
|
||||
${{ job.status == 'success' && '✅ Rollback completed successfully! Services restored to specified version.' || '❌ Rollback failed! Check logs for details.' }}
|
||||
|
||||
View details: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
🔗 View details: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
continue-on-error: true
|
||||
|
||||
Reference in New Issue
Block a user