refactor: упростил скрипты deploy.yml и ci.yml
This commit is contained in:
99
.github/workflows/ci.yml
vendored
Normal file
99
.github/workflows/ci.yml
vendored
Normal file
@@ -0,0 +1,99 @@
|
|||||||
|
name: CI pipeline
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [ 'dev-*', 'feature/**' ]
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
action:
|
||||||
|
description: 'Action to perform'
|
||||||
|
required: true
|
||||||
|
type: choice
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
test:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
name: Test & Code Quality
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Set up Python 3.11
|
||||||
|
uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: '3.11'
|
||||||
|
cache: 'pip'
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
run: |
|
||||||
|
python -m pip install --upgrade pip
|
||||||
|
pip install -r tests/infra/requirements-test.txt
|
||||||
|
pip install flake8 black isort mypy || true
|
||||||
|
|
||||||
|
- name: Code formatting check (Black)
|
||||||
|
run: |
|
||||||
|
echo "🔍 Checking code formatting with Black..."
|
||||||
|
black --check . || (echo "❌ Code formatting issues found. Run 'black .' to fix." && exit 1)
|
||||||
|
|
||||||
|
- name: Import sorting check (isort)
|
||||||
|
run: |
|
||||||
|
echo "🔍 Checking import sorting with isort..."
|
||||||
|
isort --check-only . || (echo "❌ Import sorting issues found. Run 'isort .' to fix." && exit 1)
|
||||||
|
|
||||||
|
- name: Linting (flake8) - Critical errors
|
||||||
|
run: |
|
||||||
|
echo "🔍 Running flake8 linter (critical errors only)..."
|
||||||
|
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
|
||||||
|
|
||||||
|
- name: Linting (flake8) - Warnings
|
||||||
|
run: |
|
||||||
|
echo "🔍 Running flake8 linter (warnings)..."
|
||||||
|
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics || true
|
||||||
|
continue-on-error: true
|
||||||
|
|
||||||
|
- name: Run infrastructure tests
|
||||||
|
run: |
|
||||||
|
python -m pytest tests/infra/ -v --tb=short
|
||||||
|
|
||||||
|
- name: Validate Prometheus config
|
||||||
|
run: |
|
||||||
|
python -m pytest tests/infra/test_prometheus_config.py -v
|
||||||
|
|
||||||
|
- name: Send test success notification
|
||||||
|
if: success()
|
||||||
|
uses: appleboy/telegram-action@v1.0.0
|
||||||
|
with:
|
||||||
|
to: ${{ secrets.TELEGRAM_CHAT_ID }}
|
||||||
|
token: ${{ secrets.TELEGRAM_BOT_TOKEN }}
|
||||||
|
message: |
|
||||||
|
✅ CI Tests Passed
|
||||||
|
|
||||||
|
📦 Repository: prod
|
||||||
|
🌿 Branch: ${{ github.ref_name }}
|
||||||
|
📝 Commit: ${{ github.sha }}
|
||||||
|
👤 Author: ${{ github.actor }}
|
||||||
|
|
||||||
|
✅ All tests passed! Code quality checks completed successfully.
|
||||||
|
|
||||||
|
🔗 View details: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||||
|
continue-on-error: true
|
||||||
|
|
||||||
|
- name: Send test failure notification
|
||||||
|
if: failure()
|
||||||
|
uses: appleboy/telegram-action@v1.0.0
|
||||||
|
with:
|
||||||
|
to: ${{ secrets.TELEGRAM_CHAT_ID }}
|
||||||
|
token: ${{ secrets.TELEGRAM_BOT_TOKEN }}
|
||||||
|
message: |
|
||||||
|
❌ CI Tests Failed
|
||||||
|
|
||||||
|
📦 Repository: prod
|
||||||
|
🌿 Branch: ${{ github.ref_name }}
|
||||||
|
📝 Commit: ${{ github.sha }}
|
||||||
|
👤 Author: ${{ github.actor }}
|
||||||
|
|
||||||
|
❌ Tests failed! Deployment blocked. Please fix the issues and try again.
|
||||||
|
|
||||||
|
🔗 View details: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||||
|
continue-on-error: true
|
||||||
870
.github/workflows/deploy.yml
vendored
870
.github/workflows/deploy.yml
vendored
@@ -1,21 +1,32 @@
|
|||||||
name: Deploy to Production
|
name: Deploy to Production
|
||||||
|
|
||||||
on:
|
on:
|
||||||
pull_request:
|
push:
|
||||||
types: [closed]
|
|
||||||
branches: [ main ]
|
branches: [ main ]
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
action:
|
||||||
|
description: 'Action to perform'
|
||||||
|
required: true
|
||||||
|
type: choice
|
||||||
|
options:
|
||||||
|
- deploy
|
||||||
|
- rollback
|
||||||
|
rollback_commit:
|
||||||
|
description: 'Commit hash to rollback to (optional, uses last successful if empty)'
|
||||||
|
required: false
|
||||||
|
type: string
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
deploy:
|
deploy:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
name: Deploy to Production
|
name: Deploy to Production
|
||||||
|
if: |
|
||||||
|
github.event_name == 'push' ||
|
||||||
|
(github.event_name == 'workflow_dispatch' && github.event.inputs.action == 'deploy')
|
||||||
concurrency:
|
concurrency:
|
||||||
group: production-deploy
|
group: production-deploy
|
||||||
cancel-in-progress: false
|
cancel-in-progress: false
|
||||||
if: |
|
|
||||||
(github.event_name == 'pull_request' && github.event.pull_request.merged == true) ||
|
|
||||||
github.event_name == 'workflow_dispatch'
|
|
||||||
environment:
|
environment:
|
||||||
name: production
|
name: production
|
||||||
|
|
||||||
@@ -25,101 +36,6 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
ref: main
|
ref: main
|
||||||
|
|
||||||
- name: Debug secrets availability
|
|
||||||
run: |
|
|
||||||
echo "🔍 Checking secrets availability..."
|
|
||||||
echo "TELEGRAM_BOT_TOKEN: $([ -n '${{ secrets.TELEGRAM_BOT_TOKEN }}' ] && echo '✅ Set' || echo '❌ Not set')"
|
|
||||||
echo "TELEGRAM_TEST_BOT_TOKEN: $([ -n '${{ secrets.TELEGRAM_TEST_BOT_TOKEN }}' ] && echo '✅ Set' || echo '⚠️ Not set (optional)')"
|
|
||||||
echo "ANON_BOT_TOKEN: $([ -n '${{ secrets.ANON_BOT_TOKEN }}' ] && echo '✅ Set' || echo '⚠️ Not set (optional)')"
|
|
||||||
echo "SSH_PRIVATE_KEY: $([ -n '${{ secrets.SSH_PRIVATE_KEY }}' ] && echo '✅ Set' || echo '❌ Not set')"
|
|
||||||
echo "SERVER_HOST: $([ -n '${{ vars.SERVER_HOST || secrets.SERVER_HOST }}' ] && echo '✅ Set' || echo '❌ Not set')"
|
|
||||||
echo "SERVER_USER: $([ -n '${{ vars.SERVER_USER || secrets.SERVER_USER }}' ] && echo '✅ Set' || echo '❌ Not set')"
|
|
||||||
|
|
||||||
- name: Validate Telegram Bot Tokens
|
|
||||||
uses: appleboy/ssh-action@v1.0.0
|
|
||||||
with:
|
|
||||||
host: ${{ vars.SERVER_HOST || secrets.SERVER_HOST }}
|
|
||||||
username: ${{ vars.SERVER_USER || secrets.SERVER_USER }}
|
|
||||||
key: ${{ secrets.SSH_PRIVATE_KEY }}
|
|
||||||
port: ${{ vars.SSH_PORT || secrets.SSH_PORT || 22 }}
|
|
||||||
script: |
|
|
||||||
set -e
|
|
||||||
export TELEGRAM_BOT_TOKEN="${{ secrets.TELEGRAM_BOT_TOKEN }}"
|
|
||||||
export TELEGRAM_TEST_BOT_TOKEN="${{ secrets.TELEGRAM_TEST_BOT_TOKEN }}"
|
|
||||||
export ANON_BOT_TOKEN="${{ secrets.ANON_BOT_TOKEN }}"
|
|
||||||
|
|
||||||
echo "🔍 Debug: Checking environment variables..."
|
|
||||||
echo "TELEGRAM_BOT_TOKEN length: ${#TELEGRAM_BOT_TOKEN}"
|
|
||||||
echo "TELEGRAM_TEST_BOT_TOKEN length: ${#TELEGRAM_TEST_BOT_TOKEN}"
|
|
||||||
echo "ANON_BOT_TOKEN length: ${#ANON_BOT_TOKEN}"
|
|
||||||
echo "TELEGRAM_BOT_TOKEN is empty: $([ -z "$TELEGRAM_BOT_TOKEN" ] && echo 'YES' || echo 'NO')"
|
|
||||||
|
|
||||||
echo "🔍 Validating Telegram Bot tokens from GitHub Secrets..."
|
|
||||||
|
|
||||||
# Функция для проверки токена с retry
|
|
||||||
validate_token() {
|
|
||||||
local token_name=$1
|
|
||||||
local token=$2
|
|
||||||
local max_retries=3
|
|
||||||
local retry=1
|
|
||||||
|
|
||||||
while [ $retry -le $max_retries ]; do
|
|
||||||
echo "🔍 Checking $token_name (attempt $retry/$max_retries)..."
|
|
||||||
|
|
||||||
response=$(curl -s --max-time 10 "https://api.telegram.org/bot${token}/getMe" || echo "")
|
|
||||||
|
|
||||||
if echo "$response" | grep -q '"ok":true'; then
|
|
||||||
bot_username=$(echo "$response" | grep -o '"username":"[^"]*"' | cut -d'"' -f4 || echo "unknown")
|
|
||||||
echo "✅ $token_name is valid (bot: @$bot_username)"
|
|
||||||
return 0
|
|
||||||
else
|
|
||||||
if [ $retry -lt $max_retries ]; then
|
|
||||||
echo "⏳ $token_name validation failed, retrying in 5 seconds..."
|
|
||||||
sleep 5
|
|
||||||
else
|
|
||||||
echo "❌ $token_name is invalid or unreachable"
|
|
||||||
echo "Response: $response"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
retry=$((retry + 1))
|
|
||||||
done
|
|
||||||
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
# Проверяем Telegram Helper Bot токен из Secrets
|
|
||||||
if [ -z "$TELEGRAM_BOT_TOKEN" ]; then
|
|
||||||
echo "❌ TELEGRAM_BOT_TOKEN not found in GitHub Secrets"
|
|
||||||
echo "💡 Make sure the secret is added to the 'production' environment or repository secrets"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if ! validate_token "Telegram Helper Bot Token" "$TELEGRAM_BOT_TOKEN"; then
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Проверяем TELEGRAM_TEST_BOT_TOKEN (опционально)
|
|
||||||
if [ -n "$TELEGRAM_TEST_BOT_TOKEN" ]; then
|
|
||||||
if ! validate_token "Telegram Test Bot Token" "$TELEGRAM_TEST_BOT_TOKEN"; then
|
|
||||||
echo "⚠️ Test bot token validation failed, but continuing..."
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
echo "ℹ️ TELEGRAM_TEST_BOT_TOKEN not set, skipping"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Проверяем AnonBot токен из Secrets
|
|
||||||
if [ -z "$ANON_BOT_TOKEN" ]; then
|
|
||||||
echo "⚠️ ANON_BOT_TOKEN not found in GitHub Secrets, skipping validation"
|
|
||||||
else
|
|
||||||
if ! validate_token "AnonBot Token" "$ANON_BOT_TOKEN"; then
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "✅ All token validations passed!"
|
|
||||||
|
|
||||||
- name: Deploy to server
|
- name: Deploy to server
|
||||||
uses: appleboy/ssh-action@v1.0.0
|
uses: appleboy/ssh-action@v1.0.0
|
||||||
with:
|
with:
|
||||||
@@ -133,214 +49,60 @@ jobs:
|
|||||||
export TELEGRAM_TEST_BOT_TOKEN="${{ secrets.TELEGRAM_TEST_BOT_TOKEN }}"
|
export TELEGRAM_TEST_BOT_TOKEN="${{ secrets.TELEGRAM_TEST_BOT_TOKEN }}"
|
||||||
export ANON_BOT_TOKEN="${{ secrets.ANON_BOT_TOKEN }}"
|
export ANON_BOT_TOKEN="${{ secrets.ANON_BOT_TOKEN }}"
|
||||||
|
|
||||||
echo "🔍 Debug: Checking environment variables in Deploy step..."
|
|
||||||
echo "TELEGRAM_BOT_TOKEN is set: $([ -n "$TELEGRAM_BOT_TOKEN" ] && echo 'YES' || echo 'NO')"
|
|
||||||
echo "TELEGRAM_TEST_BOT_TOKEN is set: $([ -n "$TELEGRAM_TEST_BOT_TOKEN" ] && echo 'YES' || echo 'NO')"
|
|
||||||
echo "ANON_BOT_TOKEN is set: $([ -n "$ANON_BOT_TOKEN" ] && echo 'YES' || echo 'NO')"
|
|
||||||
|
|
||||||
echo "🚀 Starting deployment to production..."
|
echo "🚀 Starting deployment to production..."
|
||||||
|
|
||||||
# Функция для безопасной записи в историю деплоев с использованием flock
|
|
||||||
safe_write_history() {
|
|
||||||
local entry="$1"
|
|
||||||
local history_file="/home/prod/.deploy_history.txt"
|
|
||||||
local lock_file="${history_file}.lock"
|
|
||||||
local history_size="${DEPLOY_HISTORY_SIZE:-10}"
|
|
||||||
|
|
||||||
if command -v flock > /dev/null 2>&1; then
|
|
||||||
# Используем flock напрямую с файлом (работает в zsh и bash)
|
|
||||||
flock -x "$lock_file" sh -c "
|
|
||||||
# Записываем новую запись
|
|
||||||
echo \"$entry\" >> \"$history_file\"
|
|
||||||
|
|
||||||
# Обрезаем файл атомарно
|
|
||||||
tail -n $history_size \"$history_file\" > \"${history_file}.tmp\"
|
|
||||||
mv \"${history_file}.tmp\" \"$history_file\"
|
|
||||||
|
|
||||||
echo \"✅ History updated safely\"
|
|
||||||
"
|
|
||||||
else
|
|
||||||
# Fallback: простая запись без блокировки
|
|
||||||
echo "$entry" >> "$history_file"
|
|
||||||
tail -n "$history_size" "$history_file" > "${history_file}.tmp"
|
|
||||||
mv "${history_file}.tmp" "$history_file"
|
|
||||||
echo "✅ History updated (fallback method)"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# Переходим в директорию проекта под пользователем deploy
|
|
||||||
cd /home/prod
|
cd /home/prod
|
||||||
|
|
||||||
# Сохраняем текущий коммит для отката
|
# Сохраняем информацию о коммите
|
||||||
CURRENT_COMMIT=$(git rev-parse HEAD)
|
CURRENT_COMMIT=$(git rev-parse HEAD)
|
||||||
COMMIT_MESSAGE=$(git log -1 --pretty=format:"%s" || echo "Unknown")
|
COMMIT_MESSAGE=$(git log -1 --pretty=format:"%s" || echo "Unknown")
|
||||||
COMMIT_AUTHOR=$(git log -1 --pretty=format:"%an" || echo "Unknown")
|
COMMIT_AUTHOR=$(git log -1 --pretty=format:"%an" || echo "Unknown")
|
||||||
TIMESTAMP=$(date +"%Y-%m-%d %H:%M:%S")
|
TIMESTAMP=$(date +"%Y-%m-%d %H:%M:%S")
|
||||||
|
|
||||||
# Сохраняем для быстрого доступа
|
|
||||||
echo "$CURRENT_COMMIT" > /tmp/last_deploy_commit.txt
|
|
||||||
echo "📝 Current commit: $CURRENT_COMMIT"
|
echo "📝 Current commit: $CURRENT_COMMIT"
|
||||||
echo "📝 Commit message: $COMMIT_MESSAGE"
|
echo "📝 Commit message: $COMMIT_MESSAGE"
|
||||||
echo "📝 Author: $COMMIT_AUTHOR"
|
echo "📝 Author: $COMMIT_AUTHOR"
|
||||||
|
|
||||||
# Сохраняем в файл истории деплоев безопасно
|
# Записываем в историю деплоев
|
||||||
DEPLOY_HISTORY="/home/prod/.deploy_history.txt"
|
HISTORY_FILE="/home/prod/.deploy_history.txt"
|
||||||
DEPLOY_HISTORY_SIZE="${DEPLOY_HISTORY_SIZE:-10}"
|
HISTORY_SIZE="${DEPLOY_HISTORY_SIZE:-10}"
|
||||||
|
echo "${TIMESTAMP}|${CURRENT_COMMIT}|${COMMIT_MESSAGE}|${COMMIT_AUTHOR}|deploying" >> "$HISTORY_FILE"
|
||||||
|
tail -n "$HISTORY_SIZE" "$HISTORY_FILE" > "${HISTORY_FILE}.tmp" && mv "${HISTORY_FILE}.tmp" "$HISTORY_FILE"
|
||||||
|
|
||||||
# Добавляем запись о начале деплоя с блокировкой
|
# Обновляем код
|
||||||
safe_write_history "${TIMESTAMP}|${CURRENT_COMMIT}|${COMMIT_MESSAGE}|${COMMIT_AUTHOR}|deploying"
|
|
||||||
|
|
||||||
# Обновляем код из main
|
|
||||||
echo "📥 Pulling latest changes from main..."
|
echo "📥 Pulling latest changes from main..."
|
||||||
|
sudo chown -R deploy:deploy /home/prod/bots || true
|
||||||
# Исправляем права на файлы в bots директории перед обновлением
|
|
||||||
fix_bots_permissions() {
|
|
||||||
local bots_dir="/home/prod/bots"
|
|
||||||
|
|
||||||
if [ ! -d "$bots_dir" ]; then
|
|
||||||
echo "⚠️ Bots directory not found, skipping permissions fix"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "🔧 Fixing permissions for bots directory..."
|
|
||||||
sudo chown -R deploy:deploy "$bots_dir" || true
|
|
||||||
echo "✅ Permissions fixed"
|
|
||||||
}
|
|
||||||
|
|
||||||
fix_bots_permissions
|
|
||||||
|
|
||||||
# Проверяем наличие локальных изменений перед reset
|
|
||||||
check_local_changes() {
|
|
||||||
echo "🔍 Checking for local changes..."
|
|
||||||
|
|
||||||
# Сохраняем текущее состояние
|
|
||||||
git fetch origin main
|
|
||||||
|
|
||||||
# Проверяем, есть ли локальные изменения
|
|
||||||
if ! git diff --quiet HEAD origin/main 2>/dev/null; then
|
|
||||||
echo "⚠️ Local changes detected! They will be overwritten by git reset --hard"
|
|
||||||
echo "📋 Diff summary:"
|
|
||||||
git diff --stat HEAD origin/main || true
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Проверяем uncommitted changes
|
|
||||||
if ! git diff --quiet 2>/dev/null || ! git diff --cached --quiet 2>/dev/null; then
|
|
||||||
echo "⚠️ Uncommitted changes detected! They will be lost."
|
|
||||||
git status --short || true
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "✅ Proceeding with git reset --hard"
|
|
||||||
}
|
|
||||||
|
|
||||||
check_local_changes
|
|
||||||
|
|
||||||
git fetch origin main
|
git fetch origin main
|
||||||
git reset --hard origin/main
|
git reset --hard origin/main
|
||||||
|
sudo chown -R deploy:deploy /home/prod/bots || true
|
||||||
|
|
||||||
# Исправляем права на файлы в bots директории после обновления
|
|
||||||
fix_bots_permissions
|
|
||||||
|
|
||||||
# Проверяем, что изменения есть
|
|
||||||
NEW_COMMIT=$(git rev-parse HEAD)
|
NEW_COMMIT=$(git rev-parse HEAD)
|
||||||
if [ "$CURRENT_COMMIT" = "$NEW_COMMIT" ]; then
|
|
||||||
echo "ℹ️ No new changes to deploy"
|
|
||||||
else
|
|
||||||
echo "✅ Code updated: $CURRENT_COMMIT → $NEW_COMMIT"
|
echo "✅ Code updated: $CURRENT_COMMIT → $NEW_COMMIT"
|
||||||
fi
|
|
||||||
|
|
||||||
# Проверяем docker-compose файл
|
|
||||||
validate_docker_compose() {
|
|
||||||
local compose_file="docker-compose.yml"
|
|
||||||
|
|
||||||
|
# Валидация docker-compose
|
||||||
echo "🔍 Validating docker-compose configuration..."
|
echo "🔍 Validating docker-compose configuration..."
|
||||||
|
docker-compose config > /dev/null || exit 1
|
||||||
if [ ! -f "$compose_file" ]; then
|
|
||||||
echo "❌ $compose_file not found!"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if ! docker-compose config > /dev/null 2>&1; then
|
|
||||||
echo "❌ Invalid docker-compose.yml syntax!"
|
|
||||||
docker-compose config # Показываем ошибки
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "✅ docker-compose.yml is valid"
|
echo "✅ docker-compose.yml is valid"
|
||||||
}
|
|
||||||
|
|
||||||
validate_docker_compose
|
# Проверка дискового пространства
|
||||||
|
MIN_FREE_GB=5
|
||||||
|
AVAILABLE_SPACE=$(df -BG /home/prod 2>/dev/null | tail -1 | awk '{print $4}' | sed 's/G//' || echo "0")
|
||||||
|
echo "💾 Available disk space: ${AVAILABLE_SPACE}GB"
|
||||||
|
|
||||||
# Проверяем дисковое пространство перед сборкой
|
if [ "$AVAILABLE_SPACE" -lt "$MIN_FREE_GB" ]; then
|
||||||
check_disk_space() {
|
echo "⚠️ Insufficient disk space! Cleaning up Docker resources..."
|
||||||
local min_free_gb=5
|
|
||||||
local available_space=$(df -BG /home/prod 2>/dev/null | tail -1 | awk '{print $4}' | sed 's/G//' || echo "0")
|
|
||||||
|
|
||||||
echo "💾 Checking disk space..."
|
|
||||||
echo "Available space: ${available_space}GB"
|
|
||||||
|
|
||||||
if [ "$available_space" -lt "$min_free_gb" ]; then
|
|
||||||
echo "⚠️ Insufficient disk space! Need at least ${min_free_gb}GB, but only ${available_space}GB available"
|
|
||||||
echo "🧹 Attempting to clean up unused Docker resources..."
|
|
||||||
docker system prune -f --volumes || true
|
docker system prune -f --volumes || true
|
||||||
|
|
||||||
# Проверяем снова после очистки
|
|
||||||
available_space=$(df -BG /home/prod 2>/dev/null | tail -1 | awk '{print $4}' | sed 's/G//' || echo "0")
|
|
||||||
echo "Available space after cleanup: ${available_space}GB"
|
|
||||||
|
|
||||||
if [ "$available_space" -lt "$min_free_gb" ]; then
|
|
||||||
echo "❌ Still insufficient disk space after cleanup!"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "✅ Sufficient disk space available"
|
# Сборка и запуск контейнеров (кроме ботов для ускорения деплоя)
|
||||||
}
|
echo "🔨 Rebuilding infrastructure containers (excluding bots)..."
|
||||||
|
docker-compose stop prometheus grafana uptime-kuma alertmanager || true
|
||||||
|
|
||||||
# Проверяем доступность памяти и CPU (опционально)
|
export TELEGRAM_BOT_TOKEN TELEGRAM_TEST_BOT_TOKEN ANON_BOT_TOKEN
|
||||||
check_resources() {
|
docker-compose build --pull prometheus grafana uptime-kuma alertmanager
|
||||||
echo "💻 Checking system resources..."
|
docker-compose up -d prometheus grafana uptime-kuma alertmanager
|
||||||
|
|
||||||
# Проверяем доступную память (в MB)
|
echo "✅ Infrastructure containers rebuilt and started (bots remain running)"
|
||||||
available_mem=$(free -m 2>/dev/null | awk '/^Mem:/ {print $7}' || echo "0")
|
|
||||||
min_mem_mb=512
|
|
||||||
|
|
||||||
if [ "$available_mem" -lt "$min_mem_mb" ] && [ "$available_mem" -gt 0 ]; then
|
|
||||||
echo "⚠️ Low available memory: ${available_mem}MB (recommended: ${min_mem_mb}MB+)"
|
|
||||||
else
|
|
||||||
echo "✅ Available memory: ${available_mem}MB"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Проверяем загрузку CPU (опционально)
|
|
||||||
load_avg=$(uptime 2>/dev/null | awk -F'load average:' '{print $2}' | awk '{print $1}' | sed 's/,//' || echo "0")
|
|
||||||
echo "📊 System load average: ${load_avg}"
|
|
||||||
}
|
|
||||||
|
|
||||||
check_disk_space
|
|
||||||
check_resources
|
|
||||||
|
|
||||||
# Пересобираем все контейнеры с обновлением базовых образов и кешированием
|
|
||||||
echo "🔨 Rebuilding all containers with --pull (updating base images, using cache)..."
|
|
||||||
cd /home/prod
|
|
||||||
|
|
||||||
# Останавливаем все контейнеры с graceful shutdown (30 секунд на остановку)
|
|
||||||
echo "🛑 Stopping containers gracefully..."
|
|
||||||
docker-compose down -t 30 || true
|
|
||||||
|
|
||||||
# Пересобираем все контейнеры с --pull (обновляет базовые образы, использует кеш слоев)
|
|
||||||
# Передаем токены из GitHub Secrets через переменные окружения
|
|
||||||
TELEGRAM_BOT_TOKEN="$TELEGRAM_BOT_TOKEN" \
|
|
||||||
TELEGRAM_TEST_BOT_TOKEN="$TELEGRAM_TEST_BOT_TOKEN" \
|
|
||||||
ANON_BOT_TOKEN="$ANON_BOT_TOKEN" \
|
|
||||||
docker-compose build --pull
|
|
||||||
|
|
||||||
# Запускаем все контейнеры с токенами из Secrets
|
|
||||||
echo "🚀 Starting all containers..."
|
|
||||||
TELEGRAM_BOT_TOKEN="$TELEGRAM_BOT_TOKEN" \
|
|
||||||
TELEGRAM_TEST_BOT_TOKEN="$TELEGRAM_TEST_BOT_TOKEN" \
|
|
||||||
ANON_BOT_TOKEN="$ANON_BOT_TOKEN" \
|
|
||||||
docker-compose up -d
|
|
||||||
|
|
||||||
echo "✅ Containers rebuilt and started"
|
|
||||||
|
|
||||||
- name: Update deploy history
|
- name: Update deploy history
|
||||||
if: always()
|
if: always()
|
||||||
@@ -351,52 +113,16 @@ jobs:
|
|||||||
key: ${{ secrets.SSH_PRIVATE_KEY }}
|
key: ${{ secrets.SSH_PRIVATE_KEY }}
|
||||||
port: ${{ vars.SSH_PORT || secrets.SSH_PORT || 22 }}
|
port: ${{ vars.SSH_PORT || secrets.SSH_PORT || 22 }}
|
||||||
script: |
|
script: |
|
||||||
# Функция для безопасной записи в историю деплоев с использованием flock
|
HISTORY_FILE="/home/prod/.deploy_history.txt"
|
||||||
# С fallback для файловых систем, которые не поддерживают flock (например, NFS)
|
|
||||||
safe_update_history_status() {
|
|
||||||
local new_status="$1"
|
|
||||||
local history_file="/home/prod/.deploy_history.txt"
|
|
||||||
local lock_file="${history_file}.lock"
|
|
||||||
|
|
||||||
if command -v flock > /dev/null 2>&1; then
|
if [ -f "$HISTORY_FILE" ]; then
|
||||||
# Используем flock напрямую с файлом (работает в zsh и bash)
|
DEPLOY_STATUS="failed"
|
||||||
flock -x "$lock_file" sh -c "
|
|
||||||
if [ -f \"$history_file\" ]; then
|
|
||||||
# Заменяем последнюю строку со статусом deploying на финальный статус
|
|
||||||
sed -i '\$s/|deploying\$/|$new_status/' \"$history_file\"
|
|
||||||
echo \"✅ Deploy history updated with status: $new_status (with flock)\"
|
|
||||||
else
|
|
||||||
echo \"⚠️ History file not found, skipping update\"
|
|
||||||
fi
|
|
||||||
" || {
|
|
||||||
echo "⚠️ Failed to acquire lock, using fallback method"
|
|
||||||
# Fallback: простая запись без блокировки
|
|
||||||
if [ -f "$history_file" ]; then
|
|
||||||
sed -i '$s/|deploying$/|'"$new_status"'/' "$history_file"
|
|
||||||
echo "✅ Deploy history updated (fallback method)"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
else
|
|
||||||
# Fallback: если flock недоступен
|
|
||||||
echo "⚠️ flock not available, using simple update"
|
|
||||||
if [ -f "$history_file" ]; then
|
|
||||||
sed -i '$s/|deploying$/|'"$new_status"'/' "$history_file"
|
|
||||||
echo "✅ Deploy history updated (simple method)"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
DEPLOY_HISTORY="/home/prod/.deploy_history.txt"
|
|
||||||
|
|
||||||
if [ -f "$DEPLOY_HISTORY" ]; then
|
|
||||||
# Обновляем последнюю запись со статусом deploying на success или failed
|
|
||||||
deploy_status="failed"
|
|
||||||
if [ "${{ job.status }}" = "success" ]; then
|
if [ "${{ job.status }}" = "success" ]; then
|
||||||
deploy_status="success"
|
DEPLOY_STATUS="success"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Обновляем статус безопасно
|
sed -i '$s/|deploying$/|'"$DEPLOY_STATUS"'/' "$HISTORY_FILE"
|
||||||
safe_update_history_status "$deploy_status"
|
echo "✅ Deploy history updated: $DEPLOY_STATUS"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
- name: Send deployment notification
|
- name: Send deployment notification
|
||||||
@@ -419,118 +145,12 @@ jobs:
|
|||||||
🔗 View details: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
🔗 View details: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||||
continue-on-error: true
|
continue-on-error: true
|
||||||
|
|
||||||
smoke-tests:
|
rollback:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
name: Smoke Tests
|
name: Rollback to Previous Version
|
||||||
needs: deploy
|
|
||||||
if: |
|
if: |
|
||||||
always() &&
|
github.event_name == 'workflow_dispatch' &&
|
||||||
needs.deploy.result == 'success'
|
github.event.inputs.action == 'rollback'
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: Run Smoke Tests
|
|
||||||
uses: appleboy/ssh-action@v1.0.0
|
|
||||||
with:
|
|
||||||
host: ${{ vars.SERVER_HOST || secrets.SERVER_HOST }}
|
|
||||||
username: ${{ vars.SERVER_USER || secrets.SERVER_USER }}
|
|
||||||
key: ${{ secrets.SSH_PRIVATE_KEY }}
|
|
||||||
port: ${{ vars.SSH_PORT || secrets.SSH_PORT || 22 }}
|
|
||||||
script: |
|
|
||||||
set -e
|
|
||||||
export TELEGRAM_BOT_TOKEN="${{ secrets.TELEGRAM_BOT_TOKEN }}"
|
|
||||||
export ANON_BOT_TOKEN="${{ secrets.ANON_BOT_TOKEN }}"
|
|
||||||
echo "🧪 Running smoke tests..."
|
|
||||||
|
|
||||||
SMOKE_TEST_CHAT_ID="${SMOKE_TEST_CHAT_ID:--898316252}"
|
|
||||||
echo "📝 Using test chat ID: $SMOKE_TEST_CHAT_ID"
|
|
||||||
|
|
||||||
# Проверка health endpoints
|
|
||||||
echo "🔍 Checking health endpoints..."
|
|
||||||
|
|
||||||
if ! curl -f -s --max-time 10 "http://localhost:8080/health" > /dev/null 2>&1; then
|
|
||||||
echo "❌ Telegram Bot health endpoint failed"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
echo "✅ Telegram Bot health endpoint OK"
|
|
||||||
|
|
||||||
if ! curl -f -s --max-time 10 "http://localhost:8081/health" > /dev/null 2>&1; then
|
|
||||||
echo "❌ AnonBot health endpoint failed"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
echo "✅ AnonBot health endpoint OK"
|
|
||||||
|
|
||||||
# Проверка метрик (опционально)
|
|
||||||
echo "🔍 Checking metrics endpoints..."
|
|
||||||
curl -f -s --max-time 10 "http://localhost:8080/metrics" > /dev/null 2>&1 && echo "✅ Telegram Bot metrics OK" || echo "⚠️ Telegram Bot metrics not available"
|
|
||||||
curl -f -s --max-time 10 "http://localhost:8081/metrics" > /dev/null 2>&1 && echo "✅ AnonBot metrics OK" || echo "⚠️ AnonBot metrics not available"
|
|
||||||
|
|
||||||
# Smoke-тест Telegram Helper Bot (используем токен из GitHub Secrets)
|
|
||||||
echo "🔍 Testing Telegram Helper Bot..."
|
|
||||||
if [ -z "$TELEGRAM_BOT_TOKEN" ]; then
|
|
||||||
echo "❌ TELEGRAM_BOT_TOKEN not found in GitHub Secrets"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Отправляем сообщение "ping" в тестовый чат
|
|
||||||
response=$(curl -s --max-time 30 -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
|
|
||||||
-d "chat_id=${SMOKE_TEST_CHAT_ID}" \
|
|
||||||
-d "text=ping" || echo "")
|
|
||||||
|
|
||||||
if echo "$response" | grep -q '"ok":true'; then
|
|
||||||
echo "✅ Telegram Helper Bot smoke test passed (message sent successfully)"
|
|
||||||
else
|
|
||||||
echo "❌ Telegram Helper Bot smoke test failed"
|
|
||||||
echo "Response: $response"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Smoke-тест AnonBot (используем токен из GitHub Secrets)
|
|
||||||
echo "🔍 Testing AnonBot..."
|
|
||||||
if [ -n "$ANON_BOT_TOKEN" ]; then
|
|
||||||
response=$(curl -s --max-time 30 -X POST "https://api.telegram.org/bot${ANON_BOT_TOKEN}/sendMessage" \
|
|
||||||
-d "chat_id=${SMOKE_TEST_CHAT_ID}" \
|
|
||||||
-d "text=ping" || echo "")
|
|
||||||
|
|
||||||
if echo "$response" | grep -q '"ok":true'; then
|
|
||||||
echo "✅ AnonBot smoke test passed (message sent successfully)"
|
|
||||||
else
|
|
||||||
echo "⚠️ AnonBot smoke test failed (non-critical)"
|
|
||||||
echo "Response: $response"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
echo "ℹ️ ANON_BOT_TOKEN not set, skipping smoke test"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "✅ All smoke tests passed!"
|
|
||||||
|
|
||||||
- name: Send smoke tests notification
|
|
||||||
if: always()
|
|
||||||
uses: appleboy/telegram-action@v1.0.0
|
|
||||||
with:
|
|
||||||
to: ${{ secrets.TELEGRAM_CHAT_ID }}
|
|
||||||
token: ${{ secrets.TELEGRAM_BOT_TOKEN }}
|
|
||||||
message: |
|
|
||||||
${{ job.status == 'success' && '✅' || '❌' }} Smoke Tests: ${{ job.status }}
|
|
||||||
|
|
||||||
📦 Repository: prod
|
|
||||||
🌿 Branch: main
|
|
||||||
📝 Commit: ${{ github.event.pull_request.merge_commit_sha || github.sha }}
|
|
||||||
|
|
||||||
${{ job.status == 'success' && '✅ All smoke tests passed! Bots are working correctly.' || '❌ Smoke tests failed! Auto-rollback will be triggered.' }}
|
|
||||||
|
|
||||||
🔗 View details: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
||||||
continue-on-error: true
|
|
||||||
|
|
||||||
auto-rollback:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
name: Auto Rollback
|
|
||||||
concurrency:
|
|
||||||
group: production-rollback
|
|
||||||
cancel-in-progress: false
|
|
||||||
needs: [deploy, smoke-tests]
|
|
||||||
if: |
|
|
||||||
always() &&
|
|
||||||
needs.smoke-tests.result == 'failure'
|
|
||||||
environment:
|
environment:
|
||||||
name: production
|
name: production
|
||||||
|
|
||||||
@@ -540,7 +160,7 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
ref: main
|
ref: main
|
||||||
|
|
||||||
- name: Auto Rollback
|
- name: Rollback on server
|
||||||
uses: appleboy/ssh-action@v1.0.0
|
uses: appleboy/ssh-action@v1.0.0
|
||||||
with:
|
with:
|
||||||
host: ${{ vars.SERVER_HOST || secrets.SERVER_HOST }}
|
host: ${{ vars.SERVER_HOST || secrets.SERVER_HOST }}
|
||||||
@@ -552,351 +172,87 @@ jobs:
|
|||||||
export TELEGRAM_BOT_TOKEN="${{ secrets.TELEGRAM_BOT_TOKEN }}"
|
export TELEGRAM_BOT_TOKEN="${{ secrets.TELEGRAM_BOT_TOKEN }}"
|
||||||
export TELEGRAM_TEST_BOT_TOKEN="${{ secrets.TELEGRAM_TEST_BOT_TOKEN }}"
|
export TELEGRAM_TEST_BOT_TOKEN="${{ secrets.TELEGRAM_TEST_BOT_TOKEN }}"
|
||||||
export ANON_BOT_TOKEN="${{ secrets.ANON_BOT_TOKEN }}"
|
export ANON_BOT_TOKEN="${{ secrets.ANON_BOT_TOKEN }}"
|
||||||
echo "🔄 Starting automatic rollback after smoke tests failure..."
|
|
||||||
|
|
||||||
# Функция для безопасного чтения истории деплоев с использованием flock
|
echo "🔄 Starting rollback..."
|
||||||
# С fallback для файловых систем, которые не поддерживают flock (например, NFS)
|
|
||||||
safe_read_history() {
|
|
||||||
local history_file="/home/prod/.deploy_history.txt"
|
|
||||||
local lock_file="${history_file}.lock"
|
|
||||||
|
|
||||||
if command -v flock > /dev/null 2>&1; then
|
|
||||||
# Используем flock напрямую с файлом для чтения
|
|
||||||
flock -s "$lock_file" sh -c "
|
|
||||||
if [ -f \"$history_file\" ]; then
|
|
||||||
cat \"$history_file\"
|
|
||||||
else
|
|
||||||
echo \"\"
|
|
||||||
fi
|
|
||||||
" || {
|
|
||||||
echo "⚠️ Failed to acquire lock, using fallback method"
|
|
||||||
# Fallback: простое чтение без блокировки
|
|
||||||
if [ -f "$history_file" ]; then
|
|
||||||
cat "$history_file"
|
|
||||||
else
|
|
||||||
echo ""
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
else
|
|
||||||
# Fallback: если flock недоступен
|
|
||||||
if [ -f "$history_file" ]; then
|
|
||||||
cat "$history_file"
|
|
||||||
else
|
|
||||||
echo ""
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# Функция для безопасной записи в историю деплоев с использованием flock
|
|
||||||
safe_write_history() {
|
|
||||||
local entry="$1"
|
|
||||||
local history_file="/home/prod/.deploy_history.txt"
|
|
||||||
local lock_file="${history_file}.lock"
|
|
||||||
local history_size="${DEPLOY_HISTORY_SIZE:-10}"
|
|
||||||
|
|
||||||
if command -v flock > /dev/null 2>&1; then
|
|
||||||
# Используем flock напрямую с файлом (работает в zsh и bash)
|
|
||||||
flock -x "$lock_file" sh -c "
|
|
||||||
# Записываем новую запись
|
|
||||||
echo \"$entry\" >> \"$history_file\"
|
|
||||||
|
|
||||||
# Обрезаем файл атомарно
|
|
||||||
tail -n $history_size \"$history_file\" > \"${history_file}.tmp\"
|
|
||||||
mv \"${history_file}.tmp\" \"$history_file\"
|
|
||||||
|
|
||||||
echo \"✅ History updated safely\"
|
|
||||||
"
|
|
||||||
else
|
|
||||||
# Fallback: простая запись без блокировки
|
|
||||||
echo "$entry" >> "$history_file"
|
|
||||||
tail -n "$history_size" "$history_file" > "${history_file}.tmp"
|
|
||||||
mv "${history_file}.tmp" "$history_file"
|
|
||||||
echo "✅ History updated (fallback method)"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# Функция для безопасного изменения прав на bots директорию
|
|
||||||
fix_bots_permissions() {
|
|
||||||
local bots_dir="/home/prod/bots"
|
|
||||||
|
|
||||||
if [ ! -d "$bots_dir" ]; then
|
|
||||||
echo "⚠️ Bots directory not found, skipping permissions fix"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "🔧 Fixing permissions for bots directory..."
|
|
||||||
sudo chown -R deploy:deploy "$bots_dir" || true
|
|
||||||
echo "✅ Permissions fixed"
|
|
||||||
}
|
|
||||||
|
|
||||||
cd /home/prod
|
cd /home/prod
|
||||||
DEPLOY_HISTORY="/home/prod/.deploy_history.txt"
|
|
||||||
DEPLOY_HISTORY_SIZE="${DEPLOY_HISTORY_SIZE:-10}"
|
|
||||||
|
|
||||||
# Находим последний успешный деплой из истории (безопасно)
|
# Определяем коммит для отката
|
||||||
HISTORY_CONTENT=$(safe_read_history)
|
ROLLBACK_COMMIT="${{ github.event.inputs.rollback_commit }}"
|
||||||
LAST_SUCCESSFUL_COMMIT=$(echo "$HISTORY_CONTENT" | grep "|success" | tail -1 | cut -d'|' -f2 || echo "")
|
HISTORY_FILE="/home/prod/.deploy_history.txt"
|
||||||
|
|
||||||
# Если нет успешного деплоя в истории, используем сохраненный коммит
|
if [ -z "$ROLLBACK_COMMIT" ]; then
|
||||||
if [ -z "$LAST_SUCCESSFUL_COMMIT" ]; then
|
echo "📝 No commit specified, finding last successful deploy..."
|
||||||
if [ -f "/tmp/last_deploy_commit.txt" ]; then
|
if [ -f "$HISTORY_FILE" ]; then
|
||||||
LAST_SUCCESSFUL_COMMIT=$(cat /tmp/last_deploy_commit.txt)
|
ROLLBACK_COMMIT=$(grep "|success$" "$HISTORY_FILE" | tail -1 | cut -d'|' -f2 || echo "")
|
||||||
echo "📝 Using saved commit from /tmp/last_deploy_commit.txt: $LAST_SUCCESSFUL_COMMIT"
|
fi
|
||||||
else
|
|
||||||
echo "❌ No previous successful deploy found in history and no saved commit!"
|
if [ -z "$ROLLBACK_COMMIT" ]; then
|
||||||
|
echo "❌ No successful deploy found in history!"
|
||||||
|
echo "💡 Please specify commit hash manually or check deploy history"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
else
|
|
||||||
echo "📝 Found last successful deploy in history: $LAST_SUCCESSFUL_COMMIT"
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
echo "📝 Rolling back to commit: $ROLLBACK_COMMIT"
|
||||||
|
|
||||||
|
# Проверяем, что коммит существует
|
||||||
|
if ! git cat-file -e "$ROLLBACK_COMMIT" 2>/dev/null; then
|
||||||
|
echo "❌ Commit $ROLLBACK_COMMIT not found!"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Сохраняем текущий коммит
|
||||||
|
CURRENT_COMMIT=$(git rev-parse HEAD)
|
||||||
|
COMMIT_MESSAGE=$(git log -1 --pretty=format:"%s" "$ROLLBACK_COMMIT" || echo "Rollback")
|
||||||
|
TIMESTAMP=$(date +"%Y-%m-%d %H:%M:%S")
|
||||||
|
|
||||||
|
echo "📝 Current commit: $CURRENT_COMMIT"
|
||||||
|
echo "📝 Target commit: $ROLLBACK_COMMIT"
|
||||||
|
echo "📝 Commit message: $COMMIT_MESSAGE"
|
||||||
|
|
||||||
|
# Исправляем права перед откатом
|
||||||
|
sudo chown -R deploy:deploy /home/prod/bots || true
|
||||||
|
|
||||||
# Откатываем код
|
# Откатываем код
|
||||||
echo "🔄 Rolling back to commit: $LAST_SUCCESSFUL_COMMIT"
|
echo "🔄 Rolling back code..."
|
||||||
|
|
||||||
# Исправляем права на файлы в bots директории
|
|
||||||
fix_bots_permissions
|
|
||||||
|
|
||||||
# Проверяем наличие локальных изменений перед reset
|
|
||||||
echo "🔍 Checking for local changes before rollback..."
|
|
||||||
git fetch origin main
|
git fetch origin main
|
||||||
|
git reset --hard "$ROLLBACK_COMMIT"
|
||||||
|
|
||||||
# Проверяем uncommitted changes
|
# Исправляем права после отката
|
||||||
if ! git diff --quiet 2>/dev/null || ! git diff --cached --quiet 2>/dev/null; then
|
sudo chown -R deploy:deploy /home/prod/bots || true
|
||||||
echo "⚠️ Uncommitted changes detected! They will be lost during rollback."
|
|
||||||
git status --short || true
|
|
||||||
fi
|
|
||||||
|
|
||||||
git fetch origin main
|
echo "✅ Code rolled back: $CURRENT_COMMIT → $ROLLBACK_COMMIT"
|
||||||
git reset --hard "$LAST_SUCCESSFUL_COMMIT"
|
|
||||||
|
|
||||||
# Устанавливаем правильные права после отката
|
|
||||||
fix_bots_permissions
|
|
||||||
|
|
||||||
echo "✅ Code rolled back to: $LAST_SUCCESSFUL_COMMIT"
|
|
||||||
|
|
||||||
# Проверяем docker-compose файл
|
|
||||||
validate_docker_compose() {
|
|
||||||
local compose_file="docker-compose.yml"
|
|
||||||
|
|
||||||
|
# Валидация docker-compose
|
||||||
echo "🔍 Validating docker-compose configuration..."
|
echo "🔍 Validating docker-compose configuration..."
|
||||||
|
docker-compose config > /dev/null || exit 1
|
||||||
if [ ! -f "$compose_file" ]; then
|
|
||||||
echo "❌ $compose_file not found!"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if ! docker-compose config > /dev/null 2>&1; then
|
|
||||||
echo "❌ Invalid docker-compose.yml syntax!"
|
|
||||||
docker-compose config # Показываем ошибки
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "✅ docker-compose.yml is valid"
|
echo "✅ docker-compose.yml is valid"
|
||||||
}
|
|
||||||
|
|
||||||
validate_docker_compose
|
# Проверка дискового пространства
|
||||||
|
MIN_FREE_GB=5
|
||||||
|
AVAILABLE_SPACE=$(df -BG /home/prod 2>/dev/null | tail -1 | awk '{print $4}' | sed 's/G//' || echo "0")
|
||||||
|
echo "💾 Available disk space: ${AVAILABLE_SPACE}GB"
|
||||||
|
|
||||||
# Проверяем дисковое пространство перед сборкой
|
if [ "$AVAILABLE_SPACE" -lt "$MIN_FREE_GB" ]; then
|
||||||
check_disk_space() {
|
echo "⚠️ Insufficient disk space! Cleaning up Docker resources..."
|
||||||
local min_free_gb=5
|
|
||||||
local available_space=$(df -BG /home/prod 2>/dev/null | tail -1 | awk '{print $4}' | sed 's/G//' || echo "0")
|
|
||||||
|
|
||||||
echo "💾 Checking disk space..."
|
|
||||||
echo "Available space: ${available_space}GB"
|
|
||||||
|
|
||||||
if [ "$available_space" -lt "$min_free_gb" ]; then
|
|
||||||
echo "⚠️ Insufficient disk space! Need at least ${min_free_gb}GB, but only ${available_space}GB available"
|
|
||||||
echo "🧹 Attempting to clean up unused Docker resources..."
|
|
||||||
docker system prune -f --volumes || true
|
docker system prune -f --volumes || true
|
||||||
|
|
||||||
# Проверяем снова после очистки
|
|
||||||
available_space=$(df -BG /home/prod 2>/dev/null | tail -1 | awk '{print $4}' | sed 's/G//' || echo "0")
|
|
||||||
echo "Available space after cleanup: ${available_space}GB"
|
|
||||||
|
|
||||||
if [ "$available_space" -lt "$min_free_gb" ]; then
|
|
||||||
echo "❌ Still insufficient disk space after cleanup!"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "✅ Sufficient disk space available"
|
# Пересобираем и запускаем контейнеры (кроме ботов для ускорения отката)
|
||||||
}
|
echo "🔨 Rebuilding infrastructure containers (excluding bots)..."
|
||||||
|
docker-compose stop prometheus grafana uptime-kuma alertmanager || true
|
||||||
|
|
||||||
# Проверяем доступность памяти и CPU (опционально)
|
export TELEGRAM_BOT_TOKEN TELEGRAM_TEST_BOT_TOKEN ANON_BOT_TOKEN
|
||||||
check_resources() {
|
docker-compose build --pull prometheus grafana uptime-kuma alertmanager
|
||||||
echo "💻 Checking system resources..."
|
docker-compose up -d prometheus grafana uptime-kuma alertmanager
|
||||||
|
|
||||||
# Проверяем доступную память (в MB)
|
echo "✅ Infrastructure containers rebuilt and started (bots remain running)"
|
||||||
available_mem=$(free -m 2>/dev/null | awk '/^Mem:/ {print $7}' || echo "0")
|
|
||||||
min_mem_mb=512
|
|
||||||
|
|
||||||
if [ "$available_mem" -lt "$min_mem_mb" ] && [ "$available_mem" -gt 0 ]; then
|
# Записываем в историю
|
||||||
echo "⚠️ Low available memory: ${available_mem}MB (recommended: ${min_mem_mb}MB+)"
|
echo "${TIMESTAMP}|${ROLLBACK_COMMIT}|Rollback to: ${COMMIT_MESSAGE}|github-actions|rolled_back" >> "$HISTORY_FILE"
|
||||||
else
|
HISTORY_SIZE="${DEPLOY_HISTORY_SIZE:-10}"
|
||||||
echo "✅ Available memory: ${available_mem}MB"
|
tail -n "$HISTORY_SIZE" "$HISTORY_FILE" > "${HISTORY_FILE}.tmp" && mv "${HISTORY_FILE}.tmp" "$HISTORY_FILE"
|
||||||
fi
|
|
||||||
|
|
||||||
# Проверяем загрузку CPU (опционально)
|
|
||||||
load_avg=$(uptime 2>/dev/null | awk -F'load average:' '{print $2}' | awk '{print $1}' | sed 's/,//' || echo "0")
|
|
||||||
echo "📊 System load average: ${load_avg}"
|
|
||||||
}
|
|
||||||
|
|
||||||
check_disk_space
|
|
||||||
check_resources
|
|
||||||
|
|
||||||
# Пересобираем все контейнеры с обновлением базовых образов и кешированием
|
|
||||||
echo "🔨 Rebuilding all containers with --pull (updating base images, using cache)..."
|
|
||||||
|
|
||||||
# Останавливаем все контейнеры с graceful shutdown (30 секунд на остановку)
|
|
||||||
echo "🛑 Stopping containers gracefully..."
|
|
||||||
docker-compose down -t 30 || true
|
|
||||||
|
|
||||||
# Пересобираем с токенами из Secrets
|
|
||||||
TELEGRAM_BOT_TOKEN="$TELEGRAM_BOT_TOKEN" \
|
|
||||||
TELEGRAM_TEST_BOT_TOKEN="$TELEGRAM_TEST_BOT_TOKEN" \
|
|
||||||
ANON_BOT_TOKEN="$ANON_BOT_TOKEN" \
|
|
||||||
docker-compose build --pull
|
|
||||||
|
|
||||||
# Запускаем с токенами из Secrets
|
|
||||||
echo "🚀 Starting all containers..."
|
|
||||||
TELEGRAM_BOT_TOKEN="$TELEGRAM_BOT_TOKEN" \
|
|
||||||
TELEGRAM_TEST_BOT_TOKEN="$TELEGRAM_TEST_BOT_TOKEN" \
|
|
||||||
ANON_BOT_TOKEN="$ANON_BOT_TOKEN" \
|
|
||||||
docker-compose up -d
|
|
||||||
|
|
||||||
echo "✅ Containers rebuilt and started"
|
|
||||||
|
|
||||||
# Проверяем доступность сети
|
|
||||||
check_network_availability() {
|
|
||||||
echo "🔍 Checking network availability..."
|
|
||||||
|
|
||||||
if ! ping -c 1 -W 2 localhost > /dev/null 2>&1; then
|
|
||||||
echo "❌ Localhost not reachable! Network issue detected."
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "✅ Network is available"
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
# Адаптивное ожидание готовности контейнеров
|
|
||||||
wait_for_containers_ready() {
|
|
||||||
local max_wait=180 # 3 минуты максимум
|
|
||||||
local check_interval=5
|
|
||||||
local elapsed=0
|
|
||||||
|
|
||||||
echo "⏳ Waiting for containers to be ready..."
|
|
||||||
|
|
||||||
while [ $elapsed -lt $max_wait ]; do
|
|
||||||
if docker-compose ps 2>/dev/null | grep -q "Exit\|Restarting"; then
|
|
||||||
echo "⏳ Some containers not ready yet, waiting ${check_interval}s... (${elapsed}/${max_wait}s)"
|
|
||||||
sleep $check_interval
|
|
||||||
elapsed=$((elapsed + check_interval))
|
|
||||||
else
|
|
||||||
local running_count=$(docker-compose ps 2>/dev/null | grep -c "Up" || echo "0")
|
|
||||||
if [ "$running_count" -gt 0 ]; then
|
|
||||||
echo "✅ All containers are ready! (waited ${elapsed}s, ${running_count} containers running)"
|
|
||||||
return 0
|
|
||||||
else
|
|
||||||
echo "⏳ Waiting for containers to start... (${elapsed}/${max_wait}s)"
|
|
||||||
sleep $check_interval
|
|
||||||
elapsed=$((elapsed + check_interval))
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
echo "⚠️ Containers not fully ready after ${max_wait}s, but continuing with health checks..."
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
# Проверяем сеть перед health checks
|
|
||||||
if ! check_network_availability; then
|
|
||||||
echo "⚠️ Network check failed, but continuing with health checks..."
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Ждем готовности контейнеров адаптивно
|
|
||||||
wait_for_containers_ready
|
|
||||||
|
|
||||||
# Функция для проверки с экспоненциальным retry
|
|
||||||
check_health() {
|
|
||||||
local service=$1
|
|
||||||
local url=$2
|
|
||||||
local attempt=1
|
|
||||||
local delays=(5 15 45)
|
|
||||||
local max_attempts=${#delays[@]}
|
|
||||||
|
|
||||||
echo "🔍 Checking $service health..."
|
|
||||||
|
|
||||||
while [ $attempt -le $max_attempts ]; do
|
|
||||||
if curl -f -s --max-time 10 "$url" > /dev/null 2>&1; then
|
|
||||||
echo "✅ $service is healthy (attempt $attempt/$max_attempts)"
|
|
||||||
return 0
|
|
||||||
else
|
|
||||||
if [ $attempt -lt $max_attempts ]; then
|
|
||||||
delay=${delays[$((attempt - 1))]}
|
|
||||||
echo "⏳ $service not ready yet (attempt $attempt/$max_attempts), waiting ${delay} seconds..."
|
|
||||||
sleep $delay
|
|
||||||
else
|
|
||||||
echo "❌ $service health check failed after $max_attempts attempts"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
attempt=$((attempt + 1))
|
|
||||||
done
|
|
||||||
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
# Общая функция для проверки всех сервисов
|
|
||||||
run_health_checks() {
|
|
||||||
local failed=0
|
|
||||||
local services=(
|
|
||||||
"Prometheus:http://localhost:9090/-/healthy:prometheus"
|
|
||||||
"Grafana:http://localhost:3000/api/health:grafana"
|
|
||||||
"Telegram Bot:http://localhost:8080/health:telegram-bot"
|
|
||||||
"AnonBot:http://localhost:8081/health:anon-bot"
|
|
||||||
)
|
|
||||||
|
|
||||||
for service_info in "${services[@]}"; do
|
|
||||||
IFS=':' read -r service_name url container_name <<< "$service_info"
|
|
||||||
echo "🔍 Checking $service_name..."
|
|
||||||
if ! check_health "$service_name" "$url"; then
|
|
||||||
echo "⚠️ $service_name health check failed"
|
|
||||||
docker-compose logs --tail=30 "$container_name" || true
|
|
||||||
failed=1
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
return $failed
|
|
||||||
}
|
|
||||||
|
|
||||||
HEALTH_CHECK_FAILED=0
|
|
||||||
if ! run_health_checks; then
|
|
||||||
HEALTH_CHECK_FAILED=1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Проверяем статус всех контейнеров
|
|
||||||
echo "📊 Final container status:"
|
|
||||||
docker-compose ps
|
|
||||||
|
|
||||||
# Проверяем, что все контейнеры запущены
|
|
||||||
FAILED_CONTAINERS=$(docker-compose ps | grep -c "Exit\|Restarting" || true)
|
|
||||||
if [ "$FAILED_CONTAINERS" -gt 0 ]; then
|
|
||||||
echo "❌ Some containers are not running properly"
|
|
||||||
docker-compose ps
|
|
||||||
HEALTH_CHECK_FAILED=1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ $HEALTH_CHECK_FAILED -eq 1 ]; then
|
|
||||||
echo "⚠️ Some health checks failed, but rollback completed"
|
|
||||||
else
|
|
||||||
echo "✅ All health checks passed after rollback!"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Обновляем историю безопасно
|
|
||||||
TIMESTAMP=$(date +"%Y-%m-%d %H:%M:%S")
|
|
||||||
COMMIT_MESSAGE=$(git log -1 --pretty=format:"%s" "$LAST_SUCCESSFUL_COMMIT" 2>/dev/null || echo "Auto-rollback")
|
|
||||||
safe_write_history "${TIMESTAMP}|${LAST_SUCCESSFUL_COMMIT}|Auto-rollback after smoke tests failure|github-actions|rolled_back"
|
|
||||||
|
|
||||||
echo "✅ Rollback completed successfully"
|
echo "✅ Rollback completed successfully"
|
||||||
|
|
||||||
@@ -907,16 +263,14 @@ jobs:
|
|||||||
to: ${{ secrets.TELEGRAM_CHAT_ID }}
|
to: ${{ secrets.TELEGRAM_CHAT_ID }}
|
||||||
token: ${{ secrets.TELEGRAM_BOT_TOKEN }}
|
token: ${{ secrets.TELEGRAM_BOT_TOKEN }}
|
||||||
message: |
|
message: |
|
||||||
🔄 Automatic Rollback: ${{ job.status }}
|
${{ job.status == 'success' && '🔄' || '❌' }} Rollback: ${{ job.status }}
|
||||||
|
|
||||||
📦 Repository: prod
|
📦 Repository: prod
|
||||||
🌿 Branch: main
|
🌿 Branch: main
|
||||||
📝 Rolled back to previous successful commit
|
📝 Rolled back to: ${{ github.event.inputs.rollback_commit || 'Last successful commit' }}
|
||||||
${{ github.event.pull_request.number && format('🔀 PR: #{0}', github.event.pull_request.number) || '' }}
|
👤 Triggered by: ${{ github.actor }}
|
||||||
|
|
||||||
⚠️ Rollback was triggered automatically due to smoke tests failure.
|
${{ job.status == 'success' && '✅ Rollback completed successfully! Services restored to previous version.' || '❌ Rollback failed! Check logs for details.' }}
|
||||||
|
|
||||||
${{ job.status == 'success' && '✅ Rollback completed successfully! Services restored to previous version.' || '❌ Rollback failed! Manual intervention required.' }}
|
|
||||||
|
|
||||||
🔗 View details: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
🔗 View details: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||||
continue-on-error: true
|
continue-on-error: true
|
||||||
|
|||||||
451
.github/workflows/pipeline.yml
vendored
451
.github/workflows/pipeline.yml
vendored
@@ -1,451 +0,0 @@
|
|||||||
name: CI & CD pipeline
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches: [ main, 'develop', 'dev-*', 'feature/**' ]
|
|
||||||
workflow_dispatch:
|
|
||||||
inputs:
|
|
||||||
action:
|
|
||||||
description: 'Action to perform'
|
|
||||||
required: true
|
|
||||||
type: choice
|
|
||||||
options:
|
|
||||||
- rollback
|
|
||||||
rollback_to_commit:
|
|
||||||
description: 'Commit hash to rollback to (optional, uses last deploy if empty)'
|
|
||||||
required: false
|
|
||||||
type: string
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
test:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
name: Test & Code Quality
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: Checkout code
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Set up Python 3.11
|
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version: '3.11'
|
|
||||||
cache: 'pip'
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
run: |
|
|
||||||
python -m pip install --upgrade pip
|
|
||||||
pip install -r tests/infra/requirements-test.txt
|
|
||||||
pip install flake8 black isort mypy || true
|
|
||||||
|
|
||||||
- name: Code formatting check (Black)
|
|
||||||
run: |
|
|
||||||
echo "🔍 Checking code formatting with Black..."
|
|
||||||
black --check . || (echo "❌ Code formatting issues found. Run 'black .' to fix." && exit 1)
|
|
||||||
|
|
||||||
- name: Import sorting check (isort)
|
|
||||||
run: |
|
|
||||||
echo "🔍 Checking import sorting with isort..."
|
|
||||||
isort --check-only . || (echo "❌ Import sorting issues found. Run 'isort .' to fix." && exit 1)
|
|
||||||
|
|
||||||
- name: Linting (flake8) - Critical errors
|
|
||||||
run: |
|
|
||||||
echo "🔍 Running flake8 linter (critical errors only)..."
|
|
||||||
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
|
|
||||||
|
|
||||||
- name: Linting (flake8) - Warnings
|
|
||||||
run: |
|
|
||||||
echo "🔍 Running flake8 linter (warnings)..."
|
|
||||||
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics || true
|
|
||||||
continue-on-error: true
|
|
||||||
|
|
||||||
- name: Run infrastructure tests
|
|
||||||
run: |
|
|
||||||
python -m pytest tests/infra/ -v --tb=short
|
|
||||||
|
|
||||||
- name: Validate Prometheus config
|
|
||||||
run: |
|
|
||||||
python -m pytest tests/infra/test_prometheus_config.py -v
|
|
||||||
|
|
||||||
|
|
||||||
- name: Upload test results
|
|
||||||
if: always()
|
|
||||||
uses: actions/upload-artifact@v4
|
|
||||||
with:
|
|
||||||
name: test-results
|
|
||||||
path: |
|
|
||||||
.pytest_cache/
|
|
||||||
htmlcov/
|
|
||||||
retention-days: 7
|
|
||||||
|
|
||||||
- name: Send test failure notification
|
|
||||||
if: failure()
|
|
||||||
uses: appleboy/telegram-action@v1.0.0
|
|
||||||
with:
|
|
||||||
to: ${{ secrets.TELEGRAM_CHAT_ID }}
|
|
||||||
token: ${{ secrets.TELEGRAM_BOT_TOKEN }}
|
|
||||||
message: |
|
|
||||||
❌ CI Tests Failed
|
|
||||||
|
|
||||||
📦 Repository: prod
|
|
||||||
🌿 Branch: ${{ github.ref_name }}
|
|
||||||
📝 Commit: ${{ github.sha }}
|
|
||||||
👤 Author: ${{ github.actor }}
|
|
||||||
|
|
||||||
❌ Tests failed! Deployment blocked. Please fix the issues and try again.
|
|
||||||
|
|
||||||
🔗 View details: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
||||||
continue-on-error: true
|
|
||||||
|
|
||||||
create-pr:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
name: Create Pull Request
|
|
||||||
needs: test
|
|
||||||
if: |
|
|
||||||
github.event_name == 'push' &&
|
|
||||||
needs.test.result == 'success' &&
|
|
||||||
github.ref_name != 'main' &&
|
|
||||||
github.ref_name != 'develop' &&
|
|
||||||
(startsWith(github.ref_name, 'dev-') || startsWith(github.ref_name, 'feature/'))
|
|
||||||
# Примечание: Для создания PR из той же ветки нужен PAT (Personal Access Token)
|
|
||||||
# Создайте PAT с правами repo и добавьте в Secrets как PAT
|
|
||||||
permissions:
|
|
||||||
contents: write
|
|
||||||
pull-requests: write
|
|
||||||
issues: write
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: Checkout code
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
fetch-depth: 0
|
|
||||||
token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }}
|
|
||||||
persist-credentials: true
|
|
||||||
|
|
||||||
- name: Check if PR already exists
|
|
||||||
id: check-pr
|
|
||||||
uses: actions/github-script@v6
|
|
||||||
with:
|
|
||||||
github-token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }}
|
|
||||||
script: |
|
|
||||||
const branchName = context.ref.replace('refs/heads/', '');
|
|
||||||
const { data: prs } = await github.rest.pulls.list({
|
|
||||||
owner: context.repo.owner,
|
|
||||||
repo: context.repo.repo,
|
|
||||||
head: context.repo.owner + ':' + branchName,
|
|
||||||
base: 'main',
|
|
||||||
state: 'open'
|
|
||||||
});
|
|
||||||
|
|
||||||
if (prs.length > 0) {
|
|
||||||
core.setOutput('exists', 'true');
|
|
||||||
core.setOutput('number', prs[0].number);
|
|
||||||
core.setOutput('url', prs[0].html_url);
|
|
||||||
} else {
|
|
||||||
core.setOutput('exists', 'false');
|
|
||||||
}
|
|
||||||
|
|
||||||
- name: Update existing PR
|
|
||||||
if: steps.check-pr.outputs.exists == 'true'
|
|
||||||
uses: actions/github-script@v6
|
|
||||||
with:
|
|
||||||
github-token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }}
|
|
||||||
script: |
|
|
||||||
const prNumber = parseInt('${{ steps.check-pr.outputs.number }}');
|
|
||||||
const branchName = context.ref.replace('refs/heads/', '');
|
|
||||||
|
|
||||||
const commitSha = '${{ github.sha }}';
|
|
||||||
const author = '${{ github.actor }}';
|
|
||||||
|
|
||||||
const updateBody = '## Updated Changes\n\n' +
|
|
||||||
'PR updated with new commits after successful CI tests.\n\n' +
|
|
||||||
'- Latest commit: ' + commitSha + '\n' +
|
|
||||||
'- Branch: `' + branchName + '`\n' +
|
|
||||||
'- Author: @' + author + '\n\n' +
|
|
||||||
'## Test Results\n\n' +
|
|
||||||
'✅ All tests passed successfully!\n\n' +
|
|
||||||
'Please review the changes and merge when ready.';
|
|
||||||
|
|
||||||
await github.rest.pulls.update({
|
|
||||||
owner: context.repo.owner,
|
|
||||||
repo: context.repo.repo,
|
|
||||||
pull_number: prNumber,
|
|
||||||
title: 'Merge ' + branchName + ' into main',
|
|
||||||
body: updateBody
|
|
||||||
});
|
|
||||||
|
|
||||||
console.log('✅ PR #' + prNumber + ' updated successfully');
|
|
||||||
|
|
||||||
- name: Create Pull Request
|
|
||||||
if: steps.check-pr.outputs.exists == 'false'
|
|
||||||
id: create-pr
|
|
||||||
uses: actions/github-script@v6
|
|
||||||
with:
|
|
||||||
github-token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }}
|
|
||||||
script: |
|
|
||||||
const branchName = context.ref.replace('refs/heads/', '');
|
|
||||||
|
|
||||||
console.log('📝 Creating PR from ' + branchName + ' to main...');
|
|
||||||
|
|
||||||
try {
|
|
||||||
const commitSha = '${{ github.sha }}';
|
|
||||||
const author = '${{ github.actor }}';
|
|
||||||
|
|
||||||
const prBody = '## Changes\n\n' +
|
|
||||||
'This PR was automatically created after successful CI tests.\n\n' +
|
|
||||||
'- Branch: `' + branchName + '`\n' +
|
|
||||||
'- Commit: `' + commitSha + '`\n' +
|
|
||||||
'- Author: @' + author + '\n\n' +
|
|
||||||
'## Test Results\n\n' +
|
|
||||||
'✅ All tests passed successfully!\n\n' +
|
|
||||||
'Please review the changes and merge when ready.';
|
|
||||||
|
|
||||||
const { data: pr } = await github.rest.pulls.create({
|
|
||||||
owner: context.repo.owner,
|
|
||||||
repo: context.repo.repo,
|
|
||||||
title: 'Merge ' + branchName + ' into main',
|
|
||||||
head: branchName,
|
|
||||||
base: 'main',
|
|
||||||
body: prBody,
|
|
||||||
draft: false
|
|
||||||
});
|
|
||||||
|
|
||||||
// Добавляем labels
|
|
||||||
try {
|
|
||||||
await github.rest.issues.addLabels({
|
|
||||||
owner: context.repo.owner,
|
|
||||||
repo: context.repo.repo,
|
|
||||||
issue_number: pr.number,
|
|
||||||
labels: ['automated', 'ready-for-review']
|
|
||||||
});
|
|
||||||
} catch (labelError) {
|
|
||||||
console.log('⚠️ Could not add labels: ' + labelError.message);
|
|
||||||
}
|
|
||||||
|
|
||||||
core.setOutput('number', pr.number.toString());
|
|
||||||
core.setOutput('url', pr.html_url);
|
|
||||||
console.log('✅ PR #' + pr.number + ' created successfully: ' + pr.html_url);
|
|
||||||
|
|
||||||
} catch (error) {
|
|
||||||
console.error('❌ Failed to create PR: ' + error.message);
|
|
||||||
if (error.status === 422) {
|
|
||||||
console.log('⚠️ PR might already exist or branch is up to date with base');
|
|
||||||
// Пробуем найти существующий PR
|
|
||||||
const { data: prs } = await github.rest.pulls.list({
|
|
||||||
owner: context.repo.owner,
|
|
||||||
repo: context.repo.repo,
|
|
||||||
head: context.repo.owner + ':' + branchName,
|
|
||||||
base: 'main',
|
|
||||||
state: 'open'
|
|
||||||
});
|
|
||||||
|
|
||||||
if (prs.length > 0) {
|
|
||||||
const pr = prs[0];
|
|
||||||
core.setOutput('number', pr.number.toString());
|
|
||||||
core.setOutput('url', pr.html_url);
|
|
||||||
console.log('✅ Found existing PR #' + pr.number + ': ' + pr.html_url);
|
|
||||||
} else {
|
|
||||||
core.setOutput('number', '');
|
|
||||||
const serverUrl = '${{ github.server_url }}';
|
|
||||||
const repo = '${{ github.repository }}';
|
|
||||||
core.setOutput('url', serverUrl + '/' + repo + '/pulls');
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
- name: Send PR notification - PR exists
|
|
||||||
if: steps.check-pr.outputs.exists == 'true'
|
|
||||||
uses: appleboy/telegram-action@v1.0.0
|
|
||||||
with:
|
|
||||||
to: ${{ secrets.TELEGRAM_CHAT_ID }}
|
|
||||||
token: ${{ secrets.TELEGRAM_BOT_TOKEN }}
|
|
||||||
message: |
|
|
||||||
ℹ️ Pull Request Updated
|
|
||||||
|
|
||||||
📦 Repository: prod
|
|
||||||
🌿 Branch: ${{ github.ref_name }} → main
|
|
||||||
📝 Commit: ${{ github.sha }}
|
|
||||||
👤 Author: ${{ github.actor }}
|
|
||||||
|
|
||||||
✅ All tests passed! PR #${{ steps.check-pr.outputs.number }} already exists and has been updated.
|
|
||||||
|
|
||||||
🔗 View PR: ${{ steps.check-pr.outputs.url }}
|
|
||||||
continue-on-error: true
|
|
||||||
|
|
||||||
- name: Send PR notification - PR created
|
|
||||||
if: steps.check-pr.outputs.exists == 'false' && steps.create-pr.outcome == 'success'
|
|
||||||
uses: appleboy/telegram-action@v1.0.0
|
|
||||||
with:
|
|
||||||
to: ${{ secrets.TELEGRAM_CHAT_ID }}
|
|
||||||
token: ${{ secrets.TELEGRAM_BOT_TOKEN }}
|
|
||||||
message: |
|
|
||||||
📝 Pull Request Created
|
|
||||||
|
|
||||||
📦 Repository: prod
|
|
||||||
🌿 Branch: ${{ github.ref_name }} → main
|
|
||||||
📝 Commit: ${{ github.sha }}
|
|
||||||
👤 Author: ${{ github.actor }}
|
|
||||||
|
|
||||||
${{ steps.create-pr.outputs.number != '' && format('✅ All tests passed! Pull request #{0} has been created and is ready for review.', steps.create-pr.outputs.number) || '✅ All tests passed! Pull request has been created and is ready for review.' }}
|
|
||||||
|
|
||||||
${{ steps.create-pr.outputs.url != '' && format('🔗 View PR: {0}', steps.create-pr.outputs.url) || format('🔗 View PRs: {0}/{1}/pulls', github.server_url, github.repository) }}
|
|
||||||
continue-on-error: true
|
|
||||||
|
|
||||||
rollback:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
name: Manual Rollback
|
|
||||||
if: |
|
|
||||||
github.event_name == 'workflow_dispatch' &&
|
|
||||||
github.event.inputs.action == 'rollback'
|
|
||||||
environment:
|
|
||||||
name: production
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: Checkout code
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
ref: main
|
|
||||||
|
|
||||||
- name: Manual Rollback
|
|
||||||
uses: appleboy/ssh-action@v1.0.0
|
|
||||||
with:
|
|
||||||
host: ${{ vars.SERVER_HOST || secrets.SERVER_HOST }}
|
|
||||||
username: ${{ vars.SERVER_USER || secrets.SERVER_USER }}
|
|
||||||
key: ${{ secrets.SSH_PRIVATE_KEY }}
|
|
||||||
port: ${{ vars.SSH_PORT || secrets.SSH_PORT || 22 }}
|
|
||||||
script: |
|
|
||||||
set -e
|
|
||||||
echo "🔄 Starting manual rollback..."
|
|
||||||
|
|
||||||
cd /home/prod
|
|
||||||
DEPLOY_HISTORY="/home/prod/.deploy_history.txt"
|
|
||||||
DEPLOY_HISTORY_SIZE="${DEPLOY_HISTORY_SIZE:-10}"
|
|
||||||
|
|
||||||
# Определяем коммит для отката
|
|
||||||
if [ -n "${{ github.event.inputs.rollback_to_commit }}" ]; then
|
|
||||||
ROLLBACK_COMMIT="${{ github.event.inputs.rollback_to_commit }}"
|
|
||||||
echo "📝 Using specified commit: $ROLLBACK_COMMIT"
|
|
||||||
else
|
|
||||||
# Используем последний успешный деплой из истории
|
|
||||||
ROLLBACK_COMMIT=$(grep "|success" "$DEPLOY_HISTORY" 2>/dev/null | tail -1 | cut -d'|' -f2 || echo "")
|
|
||||||
|
|
||||||
# Если нет в истории, используем сохраненный коммит
|
|
||||||
if [ -z "$ROLLBACK_COMMIT" ]; then
|
|
||||||
if [ -f "/tmp/last_deploy_commit.txt" ]; then
|
|
||||||
ROLLBACK_COMMIT=$(cat /tmp/last_deploy_commit.txt)
|
|
||||||
echo "📝 Using saved commit from /tmp/last_deploy_commit.txt: $ROLLBACK_COMMIT"
|
|
||||||
else
|
|
||||||
echo "❌ No commit specified and no previous deploy found!"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
echo "📝 Using last successful deploy from history: $ROLLBACK_COMMIT"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Проверяем что коммит существует
|
|
||||||
git fetch origin main
|
|
||||||
if ! git rev-parse --verify "$ROLLBACK_COMMIT" > /dev/null 2>&1; then
|
|
||||||
echo "❌ Commit $ROLLBACK_COMMIT not found!"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Откатываем код
|
|
||||||
echo "🔄 Rolling back to commit: $ROLLBACK_COMMIT"
|
|
||||||
|
|
||||||
# Исправляем права на файлы
|
|
||||||
sudo chown -R deploy:deploy /home/prod || true
|
|
||||||
|
|
||||||
git reset --hard "$ROLLBACK_COMMIT"
|
|
||||||
|
|
||||||
# Устанавливаем правильные права после отката
|
|
||||||
sudo chown -R deploy:deploy /home/prod || true
|
|
||||||
|
|
||||||
echo "✅ Code rolled back to: $ROLLBACK_COMMIT"
|
|
||||||
|
|
||||||
# Пересобираем все контейнеры с обновлением базовых образов и кешированием
|
|
||||||
echo "🔨 Rebuilding all containers with --pull (updating base images, using cache)..."
|
|
||||||
docker-compose down || true
|
|
||||||
docker-compose build --pull
|
|
||||||
docker-compose up -d
|
|
||||||
|
|
||||||
echo "✅ Containers rebuilt and started"
|
|
||||||
|
|
||||||
# Ждем запуска сервисов
|
|
||||||
echo "⏳ Waiting for services to start (45 seconds)..."
|
|
||||||
sleep 45
|
|
||||||
|
|
||||||
# Health checks с экспоненциальным retry
|
|
||||||
check_health() {
|
|
||||||
local service=$1
|
|
||||||
local url=$2
|
|
||||||
local attempt=1
|
|
||||||
local delays=(5 15 45)
|
|
||||||
local max_attempts=${#delays[@]}
|
|
||||||
|
|
||||||
echo "🔍 Checking $service health..."
|
|
||||||
|
|
||||||
while [ $attempt -le $max_attempts ]; do
|
|
||||||
if curl -f -s --max-time 10 "$url" > /dev/null 2>&1; then
|
|
||||||
echo "✅ $service is healthy (attempt $attempt/$max_attempts)"
|
|
||||||
return 0
|
|
||||||
else
|
|
||||||
if [ $attempt -lt $max_attempts ]; then
|
|
||||||
delay=${delays[$((attempt - 1))]}
|
|
||||||
echo "⏳ $service not ready yet (attempt $attempt/$max_attempts), waiting ${delay} seconds..."
|
|
||||||
sleep $delay
|
|
||||||
else
|
|
||||||
echo "❌ $service health check failed after $max_attempts attempts"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
attempt=$((attempt + 1))
|
|
||||||
done
|
|
||||||
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
HEALTH_CHECK_FAILED=0
|
|
||||||
|
|
||||||
check_health "Prometheus" "http://localhost:9090/-/healthy" || HEALTH_CHECK_FAILED=1
|
|
||||||
check_health "Grafana" "http://localhost:3000/api/health" || HEALTH_CHECK_FAILED=1
|
|
||||||
check_health "Telegram Bot" "http://localhost:8080/health" || HEALTH_CHECK_FAILED=1
|
|
||||||
check_health "AnonBot" "http://localhost:8081/health" || HEALTH_CHECK_FAILED=1
|
|
||||||
|
|
||||||
if [ $HEALTH_CHECK_FAILED -eq 1 ]; then
|
|
||||||
echo "⚠️ Some health checks failed, but rollback completed"
|
|
||||||
else
|
|
||||||
echo "✅ All health checks passed after rollback!"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Обновляем историю
|
|
||||||
TIMESTAMP=$(date +"%Y-%m-%d %H:%M:%S")
|
|
||||||
COMMIT_MESSAGE=$(git log -1 --pretty=format:"%s" "$ROLLBACK_COMMIT" 2>/dev/null || echo "Manual rollback")
|
|
||||||
COMMIT_AUTHOR="${{ github.actor }}"
|
|
||||||
echo "${TIMESTAMP}|${ROLLBACK_COMMIT}|${COMMIT_MESSAGE}|${COMMIT_AUTHOR}|rolled_back" >> "$DEPLOY_HISTORY"
|
|
||||||
|
|
||||||
# Оставляем только последние N записей
|
|
||||||
tail -n "$DEPLOY_HISTORY_SIZE" "$DEPLOY_HISTORY" > "${DEPLOY_HISTORY}.tmp" && mv "${DEPLOY_HISTORY}.tmp" "$DEPLOY_HISTORY"
|
|
||||||
|
|
||||||
echo "✅ Rollback completed successfully"
|
|
||||||
|
|
||||||
- name: Send rollback notification
|
|
||||||
if: always()
|
|
||||||
uses: appleboy/telegram-action@v1.0.0
|
|
||||||
with:
|
|
||||||
to: ${{ secrets.TELEGRAM_CHAT_ID }}
|
|
||||||
token: ${{ secrets.TELEGRAM_BOT_TOKEN }}
|
|
||||||
message: |
|
|
||||||
🔄 Manual Rollback: ${{ job.status }}
|
|
||||||
|
|
||||||
📦 Repository: prod
|
|
||||||
🌿 Branch: main
|
|
||||||
📝 Commit: ${{ github.event.inputs.rollback_to_commit || 'Previous successful deploy' }}
|
|
||||||
👤 Author: ${{ github.actor }}
|
|
||||||
|
|
||||||
${{ job.status == 'success' && '✅ Rollback completed successfully! Services restored to specified version.' || '❌ Rollback failed! Check logs for details.' }}
|
|
||||||
|
|
||||||
🔗 View details: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
||||||
continue-on-error: true
|
|
||||||
Reference in New Issue
Block a user