From 7c9e0303b930d3c099a019ed5b464ef330754ff9 Mon Sep 17 00:00:00 2001 From: Jacky Date: Sun, 9 Aug 2026 00:10:35 +0800 Subject: [PATCH] ci: add rollback-aware production smoke checks --- .gitea/workflows/deploy.yml | 46 ++++++++++++++++----------- docs/BAOTA_UBUNTU_FROM_ZERO.md | 22 ++++++++++--- docs/DEPLOYMENT.md | 30 +++++++++++++++--- scripts/deploy_production.sh | 13 ++++++++ scripts/smoke_production.py | 57 ++++++++++++++++++++++++++++++++++ 5 files changed, 141 insertions(+), 27 deletions(-) create mode 100644 scripts/smoke_production.py diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index a248b57..b06933f 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -12,9 +12,10 @@ concurrency: cancel-in-progress: false jobs: - test: + release-check: if: ${{ github.event.pull_request.merged == true }} runs-on: ubuntu-latest + timeout-minutes: 15 services: mysql: image: mysql:8.0.35 @@ -44,44 +45,41 @@ jobs: python3-dev \ python3-venv - - name: 安装测试依赖 + - name: 安装发布检查依赖 run: | - python3 -m venv .venv-ci - .venv-ci/bin/python -m pip install \ + python3 -m venv .venv-release + .venv-release/bin/python -m pip install \ --index-url https://mirrors.aliyun.com/pypi/simple \ --timeout 120 \ --retries 5 \ - -r requirements-dev.txt + -r requirements.txt - name: 检查迁移文件 working-directory: backend - run: ../.venv-ci/bin/python manage.py makemigrations --check --dry-run + run: ../.venv-release/bin/python manage.py makemigrations --check --dry-run - name: Django 系统检查 working-directory: backend - run: ../.venv-ci/bin/python manage.py check + run: ../.venv-release/bin/python manage.py check - name: ASGI 启动导入检查 working-directory: backend - run: ../.venv-ci/bin/python -c "from config.asgi import application; print(type(application).__name__)" + run: ../.venv-release/bin/python -c "from config.asgi import application; print(type(application).__name__)" - - name: 运行测试 - run: .venv-ci/bin/python -m pytest -q - - - name: MySQL 迁移与全量测试 + - name: MySQL 发布迁移检查 env: DATABASE_URL: mysql://root:ci-root-password@mysql:3306/hulumath run: | - .venv-ci/bin/python backend/manage.py check --database default - .venv-ci/bin/python backend/manage.py check_mysql - .venv-ci/bin/python backend/manage.py migrate --noinput - .venv-ci/bin/python -m pytest -q + .venv-release/bin/python backend/manage.py check --database default + .venv-release/bin/python backend/manage.py check_mysql + .venv-release/bin/python backend/manage.py migrate --noinput deploy: if: ${{ github.event.pull_request.merged == true }} needs: - - test + - release-check runs-on: ubuntu-latest + timeout-minutes: 25 steps: - name: 配置 SSH 环境 env: @@ -92,7 +90,7 @@ jobs: mkdir -p ~/.ssh printf '%s\n' "$DEPLOY_SSH_KEY" > ~/.ssh/deploy_key chmod 600 ~/.ssh/deploy_key - ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts + timeout 10 ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts - name: 远程执行 Django 部署 env: @@ -139,6 +137,8 @@ jobs: PROJECT_DIR="$PROJECT" \ PYTHON_BIN="$PYTHON_BIN" \ PREVIOUS_REVISION="$PREVIOUS_REVISION" \ + DEPLOY_SMOKE_BASE_URL="http://127.0.0.1:4321" \ + DEPLOY_SMOKE_WS_URL="ws://127.0.0.1:4321/ws/health/" \ bash scripts/deploy_production.sh echo "========================================" @@ -146,3 +146,13 @@ jobs: echo "========================================" EOF + + - name: 从 Runner 验证 Nginx 健康状态 + env: + DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }} + run: | + curl --fail --silent --show-error \ + --retry 5 \ + --retry-delay 2 \ + "http://$DEPLOY_HOST:4321/health/" \ + | grep -q '"status": "ok"' diff --git a/docs/BAOTA_UBUNTU_FROM_ZERO.md b/docs/BAOTA_UBUNTU_FROM_ZERO.md index ebf91dd..259cdbe 100644 --- a/docs/BAOTA_UBUNTU_FROM_ZERO.md +++ b/docs/BAOTA_UBUNTU_FROM_ZERO.md @@ -477,16 +477,28 @@ chmod 600 /root/.ssh/authorized_keys 确保有能匹配 `ubuntu-latest` 的在线 Runner。 -PR 合并到 `main` 后，workflow 会： +如果 Runner 和生产环境在同一 Docker 宿主机，且公网 IP 不支持 NAT 回环， +`DEPLOY_HOST` 可使用 Docker 网桥宿主机地址 `172.17.0.1`。 + +先在 Gitea 保护 `main` 分支，禁止直接推送，并要求 `CI / test` 通过后才能合并。 + +PR 阶段会执行： ```text -SQLite 快速测试 -→ 启动 MySQL 8.0.35 临时容器 -→ MySQL 全量迁移和测试 +Ruff 静态检查 +→ SQLite 测试和 75% 覆盖率门槛 +→ MySQL 8.0.35 全量迁移和测试 +``` + +PR 合并到 `main` 后会执行： + +```text +MySQL 8.0.35 发布迁移检查 → SSH 生产服务器 → MySQL 备份 → migrate -→ 重启和健康检查 +→ 重启 +→ 应用与 Nginx HTTP/WebSocket 冒烟检查 ``` ## 16. 备份与恢复 diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 47ff915..66144c6 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -2,17 +2,39 @@ Ubuntu 宝塔面板全新服务器请优先阅读 [Ubuntu + 宝塔面板从零部署](BAOTA_UBUNTU_FROM_ZERO.md)。本文只描述自动发布机制。 +PR 质量门禁由 `.gitea/workflows/ci.yml` 执行： + +1. Ruff 静态检查。 +2. 迁移文件、Django 和 ASGI 导入检查。 +3. SQLite 单元测试，覆盖率不得低于 75%。 +4. MySQL 8.0.35 全量迁移和测试。 + +同一 PR 推送新提交时，旧 CI 会自动取消。单次 CI 最多运行 20 分钟。 + 生产发布由 `.gitea/workflows/deploy.yml` 执行。目标为 `main` 的 PR 被真正合并后： -1. Runner 检出 `main` 并运行迁移检查、Django 检查和全部测试。 -2. 测试通过后 SSH 到生产服务器。 +1. Runner 检出 `main` 并在临时 MySQL 8.0.35 上执行发布迁移检查。 +2. 检查通过后 SSH 到生产服务器。 3. 服务器备份 MySQL，安装依赖，执行迁移并收集静态资源。 4. systemd 重启 Django ASGI 服务。 -5. 分别验证 HTTP、WebSocket 和 Redis Channel Layer。 -6. 失败时回退应用代码；数据库备份保留，不自动执行破坏性反向迁移。 +5. 验证应用 HTTP、Redis Channel Layer 和直连 WebSocket。 +6. 通过 Nginx 验证健康接口、首页、后台、视频目录和 WebSocket。 +7. Runner 从外部容器网络再次验证 Nginx 健康接口。 +8. 失败时回退应用代码；数据库备份保留，不自动执行破坏性反向迁移。 关闭但未合并的 PR 不会部署。同一时间只允许一个生产部署任务执行。 +## main 分支保护 + +仓库必须保护 `main` 分支： + +- 禁止直接推送，所有改动必须通过 PR。 +- 合并前必须通过状态检查 `CI / test`。 +- 禁止在检查未完成时合并。 +- 管理员也应遵守分支保护。 + +完整测试只在 PR 阶段执行。合并后发布流程不重复运行全量测试，因此分支保护是生产发布的必要条件。 + ## Gitea Secrets 在仓库 `Settings > Actions > Secrets` 配置： diff --git a/scripts/deploy_production.sh b/scripts/deploy_production.sh index 464bb27..d4fcbe3 100755 --- a/scripts/deploy_production.sh +++ b/scripts/deploy_production.sh @@ -13,6 +13,8 @@ MYSQLDUMP_BIN="${MYSQLDUMP_BIN:-}" MYSQL_CONTAINER="${MYSQL_CONTAINER:-}" PREVIOUS_REVISION="${PREVIOUS_REVISION:-}" HEALTH_HOST="${DEPLOY_HEALTH_HOST:-}" +SMOKE_BASE_URL="${DEPLOY_SMOKE_BASE_URL:-}" +SMOKE_WS_URL="${DEPLOY_SMOKE_WS_URL:-}" ROLLBACK_ENABLED=0 MYSQL_CREDENTIALS_FILE="" MYSQL_CONTAINER_CREDENTIALS_PATH="" @@ -291,6 +293,17 @@ async def check(): asyncio.run(check()) PY +if [ -n "$SMOKE_BASE_URL" ] || [ -n "$SMOKE_WS_URL" ]; then + [ -n "$SMOKE_BASE_URL" ] && [ -n "$SMOKE_WS_URL" ] \ + || fail "DEPLOY_SMOKE_BASE_URL 和 DEPLOY_SMOKE_WS_URL 必须同时设置" + [ -f "$PROJECT_DIR/scripts/smoke_production.py" ] \ + || fail "缺少生产冒烟脚本: scripts/smoke_production.py" + log "检查 Nginx、业务页面与公网 WebSocket" + "$VENV_DIR/bin/python" "$PROJECT_DIR/scripts/smoke_production.py" \ + --base-url "$SMOKE_BASE_URL" \ + --ws-url "$SMOKE_WS_URL" +fi + trap - ERR ROLLBACK_ENABLED=0 log "部署完成，当前版本: $(git rev-parse --short HEAD)" diff --git a/scripts/smoke_production.py b/scripts/smoke_production.py new file mode 100644 index 0000000..5f7dd4d --- /dev/null +++ b/scripts/smoke_production.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +import argparse +import asyncio +import json +from urllib.request import urlopen + +from websockets.asyncio.client import connect + + +def fetch(base_url, path): + url = f"{base_url.rstrip('/')}/{path.lstrip('/')}" + with urlopen(url, timeout=10) as response: + if response.status != 200: + raise RuntimeError(f"{url} returned HTTP {response.status}") + return response.read() + + +async def check_websocket(url): + async with connect(url, open_timeout=10, close_timeout=5) as websocket: + payload = json.loads(await asyncio.wait_for(websocket.recv(), timeout=10)) + expected = {"status": "ok", "channel_layer": "ok"} + if payload != expected: + raise RuntimeError(f"{url} returned unexpected payload: {payload}") + + +def main(): + parser = argparse.ArgumentParser(description="Check the deployed Hulumath Nginx surface") + parser.add_argument("--base-url", required=True) + parser.add_argument("--ws-url", required=True) + parser.add_argument("--minimum-videos", type=int, default=54) + args = parser.parse_args() + + health = json.loads(fetch(args.base_url, "/health/")) + if health != {"status": "ok", "database": "ok"}: + raise RuntimeError(f"unexpected health payload: {health}") + + homepage = fetch(args.base_url, "/").decode("utf-8") + if "葫芦数学" not in homepage: + raise RuntimeError("homepage marker is missing") + + admin_login = fetch(args.base_url, "/admin/login/").decode("utf-8") + if "进入运营后台" not in admin_login: + raise RuntimeError("admin login marker is missing") + + catalog = json.loads(fetch(args.base_url, "/api/v1/content/videos/catalog/")) + if catalog.get("total", 0) < args.minimum_videos: + raise RuntimeError(f"video catalog is incomplete: {catalog.get('total', 0)}") + + asyncio.run(check_websocket(args.ws_url)) + print( + "Production smoke checks passed: " + f"health, homepage, admin, {catalog['total']} videos, websocket" + ) + + +if __name__ == "__main__": + main()