#!/usr/bin/env bash set -Eeuo pipefail source_dir=${1:?usage: deploy-jwebot-production.sh SOURCE_DIR [TARGET_USER]} target_user=${2:-jwebot} [[ "${target_user}" =~ ^[a-z_][a-z0-9_-]{0,31}$ ]] || { echo "[JWeBot] error: invalid target user" >&2 exit 1 } target_uid=$(id -u "${target_user}") target_gid=$(id -g "${target_user}") target_home=$(getent passwd "${target_user}" | cut -d: -f6) user_runtime="/run/user/${target_uid}" runtime_python=${JWEBOT_RUNTIME_PYTHON:-/opt/jwebot/venv/bin/python} require_wechat_ready=${REQUIRE_WECHAT_READY:-1} wechat_ready_timeout_seconds=${WECHAT_READY_TIMEOUT_SECONDS:-30} health_settle_seconds=${HEALTH_SETTLE_SECONDS:-8} backup_root=${JWEBOT_PRODUCTION_BACKUP_ROOT:-/var/backups/jwebot-production} timestamp=$(date +%Y%m%d-%H%M%S) backup_dir="${backup_root}/${timestamp}-$$" preflight_rendered_dir="" target_runtime="${target_home}/.local/lib/wechat-health-watch" target_user_units="${target_home}/.config/systemd/user" target_user_config="${target_home}/.config/jwebot" target_edge_state="${target_home}/.local/state/jwebot-edge" target_adaptive_state="${target_home}/.local/state/wechat-adaptive-ocr" root_ocr=/usr/local/sbin/wechat-local-ocr root_bridge=/usr/local/sbin/wechat-gui-bridge root_recorder_unit=/etc/systemd/system/wechat-local-ocr-recorder.service root_responder_unit=/etc/systemd/system/wechat-local-ocr-responder.service root_defaults=/etc/default/wechat-local-ocr system_state_file="${backup_dir}/metadata/system-state.tsv" user_state_file="${backup_dir}/metadata/user-state.tsv" armed=0 success=0 system_units=( wechat-local-ocr-recorder.service wechat-local-ocr-responder.service ) user_units=( jwebot-edge.service wechat-ocr-adaptive-recorder.service wechat-health-watch.timer jwebot-edge-memory-sync.timer wechat-health-watch.service jwebot-edge-memory-sync.service ) jwebot_log() { printf '[JWeBot] %s\n' "$*" } jwebot_progress() { local step=$1 local total=$2 local label=$3 local width=20 local filled=$((step * width / total)) local empty=$((width - filled)) local bar printf -v bar '%*s' "${filled}" '' bar=${bar// /#} printf -v padding '%*s' "${empty}" '' padding=${padding// /-} printf '[JWeBot] [%s%s] %3d%% %s\n' \ "${bar}" "${padding}" "$((step * 100 / total))" "${label}" } if [[ "${EUID}" -ne 0 ]]; then echo "[JWeBot] error: production deploy must run as root" >&2 exit 1 fi if [[ -z "${target_home}" || ! -d "${target_home}" ]]; then echo "[JWeBot] error: target user home is unavailable" >&2 exit 1 fi jwebot_log "production deploy starting version=$(cat "${source_dir}/VERSION")" jwebot_progress 1 8 "preflight" user_systemctl() { /usr/sbin/runuser -u "${target_user}" -- /usr/bin/env \ HOME="${target_home}" \ XDG_RUNTIME_DIR="${user_runtime}" \ DBUS_SESSION_BUS_ADDRESS="unix:path=${user_runtime}/bus" \ systemctl --user "$@" } save_system_state() { : >"${system_state_file}" local unit enabled active for unit in "${system_units[@]}"; do enabled=$(systemctl is-enabled "${unit}" 2>/dev/null || true) active=$(systemctl is-active "${unit}" 2>/dev/null || true) printf '%s\t%s\t%s\n' "${unit}" "${enabled:-unknown}" "${active:-unknown}" \ >>"${system_state_file}" done } save_user_state() { : >"${user_state_file}" local unit enabled active for unit in "${user_units[@]}"; do enabled=$(user_systemctl is-enabled "${unit}" 2>/dev/null || true) active=$(user_systemctl is-active "${unit}" 2>/dev/null || true) printf '%s\t%s\t%s\n' "${unit}" "${enabled:-unknown}" "${active:-unknown}" \ >>"${user_state_file}" done } state_field() { local file="$1" local unit="$2" local field="$3" awk -F '\t' -v unit="${unit}" -v field="${field}" \ '$1 == unit { print $field; exit }' "${file}" } restore_enable_state() { local scope="$1" local file="$2" shift 2 local unit enabled for unit in "$@"; do enabled=$(state_field "${file}" "${unit}" 2 2>/dev/null || true) case "${enabled}" in enabled) if [[ "${scope}" == "user" ]]; then user_systemctl unmask "${unit}" >/dev/null 2>&1 || true user_systemctl enable "${unit}" >/dev/null 2>&1 || true else systemctl unmask "${unit}" >/dev/null 2>&1 || true systemctl enable "${unit}" >/dev/null 2>&1 || true fi ;; enabled-runtime) if [[ "${scope}" == "user" ]]; then user_systemctl unmask "${unit}" >/dev/null 2>&1 || true user_systemctl enable --runtime "${unit}" >/dev/null 2>&1 || true else systemctl unmask "${unit}" >/dev/null 2>&1 || true systemctl enable --runtime "${unit}" >/dev/null 2>&1 || true fi ;; disabled) if [[ "${scope}" == "user" ]]; then user_systemctl unmask "${unit}" >/dev/null 2>&1 || true user_systemctl disable "${unit}" >/dev/null 2>&1 || true else systemctl unmask "${unit}" >/dev/null 2>&1 || true systemctl disable "${unit}" >/dev/null 2>&1 || true fi ;; masked) if [[ "${scope}" == "user" ]]; then user_systemctl mask "${unit}" >/dev/null 2>&1 || true else systemctl mask "${unit}" >/dev/null 2>&1 || true fi ;; masked-runtime) if [[ "${scope}" == "user" ]]; then user_systemctl mask --runtime "${unit}" >/dev/null 2>&1 || true else systemctl mask --runtime "${unit}" >/dev/null 2>&1 || true fi ;; esac done } restore_active_state() { local scope="$1" local file="$2" shift 2 local unit active for unit in "$@"; do active=$(state_field "${file}" "${unit}" 3 2>/dev/null || true) case "${active}" in active|activating|reloading) if [[ "${scope}" == "user" ]]; then user_systemctl start "${unit}" >/dev/null 2>&1 || true else systemctl start "${unit}" >/dev/null 2>&1 || true fi ;; esac done } backup_file() { local source="$1" local name="$2" if [[ -e "${source}" || -L "${source}" ]]; then cp -a "${source}" "${backup_dir}/root/${name}" : >"${backup_dir}/root/${name}.present" fi } restore_file() { local target="$1" local name="$2" if [[ -f "${backup_dir}/root/${name}.present" ]]; then rm -rf "${target}" cp -a "${backup_dir}/root/${name}" "${target}" else rm -rf "${target}" fi } copy_directory_backup() { local source="$1" local name="$2" if [[ -d "${source}" ]]; then cp -a "${source}" "${backup_dir}/user/${name}" : >"${backup_dir}/user/${name}.present" fi } restore_directory_backup() { local target="$1" local name="$2" rm -rf "${target}" if [[ -f "${backup_dir}/user/${name}.present" ]]; then cp -a "${backup_dir}/user/${name}" "${target}" fi } queue_count() { local directory="$1" if [[ ! -d "${directory}" ]]; then printf '0\n' return fi find "${directory}" -maxdepth 1 -type f -print | wc -l } assert_queues_empty() { local adaptive legacy adaptive=$(queue_count "${target_adaptive_state}/queue") legacy=$(queue_count /var/lib/wechat-local-ocr/queue) if [[ "${adaptive}" -ne 0 || "${legacy}" -ne 0 ]]; then echo "[JWeBot] error: OCR queues must be empty before deployment" >&2 return 1 fi } wechat_geometry() { local identifiers identifier title geometry width height local class_identifiers name_identifiers local best_width=0 local best_height=0 local display_environment=( HOME="${target_home}" DISPLAY=:1 XAUTHORITY="${target_home}/.Xauthority" XDG_RUNTIME_DIR="${user_runtime}" DBUS_SESSION_BUS_ADDRESS="unix:path=${user_runtime}/bus" ) class_identifiers=$( /usr/sbin/runuser -u "${target_user}" -- \ /usr/bin/env "${display_environment[@]}" \ /usr/bin/xdotool search --onlyvisible --class wechat \ 2>/dev/null || true ) name_identifiers=$( /usr/sbin/runuser -u "${target_user}" -- \ /usr/bin/env "${display_environment[@]}" \ /usr/bin/xdotool search --onlyvisible --name '^Weixin$' \ 2>/dev/null || true ) identifiers=$( printf '%s\n%s\n' "${class_identifiers}" "${name_identifiers}" \ | awk 'NF && !seen[$0]++' ) for identifier in ${identifiers}; do title=$( /usr/sbin/runuser -u "${target_user}" -- \ /usr/bin/env "${display_environment[@]}" \ /usr/bin/xdotool getwindowname "${identifier}" \ 2>/dev/null || true ) [[ "${title}" == "Weixin" ]] || continue geometry=$( /usr/sbin/runuser -u "${target_user}" -- \ /usr/bin/env "${display_environment[@]}" \ /usr/bin/xdotool getwindowgeometry --shell "${identifier}" \ 2>/dev/null || true ) width=$(sed -n 's/^WIDTH=//p' <<<"${geometry}" | head -n 1) height=$(sed -n 's/^HEIGHT=//p' <<<"${geometry}" | head -n 1) [[ "${width}" =~ ^[0-9]+$ && "${height}" =~ ^[0-9]+$ ]] || continue if (( width * height > best_width * best_height )); then best_width=${width} best_height=${height} fi done if (( best_width > 0 && best_height > 0 )); then printf 'WIDTH=%s\nHEIGHT=%s\n' "${best_width}" "${best_height}" fi } assert_wechat_ready() { local geometry="" width="" height="" local attempt=0 jwebot_log "WeChat readiness starting timeout=${wechat_ready_timeout_seconds}s" while (( attempt < wechat_ready_timeout_seconds )); do geometry=$(wechat_geometry) width=$(sed -n 's/^WIDTH=//p' <<<"${geometry}" | head -n 1) height=$(sed -n 's/^HEIGHT=//p' <<<"${geometry}" | head -n 1) if [[ "${width}" =~ ^[0-9]+$ && "${height}" =~ ^[0-9]+$ ]] \ && (( width >= 700 && height >= 500 )) then jwebot_log "WeChat ready geometry=${width}x${height}" return 0 fi attempt=$((attempt + 1)) if (( attempt < wechat_ready_timeout_seconds )); then sleep 1 fi done if [[ "${width}" =~ ^[0-9]+$ && "${height}" =~ ^[0-9]+$ ]]; then echo "[JWeBot] error: WeChat is not ready (${width}x${height})" >&2 else echo "[JWeBot] error: WeChat window geometry is unavailable" >&2 fi return 1 } verify_edge_health() { local response response=$( printf '%s\n' '{"op":"health"}' \ | /usr/sbin/runuser -u "${target_user}" -- /usr/bin/env \ HOME="${target_home}" \ "${runtime_python}" "${target_runtime}/jwebot_edge.py" respond ) printf '%s' "${response}" \ | "${runtime_python}" -c \ 'import json,sys; value=json.load(sys.stdin); raise SystemExit(0 if value.get("status") == "ok" else 1)' } send_release_lifecycle_notice() { local stage=$1 /usr/sbin/runuser -u "${target_user}" -- /usr/bin/env \ HOME="${target_home}" \ XDG_RUNTIME_DIR="${user_runtime}" \ DBUS_SESSION_BUS_ADDRESS="unix:path=${user_runtime}/bus" \ "${runtime_python}" "${source_dir}/jwebot_release_update.py" \ --release-file "${source_dir}/VERSION" \ --database "${target_edge_state}/edge.sqlite3" \ --delivery-helper "${target_runtime}/jwebot_edge_delivery.py" \ --lifecycle-stage "${stage}" } render_root_configuration() { local output_dir="$1" sed \ -e "s/^User=.*/User=${target_user}/" \ -e "s/^Group=.*/Group=${target_user}/" \ -e "s#/opt/jwebot/venv/bin/python#${runtime_python}#g" \ "${source_dir}/deploy/systemd/system/wechat-local-ocr-recorder.service" \ >"${output_dir}/wechat-local-ocr-recorder.service" sed \ -e "s#/opt/jwebot/venv/bin/python#${runtime_python}#g" \ "${source_dir}/deploy/systemd/system/wechat-local-ocr-responder.service" \ >"${output_dir}/wechat-local-ocr-responder.service" sed \ -e "s#^JWEBOT_RUNTIME_USER=.*#JWEBOT_RUNTIME_USER=${target_user}#" \ -e "s#^JWEBOT_RUNTIME_HOME=.*#JWEBOT_RUNTIME_HOME=${target_home}#" \ "${source_dir}/deploy/defaults/wechat-local-ocr.production.default" \ >"${output_dir}/wechat-local-ocr.default" } rollback() { local status=$? trap - EXIT HUP INT TERM if [[ -n "${preflight_rendered_dir}" ]]; then rm -rf "${preflight_rendered_dir}" fi if [[ "${armed}" -eq 1 && "${success}" -eq 0 ]]; then jwebot_log "rollback starting backup=${backup_dir}" set +e user_systemctl stop \ wechat-ocr-adaptive-recorder.service \ jwebot-edge.service \ wechat-health-watch.timer \ jwebot-edge-memory-sync.timer \ >/dev/null 2>&1 systemctl disable --now \ wechat-local-ocr-recorder.service \ wechat-local-ocr-responder.service \ >/dev/null 2>&1 restore_file "${root_ocr}" wechat-local-ocr restore_file "${root_bridge}" wechat-gui-bridge restore_file "${root_recorder_unit}" wechat-local-ocr-recorder.service restore_file "${root_responder_unit}" wechat-local-ocr-responder.service restore_file "${root_defaults}" wechat-local-ocr.default restore_directory_backup "${target_runtime}" runtime restore_directory_backup "${target_user_units}" systemd-user restore_directory_backup "${target_user_config}" jwebot-config if [[ -f "${backup_dir}/state/edge.sqlite3" ]]; then install -d -o "${target_uid}" -g "${target_gid}" -m 0700 \ "${target_edge_state}" rm -f \ "${target_edge_state}/edge.sqlite3" \ "${target_edge_state}/edge.sqlite3-wal" \ "${target_edge_state}/edge.sqlite3-shm" install -o "${target_uid}" -g "${target_gid}" -m 0600 \ "${backup_dir}/state/edge.sqlite3" \ "${target_edge_state}/edge.sqlite3" fi systemctl daemon-reload user_systemctl daemon-reload restore_enable_state system "${system_state_file}" "${system_units[@]}" restore_enable_state user "${user_state_file}" "${user_units[@]}" restore_active_state user "${user_state_file}" "${user_units[@]}" restore_active_state system "${system_state_file}" "${system_units[@]}" systemctl reset-failed user_systemctl reset-failed echo "[JWeBot] rollback ready backup=${backup_dir}" >&2 fi exit "${status}" } trap rollback EXIT HUP INT TERM jwebot_log "preflight starting" test -x "${runtime_python}" for path in \ "${source_dir}/wechat_local_ocr.py" \ "${source_dir}/wechat-gui-bridge" \ "${source_dir}/jwebot_release_update.py" \ "${source_dir}/deploy/scripts/install-wechat-health-watch.sh" \ "${source_dir}/deploy/systemd/system/wechat-local-ocr-recorder.service" \ "${source_dir}/deploy/systemd/system/wechat-local-ocr-responder.service" \ "${source_dir}/deploy/defaults/wechat-local-ocr.production.default" do test -f "${path}" done bash -n "${source_dir}/wechat-gui-bridge" sh -n "${source_dir}/deploy/scripts/install-wechat-health-watch.sh" "${runtime_python}" -m py_compile \ "${source_dir}/wechat_local_ocr.py" \ "${source_dir}/jwebot_release_update.py" preflight_rendered_dir=$(mktemp -d) render_root_configuration "${preflight_rendered_dir}" systemd-analyze verify \ "${preflight_rendered_dir}/wechat-local-ocr-recorder.service" \ "${preflight_rendered_dir}/wechat-local-ocr-responder.service" rm -rf "${preflight_rendered_dir}" preflight_rendered_dir="" assert_queues_empty if [[ "${require_wechat_ready}" -eq 1 ]]; then assert_wechat_ready fi jwebot_log "preflight ready" jwebot_progress 2 8 "backup" jwebot_log "backup starting path=${backup_dir}" install -d -o root -g root -m 0700 \ "${backup_root}" \ "${backup_dir}" \ "${backup_dir}/root" \ "${backup_dir}/user" \ "${backup_dir}/state" \ "${backup_dir}/metadata" save_system_state save_user_state backup_file "${root_ocr}" wechat-local-ocr backup_file "${root_bridge}" wechat-gui-bridge backup_file "${root_recorder_unit}" wechat-local-ocr-recorder.service backup_file "${root_responder_unit}" wechat-local-ocr-responder.service backup_file "${root_defaults}" wechat-local-ocr.default copy_directory_backup "${target_runtime}" runtime copy_directory_backup "${target_user_units}" systemd-user copy_directory_backup "${target_user_config}" jwebot-config jwebot_log "backup ready path=${backup_dir}" armed=1 jwebot_progress 3 8 "stop services" jwebot_log "cutover starting" jwebot_log "release lifecycle shutdown notice starting" send_release_lifecycle_notice shutdown jwebot_log "release lifecycle shutdown notice ready" user_systemctl stop \ wechat-ocr-adaptive-recorder.service \ jwebot-edge.service \ wechat-health-watch.timer \ jwebot-edge-memory-sync.timer \ wechat-health-watch.service \ jwebot-edge-memory-sync.service \ >/dev/null 2>&1 || true systemctl stop \ wechat-local-ocr-responder.service \ wechat-local-ocr-recorder.service \ >/dev/null 2>&1 || true assert_queues_empty jwebot_log "services stopped queues=empty" if [[ -f "${target_edge_state}/edge.sqlite3" ]]; then "${runtime_python}" - \ "${target_edge_state}/edge.sqlite3" \ "${backup_dir}/state/edge.sqlite3" <<'PY' import sqlite3 import sys source = sqlite3.connect(f"file:{sys.argv[1]}?mode=ro", uri=True) target = sqlite3.connect(sys.argv[2]) try: if source.execute("PRAGMA quick_check").fetchone()[0] != "ok": raise SystemExit("source SQLite quick_check failed") source.backup(target) if target.execute("PRAGMA quick_check").fetchone()[0] != "ok": raise SystemExit("backup SQLite quick_check failed") target.commit() finally: target.close() source.close() PY fi jwebot_log "database backup ready" jwebot_progress 4 8 "install root runtime" jwebot_log "runtime install starting" rendered_recorder_unit="${backup_dir}/metadata/wechat-local-ocr-recorder.service" rendered_responder_unit="${backup_dir}/metadata/wechat-local-ocr-responder.service" rendered_defaults="${backup_dir}/metadata/wechat-local-ocr.default" render_root_configuration "${backup_dir}/metadata" install -o root -g root -m 0755 \ "${source_dir}/wechat_local_ocr.py" "${root_ocr}" install -o root -g root -m 0755 \ "${source_dir}/wechat-gui-bridge" "${root_bridge}" install -o root -g root -m 0644 \ "${rendered_recorder_unit}" \ "${root_recorder_unit}" install -o root -g root -m 0644 \ "${rendered_responder_unit}" \ "${root_responder_unit}" install -o root -g root -m 0644 \ "${rendered_defaults}" \ "${root_defaults}" systemctl daemon-reload systemctl disable --now \ wechat-local-ocr-recorder.service \ wechat-local-ocr-responder.service jwebot_log "runtime install ready" jwebot_progress 5 8 "install user runtime" jwebot_log "user runtime install starting" ( cd "${target_home}" /usr/sbin/runuser -u "${target_user}" -- /usr/bin/env \ HOME="${target_home}" \ XDG_RUNTIME_DIR="${user_runtime}" \ DBUS_SESSION_BUS_ADDRESS="unix:path=${user_runtime}/bus" \ START_TIMER=1 \ HEALTH_SETTLE_SECONDS="${health_settle_seconds}" \ JWEBOT_RUNTIME_PYTHON="${runtime_python}" \ /bin/sh "${source_dir}/deploy/scripts/install-wechat-health-watch.sh" \ "${source_dir}" ) jwebot_log "user runtime install ready" jwebot_progress 6 8 "verify services" systemctl disable --now \ wechat-local-ocr-recorder.service \ wechat-local-ocr-responder.service test "$( systemctl is-enabled wechat-local-ocr-recorder.service 2>/dev/null || true )" = "disabled" test "$( systemctl is-enabled wechat-local-ocr-responder.service 2>/dev/null || true )" = "disabled" if systemctl is-active --quiet wechat-local-ocr-recorder.service \ || systemctl is-active --quiet wechat-local-ocr-responder.service then echo "[JWeBot] error: legacy OCR unexpectedly remained active" >&2 exit 1 fi user_systemctl is-enabled jwebot-edge.service \ | grep -qx enabled user_systemctl is-enabled wechat-ocr-adaptive-recorder.service \ | grep -qx enabled user_systemctl is-active --quiet jwebot-edge.service jwebot_log "edge active" user_systemctl is-active --quiet wechat-ocr-adaptive-recorder.service jwebot_log "adaptive active" verify_edge_health jwebot_log "edge health ready" restarts_before=$( user_systemctl show wechat-ocr-adaptive-recorder.service \ -p NRestarts --value ) sleep "${health_settle_seconds}" user_systemctl is-active --quiet wechat-ocr-adaptive-recorder.service restarts_after=$( user_systemctl show wechat-ocr-adaptive-recorder.service \ -p NRestarts --value ) test "${restarts_before}" = "${restarts_after}" jwebot_log "adaptive stability ready restarts=${restarts_after}" assert_queues_empty jwebot_log "queues ready depth=0" if [[ "${require_wechat_ready}" -eq 1 ]]; then assert_wechat_ready fi cmp -s "${source_dir}/wechat_local_ocr.py" "${root_ocr}" cmp -s "${source_dir}/wechat_local_ocr.py" \ "${target_runtime}/wechat_local_ocr.py" if [[ -f "${target_edge_state}/edge.sqlite3" ]]; then db_check=$( /usr/sbin/runuser -u "${target_user}" -- \ /usr/bin/env HOME="${target_home}" \ "${runtime_python}" -c \ 'import sqlite3; print(sqlite3.connect("file:'"${target_edge_state}"'/edge.sqlite3?mode=ro", uri=True).execute("pragma quick_check").fetchone()[0])' ) test "${db_check}" = "ok" fi jwebot_log "database ready quick_check=${db_check:-absent}" jwebot_log "release lifecycle startup notice starting" send_release_lifecycle_notice startup jwebot_log "release lifecycle startup notice ready" jwebot_progress 7 8 "verify backup" ( cd "${backup_dir}" find . -type f ! -name SHA256SUMS -print0 \ | LC_ALL=C sort -z \ | xargs -0 sha256sum >SHA256SUMS sha256sum -c SHA256SUMS >metadata/sha256-verification.txt ) jwebot_log "backup verification ready" success=1 armed=0 trap - EXIT HUP INT TERM jwebot_log "release settlement starting" release_update_output=$( /usr/sbin/runuser -u "${target_user}" -- /usr/bin/env \ HOME="${target_home}" \ XDG_RUNTIME_DIR="${user_runtime}" \ DBUS_SESSION_BUS_ADDRESS="unix:path=${user_runtime}/bus" \ "${runtime_python}" "${target_runtime}/jwebot_release_update.py" ) printf '%s\n' "${release_update_output}" jwebot_log "release settlement ready" jwebot_log "deployment ready version=$(cat "${source_dir}/VERSION")" jwebot_log "backup active path=${backup_dir}" jwebot_log "legacy OCR inactive" jwebot_log "edge active health=ok" jwebot_log "adaptive active restarts=${restarts_after}" jwebot_log "database ready quick_check=${db_check:-absent}" jwebot_progress 8 8 "deployment ready"