Files
JWeBot/deploy/scripts/deploy-jwebot-production.sh
T
JWeBot Contributors 127fea8cd0
check / local-gate (push) Failing after 1m7s
Release beta 1.0.4 update settlement
2026-10-16 16:10:19 +08:00

643 lines
21 KiB
Bash
Executable File

#!/usr/bin/env bash
set -Eeuo pipefail
source_dir=${1:?usage: deploy-jwebot-production.sh SOURCE_DIR [TARGET_USER]}
target_user=${2:-jwebot}
[[ "${target_user}" =~ ^[a-z_][a-z0-9_-]{0,31}$ ]] || {
echo "[JWeBot] error: invalid target user" >&2
exit 1
}
target_uid=$(id -u "${target_user}")
target_gid=$(id -g "${target_user}")
target_home=$(getent passwd "${target_user}" | cut -d: -f6)
user_runtime="/run/user/${target_uid}"
runtime_python=${JWEBOT_RUNTIME_PYTHON:-/opt/jwebot/venv/bin/python}
require_wechat_ready=${REQUIRE_WECHAT_READY:-1}
wechat_ready_timeout_seconds=${WECHAT_READY_TIMEOUT_SECONDS:-30}
health_settle_seconds=${HEALTH_SETTLE_SECONDS:-8}
backup_root=${JWEBOT_PRODUCTION_BACKUP_ROOT:-/var/backups/jwebot-production}
timestamp=$(date +%Y%m%d-%H%M%S)
backup_dir="${backup_root}/${timestamp}-$$"
preflight_rendered_dir=""
target_runtime="${target_home}/.local/lib/wechat-health-watch"
target_user_units="${target_home}/.config/systemd/user"
target_user_config="${target_home}/.config/jwebot"
target_edge_state="${target_home}/.local/state/jwebot-edge"
target_adaptive_state="${target_home}/.local/state/wechat-adaptive-ocr"
root_ocr=/usr/local/sbin/wechat-local-ocr
root_bridge=/usr/local/sbin/wechat-gui-bridge
root_recorder_unit=/etc/systemd/system/wechat-local-ocr-recorder.service
root_responder_unit=/etc/systemd/system/wechat-local-ocr-responder.service
root_defaults=/etc/default/wechat-local-ocr
system_state_file="${backup_dir}/metadata/system-state.tsv"
user_state_file="${backup_dir}/metadata/user-state.tsv"
armed=0
success=0
system_units=(
wechat-local-ocr-recorder.service
wechat-local-ocr-responder.service
)
user_units=(
jwebot-edge.service
wechat-ocr-adaptive-recorder.service
wechat-health-watch.timer
jwebot-edge-memory-sync.timer
wechat-health-watch.service
jwebot-edge-memory-sync.service
)
jwebot_log() {
printf '[JWeBot] %s\n' "$*"
}
jwebot_progress() {
local step=$1
local total=$2
local label=$3
local width=20
local filled=$((step * width / total))
local empty=$((width - filled))
local bar
printf -v bar '%*s' "${filled}" ''
bar=${bar// /#}
printf -v padding '%*s' "${empty}" ''
padding=${padding// /-}
printf '[JWeBot] [%s%s] %3d%% %s\n' \
"${bar}" "${padding}" "$((step * 100 / total))" "${label}"
}
if [[ "${EUID}" -ne 0 ]]; then
echo "[JWeBot] error: production deploy must run as root" >&2
exit 1
fi
if [[ -z "${target_home}" || ! -d "${target_home}" ]]; then
echo "[JWeBot] error: target user home is unavailable" >&2
exit 1
fi
jwebot_log "production deploy starting version=$(cat "${source_dir}/VERSION")"
jwebot_progress 1 8 "preflight"
user_systemctl() {
/usr/sbin/runuser -u "${target_user}" -- /usr/bin/env \
HOME="${target_home}" \
XDG_RUNTIME_DIR="${user_runtime}" \
DBUS_SESSION_BUS_ADDRESS="unix:path=${user_runtime}/bus" \
systemctl --user "$@"
}
save_system_state() {
: >"${system_state_file}"
local unit enabled active
for unit in "${system_units[@]}"; do
enabled=$(systemctl is-enabled "${unit}" 2>/dev/null || true)
active=$(systemctl is-active "${unit}" 2>/dev/null || true)
printf '%s\t%s\t%s\n' "${unit}" "${enabled:-unknown}" "${active:-unknown}" \
>>"${system_state_file}"
done
}
save_user_state() {
: >"${user_state_file}"
local unit enabled active
for unit in "${user_units[@]}"; do
enabled=$(user_systemctl is-enabled "${unit}" 2>/dev/null || true)
active=$(user_systemctl is-active "${unit}" 2>/dev/null || true)
printf '%s\t%s\t%s\n' "${unit}" "${enabled:-unknown}" "${active:-unknown}" \
>>"${user_state_file}"
done
}
state_field() {
local file="$1"
local unit="$2"
local field="$3"
awk -F '\t' -v unit="${unit}" -v field="${field}" \
'$1 == unit { print $field; exit }' "${file}"
}
restore_enable_state() {
local scope="$1"
local file="$2"
shift 2
local unit enabled
for unit in "$@"; do
enabled=$(state_field "${file}" "${unit}" 2 2>/dev/null || true)
case "${enabled}" in
enabled)
if [[ "${scope}" == "user" ]]; then
user_systemctl unmask "${unit}" >/dev/null 2>&1 || true
user_systemctl enable "${unit}" >/dev/null 2>&1 || true
else
systemctl unmask "${unit}" >/dev/null 2>&1 || true
systemctl enable "${unit}" >/dev/null 2>&1 || true
fi
;;
enabled-runtime)
if [[ "${scope}" == "user" ]]; then
user_systemctl unmask "${unit}" >/dev/null 2>&1 || true
user_systemctl enable --runtime "${unit}" >/dev/null 2>&1 || true
else
systemctl unmask "${unit}" >/dev/null 2>&1 || true
systemctl enable --runtime "${unit}" >/dev/null 2>&1 || true
fi
;;
disabled)
if [[ "${scope}" == "user" ]]; then
user_systemctl unmask "${unit}" >/dev/null 2>&1 || true
user_systemctl disable "${unit}" >/dev/null 2>&1 || true
else
systemctl unmask "${unit}" >/dev/null 2>&1 || true
systemctl disable "${unit}" >/dev/null 2>&1 || true
fi
;;
masked)
if [[ "${scope}" == "user" ]]; then
user_systemctl mask "${unit}" >/dev/null 2>&1 || true
else
systemctl mask "${unit}" >/dev/null 2>&1 || true
fi
;;
masked-runtime)
if [[ "${scope}" == "user" ]]; then
user_systemctl mask --runtime "${unit}" >/dev/null 2>&1 || true
else
systemctl mask --runtime "${unit}" >/dev/null 2>&1 || true
fi
;;
esac
done
}
restore_active_state() {
local scope="$1"
local file="$2"
shift 2
local unit active
for unit in "$@"; do
active=$(state_field "${file}" "${unit}" 3 2>/dev/null || true)
case "${active}" in
active|activating|reloading)
if [[ "${scope}" == "user" ]]; then
user_systemctl start "${unit}" >/dev/null 2>&1 || true
else
systemctl start "${unit}" >/dev/null 2>&1 || true
fi
;;
esac
done
}
backup_file() {
local source="$1"
local name="$2"
if [[ -e "${source}" || -L "${source}" ]]; then
cp -a "${source}" "${backup_dir}/root/${name}"
: >"${backup_dir}/root/${name}.present"
fi
}
restore_file() {
local target="$1"
local name="$2"
if [[ -f "${backup_dir}/root/${name}.present" ]]; then
rm -rf "${target}"
cp -a "${backup_dir}/root/${name}" "${target}"
else
rm -rf "${target}"
fi
}
copy_directory_backup() {
local source="$1"
local name="$2"
if [[ -d "${source}" ]]; then
cp -a "${source}" "${backup_dir}/user/${name}"
: >"${backup_dir}/user/${name}.present"
fi
}
restore_directory_backup() {
local target="$1"
local name="$2"
rm -rf "${target}"
if [[ -f "${backup_dir}/user/${name}.present" ]]; then
cp -a "${backup_dir}/user/${name}" "${target}"
fi
}
queue_count() {
local directory="$1"
if [[ ! -d "${directory}" ]]; then
printf '0\n'
return
fi
find "${directory}" -maxdepth 1 -type f -print | wc -l
}
assert_queues_empty() {
local adaptive legacy
adaptive=$(queue_count "${target_adaptive_state}/queue")
legacy=$(queue_count /var/lib/wechat-local-ocr/queue)
if [[ "${adaptive}" -ne 0 || "${legacy}" -ne 0 ]]; then
echo "[JWeBot] error: OCR queues must be empty before deployment" >&2
return 1
fi
}
wechat_geometry() {
local identifiers identifier title geometry width height
local class_identifiers name_identifiers
local best_width=0
local best_height=0
local display_environment=(
HOME="${target_home}"
DISPLAY=:1
XAUTHORITY="${target_home}/.Xauthority"
XDG_RUNTIME_DIR="${user_runtime}"
DBUS_SESSION_BUS_ADDRESS="unix:path=${user_runtime}/bus"
)
class_identifiers=$(
/usr/sbin/runuser -u "${target_user}" -- \
/usr/bin/env "${display_environment[@]}" \
/usr/bin/xdotool search --onlyvisible --class wechat \
2>/dev/null || true
)
name_identifiers=$(
/usr/sbin/runuser -u "${target_user}" -- \
/usr/bin/env "${display_environment[@]}" \
/usr/bin/xdotool search --onlyvisible --name '^Weixin$' \
2>/dev/null || true
)
identifiers=$(
printf '%s\n%s\n' "${class_identifiers}" "${name_identifiers}" \
| awk 'NF && !seen[$0]++'
)
for identifier in ${identifiers}; do
title=$(
/usr/sbin/runuser -u "${target_user}" -- \
/usr/bin/env "${display_environment[@]}" \
/usr/bin/xdotool getwindowname "${identifier}" \
2>/dev/null || true
)
[[ "${title}" == "Weixin" ]] || continue
geometry=$(
/usr/sbin/runuser -u "${target_user}" -- \
/usr/bin/env "${display_environment[@]}" \
/usr/bin/xdotool getwindowgeometry --shell "${identifier}" \
2>/dev/null || true
)
width=$(sed -n 's/^WIDTH=//p' <<<"${geometry}" | head -n 1)
height=$(sed -n 's/^HEIGHT=//p' <<<"${geometry}" | head -n 1)
[[ "${width}" =~ ^[0-9]+$ && "${height}" =~ ^[0-9]+$ ]] || continue
if (( width * height > best_width * best_height )); then
best_width=${width}
best_height=${height}
fi
done
if (( best_width > 0 && best_height > 0 )); then
printf 'WIDTH=%s\nHEIGHT=%s\n' "${best_width}" "${best_height}"
fi
}
assert_wechat_ready() {
local geometry="" width="" height=""
local attempt=0
jwebot_log "WeChat readiness starting timeout=${wechat_ready_timeout_seconds}s"
while (( attempt < wechat_ready_timeout_seconds )); do
geometry=$(wechat_geometry)
width=$(sed -n 's/^WIDTH=//p' <<<"${geometry}" | head -n 1)
height=$(sed -n 's/^HEIGHT=//p' <<<"${geometry}" | head -n 1)
if [[ "${width}" =~ ^[0-9]+$ && "${height}" =~ ^[0-9]+$ ]] \
&& (( width >= 700 && height >= 500 ))
then
jwebot_log "WeChat ready geometry=${width}x${height}"
return 0
fi
attempt=$((attempt + 1))
if (( attempt < wechat_ready_timeout_seconds )); then
sleep 1
fi
done
if [[ "${width}" =~ ^[0-9]+$ && "${height}" =~ ^[0-9]+$ ]]; then
echo "[JWeBot] error: WeChat is not ready (${width}x${height})" >&2
else
echo "[JWeBot] error: WeChat window geometry is unavailable" >&2
fi
return 1
}
verify_edge_health() {
local response
response=$(
printf '%s\n' '{"op":"health"}' \
| /usr/sbin/runuser -u "${target_user}" -- /usr/bin/env \
HOME="${target_home}" \
"${runtime_python}" "${target_runtime}/jwebot_edge.py" respond
)
printf '%s' "${response}" \
| "${runtime_python}" -c \
'import json,sys; value=json.load(sys.stdin); raise SystemExit(0 if value.get("status") == "ok" else 1)'
}
render_root_configuration() {
local output_dir="$1"
sed \
-e "s/^User=.*/User=${target_user}/" \
-e "s/^Group=.*/Group=${target_user}/" \
-e "s#/opt/jwebot/venv/bin/python#${runtime_python}#g" \
"${source_dir}/deploy/systemd/system/wechat-local-ocr-recorder.service" \
>"${output_dir}/wechat-local-ocr-recorder.service"
sed \
-e "s#/opt/jwebot/venv/bin/python#${runtime_python}#g" \
"${source_dir}/deploy/systemd/system/wechat-local-ocr-responder.service" \
>"${output_dir}/wechat-local-ocr-responder.service"
sed \
-e "s#^JWEBOT_RUNTIME_USER=.*#JWEBOT_RUNTIME_USER=${target_user}#" \
-e "s#^JWEBOT_RUNTIME_HOME=.*#JWEBOT_RUNTIME_HOME=${target_home}#" \
"${source_dir}/deploy/defaults/wechat-local-ocr.production.default" \
>"${output_dir}/wechat-local-ocr.default"
}
rollback() {
local status=$?
trap - EXIT HUP INT TERM
if [[ -n "${preflight_rendered_dir}" ]]; then
rm -rf "${preflight_rendered_dir}"
fi
if [[ "${armed}" -eq 1 && "${success}" -eq 0 ]]; then
jwebot_log "rollback starting backup=${backup_dir}"
set +e
user_systemctl stop \
wechat-ocr-adaptive-recorder.service \
jwebot-edge.service \
wechat-health-watch.timer \
jwebot-edge-memory-sync.timer \
>/dev/null 2>&1
systemctl disable --now \
wechat-local-ocr-recorder.service \
wechat-local-ocr-responder.service \
>/dev/null 2>&1
restore_file "${root_ocr}" wechat-local-ocr
restore_file "${root_bridge}" wechat-gui-bridge
restore_file "${root_recorder_unit}" wechat-local-ocr-recorder.service
restore_file "${root_responder_unit}" wechat-local-ocr-responder.service
restore_file "${root_defaults}" wechat-local-ocr.default
restore_directory_backup "${target_runtime}" runtime
restore_directory_backup "${target_user_units}" systemd-user
restore_directory_backup "${target_user_config}" jwebot-config
if [[ -f "${backup_dir}/state/edge.sqlite3" ]]; then
install -d -o "${target_uid}" -g "${target_gid}" -m 0700 \
"${target_edge_state}"
rm -f \
"${target_edge_state}/edge.sqlite3" \
"${target_edge_state}/edge.sqlite3-wal" \
"${target_edge_state}/edge.sqlite3-shm"
install -o "${target_uid}" -g "${target_gid}" -m 0600 \
"${backup_dir}/state/edge.sqlite3" \
"${target_edge_state}/edge.sqlite3"
fi
systemctl daemon-reload
user_systemctl daemon-reload
restore_enable_state system "${system_state_file}" "${system_units[@]}"
restore_enable_state user "${user_state_file}" "${user_units[@]}"
restore_active_state user "${user_state_file}" "${user_units[@]}"
restore_active_state system "${system_state_file}" "${system_units[@]}"
systemctl reset-failed
user_systemctl reset-failed
echo "[JWeBot] rollback ready backup=${backup_dir}" >&2
fi
exit "${status}"
}
trap rollback EXIT HUP INT TERM
jwebot_log "preflight starting"
test -x "${runtime_python}"
for path in \
"${source_dir}/wechat_local_ocr.py" \
"${source_dir}/wechat-gui-bridge" \
"${source_dir}/jwebot_release_update.py" \
"${source_dir}/deploy/scripts/install-wechat-health-watch.sh" \
"${source_dir}/deploy/systemd/system/wechat-local-ocr-recorder.service" \
"${source_dir}/deploy/systemd/system/wechat-local-ocr-responder.service" \
"${source_dir}/deploy/defaults/wechat-local-ocr.production.default"
do
test -f "${path}"
done
bash -n "${source_dir}/wechat-gui-bridge"
sh -n "${source_dir}/deploy/scripts/install-wechat-health-watch.sh"
"${runtime_python}" -m py_compile \
"${source_dir}/wechat_local_ocr.py" \
"${source_dir}/jwebot_release_update.py"
preflight_rendered_dir=$(mktemp -d)
render_root_configuration "${preflight_rendered_dir}"
systemd-analyze verify \
"${preflight_rendered_dir}/wechat-local-ocr-recorder.service" \
"${preflight_rendered_dir}/wechat-local-ocr-responder.service"
rm -rf "${preflight_rendered_dir}"
preflight_rendered_dir=""
assert_queues_empty
if [[ "${require_wechat_ready}" -eq 1 ]]; then
assert_wechat_ready
fi
jwebot_log "preflight ready"
jwebot_progress 2 8 "backup"
jwebot_log "backup starting path=${backup_dir}"
install -d -o root -g root -m 0700 \
"${backup_root}" \
"${backup_dir}" \
"${backup_dir}/root" \
"${backup_dir}/user" \
"${backup_dir}/state" \
"${backup_dir}/metadata"
save_system_state
save_user_state
backup_file "${root_ocr}" wechat-local-ocr
backup_file "${root_bridge}" wechat-gui-bridge
backup_file "${root_recorder_unit}" wechat-local-ocr-recorder.service
backup_file "${root_responder_unit}" wechat-local-ocr-responder.service
backup_file "${root_defaults}" wechat-local-ocr.default
copy_directory_backup "${target_runtime}" runtime
copy_directory_backup "${target_user_units}" systemd-user
copy_directory_backup "${target_user_config}" jwebot-config
jwebot_log "backup ready path=${backup_dir}"
armed=1
jwebot_progress 3 8 "stop services"
jwebot_log "cutover starting"
user_systemctl stop \
wechat-ocr-adaptive-recorder.service \
jwebot-edge.service \
wechat-health-watch.timer \
jwebot-edge-memory-sync.timer \
wechat-health-watch.service \
jwebot-edge-memory-sync.service \
>/dev/null 2>&1 || true
systemctl stop \
wechat-local-ocr-responder.service \
wechat-local-ocr-recorder.service \
>/dev/null 2>&1 || true
assert_queues_empty
jwebot_log "services stopped queues=empty"
if [[ -f "${target_edge_state}/edge.sqlite3" ]]; then
"${runtime_python}" - \
"${target_edge_state}/edge.sqlite3" \
"${backup_dir}/state/edge.sqlite3" <<'PY'
import sqlite3
import sys
source = sqlite3.connect(f"file:{sys.argv[1]}?mode=ro", uri=True)
target = sqlite3.connect(sys.argv[2])
try:
if source.execute("PRAGMA quick_check").fetchone()[0] != "ok":
raise SystemExit("source SQLite quick_check failed")
source.backup(target)
if target.execute("PRAGMA quick_check").fetchone()[0] != "ok":
raise SystemExit("backup SQLite quick_check failed")
target.commit()
finally:
target.close()
source.close()
PY
fi
jwebot_log "database backup ready"
jwebot_progress 4 8 "install root runtime"
jwebot_log "runtime install starting"
rendered_recorder_unit="${backup_dir}/metadata/wechat-local-ocr-recorder.service"
rendered_responder_unit="${backup_dir}/metadata/wechat-local-ocr-responder.service"
rendered_defaults="${backup_dir}/metadata/wechat-local-ocr.default"
render_root_configuration "${backup_dir}/metadata"
install -o root -g root -m 0755 \
"${source_dir}/wechat_local_ocr.py" "${root_ocr}"
install -o root -g root -m 0755 \
"${source_dir}/wechat-gui-bridge" "${root_bridge}"
install -o root -g root -m 0644 \
"${rendered_recorder_unit}" \
"${root_recorder_unit}"
install -o root -g root -m 0644 \
"${rendered_responder_unit}" \
"${root_responder_unit}"
install -o root -g root -m 0644 \
"${rendered_defaults}" \
"${root_defaults}"
systemctl daemon-reload
systemctl disable --now \
wechat-local-ocr-recorder.service \
wechat-local-ocr-responder.service
jwebot_log "runtime install ready"
jwebot_progress 5 8 "install user runtime"
jwebot_log "user runtime install starting"
(
cd "${target_home}"
/usr/sbin/runuser -u "${target_user}" -- /usr/bin/env \
HOME="${target_home}" \
XDG_RUNTIME_DIR="${user_runtime}" \
DBUS_SESSION_BUS_ADDRESS="unix:path=${user_runtime}/bus" \
START_TIMER=1 \
HEALTH_SETTLE_SECONDS="${health_settle_seconds}" \
JWEBOT_RUNTIME_PYTHON="${runtime_python}" \
/bin/sh "${source_dir}/deploy/scripts/install-wechat-health-watch.sh" \
"${source_dir}"
)
jwebot_log "user runtime install ready"
jwebot_progress 6 8 "verify services"
systemctl disable --now \
wechat-local-ocr-recorder.service \
wechat-local-ocr-responder.service
test "$(
systemctl is-enabled wechat-local-ocr-recorder.service 2>/dev/null || true
)" = "disabled"
test "$(
systemctl is-enabled wechat-local-ocr-responder.service 2>/dev/null || true
)" = "disabled"
if systemctl is-active --quiet wechat-local-ocr-recorder.service \
|| systemctl is-active --quiet wechat-local-ocr-responder.service
then
echo "[JWeBot] error: legacy OCR unexpectedly remained active" >&2
exit 1
fi
user_systemctl is-enabled jwebot-edge.service \
| grep -qx enabled
user_systemctl is-enabled wechat-ocr-adaptive-recorder.service \
| grep -qx enabled
user_systemctl is-active --quiet jwebot-edge.service
jwebot_log "edge active"
user_systemctl is-active --quiet wechat-ocr-adaptive-recorder.service
jwebot_log "adaptive active"
verify_edge_health
jwebot_log "edge health ready"
restarts_before=$(
user_systemctl show wechat-ocr-adaptive-recorder.service \
-p NRestarts --value
)
sleep "${health_settle_seconds}"
user_systemctl is-active --quiet wechat-ocr-adaptive-recorder.service
restarts_after=$(
user_systemctl show wechat-ocr-adaptive-recorder.service \
-p NRestarts --value
)
test "${restarts_before}" = "${restarts_after}"
jwebot_log "adaptive stability ready restarts=${restarts_after}"
assert_queues_empty
jwebot_log "queues ready depth=0"
if [[ "${require_wechat_ready}" -eq 1 ]]; then
assert_wechat_ready
fi
cmp -s "${source_dir}/wechat_local_ocr.py" "${root_ocr}"
cmp -s "${source_dir}/wechat_local_ocr.py" \
"${target_runtime}/wechat_local_ocr.py"
if [[ -f "${target_edge_state}/edge.sqlite3" ]]; then
db_check=$(
/usr/sbin/runuser -u "${target_user}" -- \
/usr/bin/env HOME="${target_home}" \
"${runtime_python}" -c \
'import sqlite3; print(sqlite3.connect("file:'"${target_edge_state}"'/edge.sqlite3?mode=ro", uri=True).execute("pragma quick_check").fetchone()[0])'
)
test "${db_check}" = "ok"
fi
jwebot_log "database ready quick_check=${db_check:-absent}"
jwebot_progress 7 8 "verify backup"
(
cd "${backup_dir}"
find . -type f ! -name SHA256SUMS -print0 \
| LC_ALL=C sort -z \
| xargs -0 sha256sum >SHA256SUMS
sha256sum -c SHA256SUMS >metadata/sha256-verification.txt
)
jwebot_log "backup verification ready"
success=1
armed=0
trap - EXIT HUP INT TERM
jwebot_log "release settlement starting"
release_update_output=$(
/usr/sbin/runuser -u "${target_user}" -- /usr/bin/env \
HOME="${target_home}" \
XDG_RUNTIME_DIR="${user_runtime}" \
DBUS_SESSION_BUS_ADDRESS="unix:path=${user_runtime}/bus" \
"${runtime_python}" "${target_runtime}/jwebot_release_update.py"
)
printf '%s\n' "${release_update_output}"
jwebot_log "release settlement ready"
jwebot_log "deployment ready version=$(cat "${source_dir}/VERSION")"
jwebot_log "backup active path=${backup_dir}"
jwebot_log "legacy OCR inactive"
jwebot_log "edge active health=ok"
jwebot_log "adaptive active restarts=${restarts_after}"
jwebot_log "database ready quick_check=${db_check:-absent}"
jwebot_progress 8 8 "deployment ready"