bot / deploy / restore-fleet.sh
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
#!/bin/sh
# deploy/restore-fleet.sh — put a deploy/backup-fleet.sh archive onto a host.
# Use it to move the fleet to a new VPS, or to roll this one back.
#
#   sh deploy/restore-fleet.sh <archive.tar.gz> <ssh-host>          # plan only
#   sh deploy/restore-fleet.sh <archive.tar.gz> <ssh-host> --yes    # do it
#
# IT PLANS BY DEFAULT AND WRITES ONLY WITH --yes. A restore overwrites identity
# seeds, and a bot that loses its seed is a different person: every friend link
# it ever handed out is dead and its leaderboard receipts stop verifying. So the
# default run touches nothing — it prints what would change, and flags any seed
# already on the host that DIFFERS from the archive's.
#
#   --yes            actually write
#   --keep-seeds     restore everything EXCEPT the seeds — the right flag when
#                    the host's identities are the ones to keep and you only
#                    want models / configs / social state back
#   --no-start       restore but leave the units stopped
#
# THE CODE IS NOT IN THE ARCHIVE, ON PURPOSE. dist/ is built on your machine
# (the VPS never builds ClojureScript) and node_modules is resolved by npm, so
# a NEW host needs this first:
#
#   rsync -az --files-from=<(git ls-files) ./ <host>:/opt/ardegazu-bot/
#   ssh <host> 'sh /opt/ardegazu-bot/deploy/install-fleet.sh'   # npm + units
#
# The archive does carry meta/code/{.,stack-a,stack-b}/package*.json — one pair
# per npm tree — and this script diffs them against the host's so you know
# whether the host resolved the SAME sha-pinned kits the backup was taken
# against, in both libp2p generations.
set -eu

ARCHIVE=""; HOST=""; DO_IT=0; KEEP_SEEDS=0; START=1
for a in "$@"; do
  case "$a" in
    --yes) DO_IT=1 ;;
    --keep-seeds) KEEP_SEEDS=1 ;;
    --no-start) START=0 ;;
    -h|--help) sed -n '2,30p' "$0"; exit 0 ;;
    -*) echo "unknown option: $a" >&2; exit 2 ;;
    *) if [ -z "$ARCHIVE" ]; then ARCHIVE="$a"; else HOST="$a"; fi ;;
  esac
done
[ -n "$ARCHIVE" ] && [ -n "$HOST" ] || {
  echo "usage: restore-fleet.sh <archive.tar.gz> <ssh-host> [--yes] [--keep-seeds] [--no-start]" >&2
  exit 2; }
[ -f "$ARCHIVE" ] || { echo "no such archive: $ARCHIVE" >&2; exit 2; }
tar tzf "$ARCHIVE" meta/MANIFEST >/dev/null 2>&1 || {
  echo "$ARCHIVE carries no meta/MANIFEST — not a backup-fleet.sh archive" >&2; exit 2; }

echo "== the archive was taken from:"
tar xzOf "$ARCHIVE" meta/MANIFEST | sed 's/^/   /'
echo

# Upload first: the archive cannot share stdin with the remote script. mode 600
# on the far side too — it holds the seeds.
REMOTE_TMP=$(ssh "$HOST" 'umask 077; mktemp -d')
[ -n "$REMOTE_TMP" ] || { echo "could not make a temp dir on $HOST" >&2; exit 1; }
# shellcheck disable=SC2064  # expand REMOTE_TMP now, so the trap cleans the right dir
trap "ssh '$HOST' 'rm -rf \"$REMOTE_TMP\"' >/dev/null 2>&1 || true" EXIT INT TERM
echo "== uploading $(du -h "$ARCHIVE" | cut -f1) to ${HOST}…"
# shellcheck disable=SC2029  # REMOTE_TMP is meant to expand HERE, client-side
ssh "$HOST" "umask 077; cat > '$REMOTE_TMP/fleet.tar.gz'" < "$ARCHIVE"

# shellcheck disable=SC2029  # the flags are meant to expand here, client-side
ssh "$HOST" "ARCHIVE='$REMOTE_TMP/fleet.tar.gz' DO_IT=$DO_IT KEEP_SEEDS=$KEEP_SEEDS START=$START sh -s" <<'REMOTE'
set -eu
STATE=/var/lib/ardegazu-bot
ETC=/etc/ardegazu-bot
OPT=/opt/ardegazu-bot
UNITS=/etc/systemd/system
BOT_USER=ardegazu-bot

TMP=$(mktemp -d)
trap 'rm -rf "$TMP"' EXIT INT TERM
umask 077
tar xzf "$ARCHIVE" -C "$TMP"

BOTS=$(cd "$TMP" && ls -1 | grep -v '^meta$' | grep -v '^models$' | tr '\n' ' ')
echo "== bots in the archive: $BOTS"
echo

# ---- what would change -------------------------------------------------------
echo "== seeds"
for b in $BOTS; do
  new=$(sha256sum "$TMP/$b/seed" 2>/dev/null | cut -c1-16 || echo "-")
  if [ -f "$STATE/$b/seed" ]; then
    cur=$(sha256sum "$STATE/$b/seed" | cut -c1-16)
    if [ "$cur" = "$new" ]; then
      printf '   %-10s identical (%s…)\n' "$b" "$cur"
    else
      printf '   %-10s DIFFERS  host=%s… archive=%s…  <- restoring CHANGES this identity\n' "$b" "$cur" "$new"
    fi
  else
    printf '   %-10s absent on host, would be created (%s…)\n' "$b" "$new"
  fi
done
[ "$KEEP_SEEDS" = 1 ] && echo "   (--keep-seeds: no seed will be written)"
echo

echo "== code pins"
CODE_SEEN=0
for d in . stack-a stack-b; do
  h="$OPT/$d/package.json"; a="$TMP/meta/code/$d/package.json"
  [ -f "$h" ] && [ -f "$a" ] || continue
  CODE_SEEN=1
  if diff -q "$h" "$a" >/dev/null 2>&1; then
    echo "   host $d/package.json matches the archive's — same sha-pinned kits"
  else
    echo "   host $d/package.json DIFFERS from the archive's:"
    diff "$h" "$a" | sed 's/^/     /' || true
    echo "     (deploy the matching code before starting, or the state may not fit)"
  fi
done
if [ "$CODE_SEEN" != 1 ]; then
  echo "   no code on the host yet — rsync it and run install-fleet.sh first (see --help)"
fi
echo

if [ "$DO_IT" != 1 ]; then
  echo "== PLAN ONLY. Nothing was written. Re-run with --yes to apply."
  exit 0
fi

# ---- apply -------------------------------------------------------------------
echo "== stopping units"
systemctl stop ardegazu-trainer.timer 2>/dev/null || true
systemctl stop ardegazu-trainer.service 2>/dev/null || true
for b in $BOTS; do
  systemctl stop "ardegazu-trainer@$b.timer" 2>/dev/null || true
  systemctl stop "ardegazu-bot@$b" 2>/dev/null || true
done

id "$BOT_USER" >/dev/null 2>&1 || useradd -r -s /usr/sbin/nologin "$BOT_USER"

echo "== restoring units and configs"
mkdir -p "$ETC"
[ -d "$TMP/meta/etc" ] && cp -a "$TMP/meta/etc/." "$ETC/" || true
for u in "$TMP"/meta/systemd/*; do [ -e "$u" ] && cp -a "$u" "$UNITS/"; done
systemctl daemon-reload

echo "== restoring state into $STATE"
mkdir -p "$STATE"
if [ "$KEEP_SEEDS" = 1 ]; then
  tar xzf "$ARCHIVE" -C "$STATE" --exclude='meta' --exclude='meta/*' --exclude='*/seed'
else
  tar xzf "$ARCHIVE" -C "$STATE" --exclude='meta' --exclude='meta/*'
fi
chown -R "$BOT_USER:$BOT_USER" "$STATE"
for b in $BOTS; do [ -f "$STATE/$b/seed" ] && chmod 600 "$STATE/$b/seed"; done

echo "== verifying restored seeds"
for b in $BOTS; do
  [ -f "$STATE/$b/seed" ] || { printf '   %-10s MISSING\n' "$b"; continue; }
  want=$(sha256sum "$TMP/$b/seed" 2>/dev/null | cut -c1-16 || echo "-")
  got=$(sha256sum "$STATE/$b/seed" | cut -c1-16)
  if [ "$KEEP_SEEDS" = 1 ]; then
    printf '   %-10s kept host seed (%s…)\n' "$b" "$got"
  elif [ "$want" = "$got" ]; then
    printf '   %-10s ok (%s…)\n' "$b" "$got"
  else
    printf '   %-10s MISMATCH after restore — host=%s… archive=%s…\n' "$b" "$got" "$want"
  fi
done

if [ "$START" = 1 ]; then
  echo "== starting, staggered"
  for b in $BOTS; do
    systemctl enable "ardegazu-bot@$b" >/dev/null 2>&1 || true
    systemctl start "ardegazu-bot@$b"
    sleep 8
  done
  for b in $BOTS; do
    systemctl start "ardegazu-trainer@$b.timer" 2>/dev/null \
      || systemctl start ardegazu-trainer.timer 2>/dev/null || true
  done
  sleep 10
  echo "== state"
  for b in $BOTS; do
    printf '   %-10s %-9s restarts=%s\n' "$b" \
      "$(systemctl is-active "ardegazu-bot@$b" 2>/dev/null || echo unknown)" \
      "$(systemctl show -p NRestarts --value "ardegazu-bot@$b" 2>/dev/null || echo '?')"
  done
  for b in $BOTS; do
    printf '   trainer@%-8s %s\n' "$b" "$(systemctl is-active "ardegazu-trainer@$b.timer" 2>/dev/null || echo none)"
  done
else
  echo "== --no-start: units left stopped"
fi
echo
echo "== done. Check identities with:  journalctl -u 'ardegazu-bot@*' | grep 'suite identity'"
REMOTE

static mirror of HEAD · about · clone: git clone https://git.ardegazu.ro/bot.git