1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168 | #!/bin/sh
# deploy/backup-fleet.sh — ONE tarball that can rebuild this fleet somewhere
# else, or roll it back to the day it was taken. Run it from your machine:
#
# sh deploy/backup-fleet.sh <ssh-host> [out-dir] [--with-episodes]
#
# Restore with deploy/restore-fleet.sh. Nothing is written on the server: the
# archive is streamed to stdout over ssh, so no copy is left behind.
#
# WHAT GOES IN, AND WHY THE TIERS. The state directory is ~1.4 GB, and ~99% of
# that is `episodes/` — the RL training JSONL, which the trainer ingests into
# `models/` and which the fleet regenerates by playing. Everything you cannot
# get back any other way is under 6 MB, so that is the default:
#
# <bot>/seed 44 bytes, and IRREPLACEABLE — the Ed25519 seed IS the
# bot's suite identity. Lose it and the bot is a new person:
# every friend link it ever handed out is dead, and its
# leaderboard receipts no longer verify.
# <bot>/kv.json ~1 MB — the social store: friends, blocks, inbox cursors,
# seen-envelope ids. Recoverable only by re-befriending.
# <bot>/rooms/ ~0.5 MB — OrbitDB/Helia data for joined rooms. Replicable
# from peers, but restoring it saves a cold resync.
# <bot>/models/ ~80 KB each, and PRECIOUS — that bot's own trained weights,
# its pending candidates and its ingest cursor. Every bot has
# an independent brain, so these do NOT interchange.
# (A pre-split host instead has one shared models/ at the
# state root; both shapes are captured.)
# meta/etc/ the per-bot configs, including the sibling friend links.
# meta/systemd/ the unit files, so a new host needs no hand-written units.
# meta/code/ package.json + package-lock.json for the repo root AND for
# BOTH stack roots (stack-a/, stack-b/) — the sha-pinned
# kits, so `install-fleet.sh` on the new host resolves the
# SAME kit builds, one npm tree per libp2p generation.
# meta/MANIFEST what was taken, when, from where, and the seed checksums.
#
# --with-episodes adds <bot>/episodes/ (~1.4 GB). Worth it only if you want
# the trainer to be able to re-derive weights from the raw
# matches rather than trusting models/.
#
# The seeds are secrets: the tarball lands mode 0600. Keep it out of any repo.
set -eu
HOST=""
OUT_DIR="."
WITH_EPISODES=0
for a in "$@"; do
case "$a" in
--with-episodes) WITH_EPISODES=1 ;;
-h|--help) sed -n '2,34p' "$0"; exit 0 ;;
-*) echo "unknown option: $a" >&2; exit 2 ;;
*) if [ -z "$HOST" ]; then HOST="$a"; else OUT_DIR="$a"; fi ;;
esac
done
[ -n "$HOST" ] || { echo "usage: backup-fleet.sh <ssh-host> [out-dir] [--with-episodes]" >&2; exit 2; }
[ -d "$OUT_DIR" ] || { echo "no such directory: $OUT_DIR" >&2; exit 2; }
STAMP=$(date -u +%Y%m%d-%H%M%S)
OUT="$OUT_DIR/ardegazu-fleet-$STAMP.tar.gz"
# umask FIRST: the redirect below creates the file, and it must never exist
# even momentarily as world-readable — it is about to hold five identity seeds.
umask 077
echo "backing up $HOST (episodes: $([ "$WITH_EPISODES" = 1 ] && echo included || echo excluded))…" >&2
# `sh -s` reads this script on the remote's stdin; the archive comes back on
# stdout. Keep the two straight — nothing here may read stdin.
# shellcheck disable=SC2029 # WITH_EPISODES is meant to expand HERE, client-side
ssh "$HOST" "WITH_EPISODES=$WITH_EPISODES sh -s" > "$OUT" <<'REMOTE'
set -eu
STATE=/var/lib/ardegazu-bot
[ -d "$STATE" ] || STATE=/var/lib/private/ardegazu-bot
ETC=/etc/ardegazu-bot
OPT=/opt/ardegazu-bot
UNITS=/etc/systemd/system
[ -d "$STATE" ] || { echo "no fleet state directory on this host" >&2; exit 1; }
TMP=$(mktemp -d)
trap 'rm -rf "$TMP"' EXIT INT TERM
mkdir -p "$TMP/meta/etc" "$TMP/meta/systemd" "$TMP/meta/code"
# every directory under the state root except the shared models/ is a bot
BOTS=$(cd "$STATE" && ls -1 | grep -v '^models$' | tr '\n' ' ')
[ -d "$ETC" ] && cp -a "$ETC/." "$TMP/meta/etc/" 2>/dev/null || true
for u in ardegazu-bot@.service ardegazu-trainer@.service ardegazu-trainer@.timer \
ardegazu-trainer.service ardegazu-trainer.timer; do
[ -f "$UNITS/$u" ] && cp -a "$UNITS/$u" "$TMP/meta/systemd/" || true
done
# the root plus the two stack roots — one manifest pair per npm tree, kept in
# the archive under the same relative shape so a restore can diff them one by one
for d in . stack-a stack-b; do
mkdir -p "$TMP/meta/code/$d"
for f in package.json package-lock.json; do
[ -f "$OPT/$d/$f" ] && cp -a "$OPT/$d/$f" "$TMP/meta/code/$d/" || true
done
done
# a kit lives in the stack root that owns its runtime; look in both
kitver() {
p="$OPT/stack-a/node_modules/ardegazu-$1/package.json"
[ -f "$p" ] || p="$OPT/stack-b/node_modules/ardegazu-$1/package.json"
[ -f "$p" ] || { echo "-"; return; }
sed -n 's/.*"version"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$p" | head -1
}
{
echo "# ardegazu fleet backup"
echo "taken_utc: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "source_host: $(hostname)"
echo "state_root: $STATE"
echo "bots: $BOTS"
echo "episodes: $([ "${WITH_EPISODES:-0}" = 1 ] && echo included || echo excluded)"
echo "node: $(command -v node >/dev/null 2>&1 && node -v || echo '-')"
echo
echo "# kit versions the running code resolved to"
for k in peer-kit rooms-kit social-kit train-kit id-kit wallet-kit; do
printf 'kit_%-12s %s\n' "$k:" "$(kitver "$k")"
done
echo
echo "# per bot: unit enabled state, and the seed's sha256 so a restore can be"
echo "# verified without ever printing the seed itself"
for b in $BOTS; do
# `is-enabled`/`is-active` PRINT the state and still exit non-zero when it
# is disabled or inactive, so `|| echo unknown` appends a second line and
# shreds the printf below. Take the first line, default only if empty.
en=$(systemctl is-enabled "ardegazu-bot@$b" 2>/dev/null | head -1); [ -n "$en" ] || en=unknown
ac=$(systemctl is-active "ardegazu-bot@$b" 2>/dev/null | head -1); [ -n "$ac" ] || ac=unknown
tr=$(systemctl is-enabled "ardegazu-trainer@$b.timer" 2>/dev/null | head -1); [ -n "$tr" ] || tr=none
sd="-"
[ -f "$STATE/$b/seed" ] && sd=$(sha256sum "$STATE/$b/seed" | cut -c1-64)
printf 'bot %-10s enabled=%-9s active=%-9s trainer=%-9s seed_sha256=%s\n' "$b" "$en" "$ac" "$tr" "$sd"
done
echo
echo "trainer_shape: $([ -f /etc/systemd/system/ardegazu-trainer@.timer ] && echo per-bot || echo singleton)"
echo
echo "# layout: anything outside meta/ is state, restored under the state root"
} > "$TMP/meta/MANIFEST"
# Build the state member list. Skip what is absent rather than failing: a fresh
# instance has a seed and nothing else yet.
PATHS=""
for b in $BOTS; do
for f in seed kv.json social.json rooms models; do
[ -e "$STATE/$b/$f" ] && PATHS="$PATHS $b/$f"
done
if [ "${WITH_EPISODES:-0}" = 1 ] && [ -d "$STATE/$b/episodes" ]; then
PATHS="$PATHS $b/episodes"
fi
done
[ -d "$STATE/models" ] && PATHS="$PATHS models"
# One stream, two roots: meta/ from the staging dir, state from the state root.
# shellcheck disable=SC2086
tar czf - -C "$TMP" meta -C "$STATE" $PATHS
REMOTE
chmod 600 "$OUT"
echo >&2
echo "wrote $OUT" >&2
ls -l "$OUT" >&2
echo >&2
echo "manifest:" >&2
tar xzOf "$OUT" meta/MANIFEST 2>/dev/null | sed 's/^/ /' >&2
echo >&2
echo "restore with: sh deploy/restore-fleet.sh $OUT <ssh-host>" >&2
|