From 43fef1050485d22dd0ff44a36afc82d696640fea Mon Sep 17 00:00:00 2001
From: Tomasz Szymczyszyn <tomasz.szymczyszyn@shopify.com>
Date: Thu, 15 Oct 2026 12:01:14 +0200
Subject: [PATCH 1/2] Add live checks for the namespace store

Scripts under libsql-server/scripts/namespace-live-checks that run a
built sqld locally and exercise the namespace store against a real
server: a load tool, two focused scenarios (eviction churn and SIGTERM
under load), a 65-check regression matrix over the admin and user APIs
including a primary plus replica pair and mixed versions, and a
side-by-side comparison of two runs.

The races in this area (an entry emptied under a request, a shutdown
with requests in flight, a destroy racing a lookup) are pinned by unit
tests; these scripts show the same races on a running server and let two
binaries be compared before and after a change.
---
 .../scripts/namespace-live-checks/README.md   |  76 ++++++
 .../scripts/namespace-live-checks/compare.py  |  36 +++
 .../scripts/namespace-live-checks/load.py     | 154 ++++++++++++
 .../scripts/namespace-live-checks/regress.sh  | 232 ++++++++++++++++++
 .../scripts/namespace-live-checks/scenario.sh |  67 +++++
 5 files changed, 565 insertions(+)
 create mode 100644 libsql-server/scripts/namespace-live-checks/README.md
 create mode 100755 libsql-server/scripts/namespace-live-checks/compare.py
 create mode 100755 libsql-server/scripts/namespace-live-checks/load.py
 create mode 100755 libsql-server/scripts/namespace-live-checks/regress.sh
 create mode 100755 libsql-server/scripts/namespace-live-checks/scenario.sh

diff --git a/libsql-server/scripts/namespace-live-checks/README.md b/libsql-server/scripts/namespace-live-checks/README.md
new file mode 100644
index 0000000000..843fcc6241
--- /dev/null
+++ b/libsql-server/scripts/namespace-live-checks/README.md
@@ -0,0 +1,76 @@
+# Namespace live checks
+
+Scripts that run a built `sqld` locally and check how the namespace store
+behaves under load, during shutdown, and across the admin API. They exist
+because the interesting failures here are races: an entry emptied under a
+request, a shutdown while requests are in flight, a destroy racing a lookup.
+Unit tests pin each race deterministically; these scripts show the same
+races against a real server, and let two binaries be compared.
+
+Nothing here touches a remote system. Everything listens on `127.0.0.1`.
+
+## Requirements
+
+`bash` 3.2 or newer, `curl`, `python3`, `lsof`. A debug build is fine and
+widens the race windows, which is what you want.
+
+```
+cargo build -p libsql-server --bin sqld
+```
+
+## Scripts
+
+| Script | What it does |
+|---|---|
+| `load.py` | Keep-alive clients driving `POST /v2/pipeline`; tallies status codes per phase, with an optional SIGTERM or shell action at a given time. |
+| `scenario.sh` | One focused scenario: `evict` (cache capacity far below the hot set) or `sigterm` (signal under load). Prints the tally, the namespace-store counters and the grouped store log lines. |
+| `regress.sh` | A 65-check matrix across the admin and user APIs, one primary and one primary plus replica, optionally mixed versions. Writes `results.tsv`. |
+| `compare.py` | Side-by-side diff of two `results.tsv` files. |
+
+## Comparing two binaries
+
+```
+S=libsql-server/scripts/namespace-live-checks
+$S/scenario.sh target/debug/sqld evict            # expect 0 non-200 answers
+$S/scenario.sh target/debug/sqld sigterm          # expect 503s after the signal, no 404s
+
+$S/regress.sh /path/to/old/sqld old /path/to/new/sqld
+$S/regress.sh /path/to/new/sqld new /path/to/old/sqld
+$S/compare.py namespace-live-checks-out/old/results.tsv namespace-live-checks-out/new/results.tsv old new
+```
+
+`regress.sh` writes server logs next to the results (`<label>/p.log`,
+`r.log`, `d.log`). When a line differs, those logs say why.
+
+## What a healthy run looks like
+
+- `scenario.sh ... evict`: every answer is 200. `libsql_server_namespace_emptied_entry`
+  is well above zero (the race happened and was retried) and
+  `libsql_server_namespace_emptied_entry_gave_up` is absent (never incremented).
+- `scenario.sh ... sigterm`: after the signal only 503 `Namespace store has shutdown`,
+  then connection errors once the port closes. No 404.
+- `regress.sh`: unknown namespaces answer 404 everywhere; deleted namespaces
+  answer 404 and their directory is gone; data survives a restart; the
+  replica catches up, proxies writes, and resumes after a primary restart.
+
+## Reading the output
+
+- A counter that was never incremented is not listed by `/metrics`; absent
+  means zero.
+- `conn_error` in a tally covers refused, reset and timed-out connections.
+  After a SIGTERM it is the expected tail. Elsewhere it usually means the
+  server closed the connection after an error answer.
+- The admin port is gone after a SIGTERM; `scenario.sh` reads `/metrics`
+  only when the server is still up.
+- Under eviction churn the server logs `unable to remove .sentinel file`
+  for namespaces reopened while the evicted instance is still shutting
+  down. That is independent of the namespace store and shows on every
+  version.
+
+## Ports
+
+Defaults: user HTTP 18080, admin 19090, gRPC 15001; replica 18081 and
+19091. Override with `PORT`/`ADMIN` (`scenario.sh`) or `P_HTTP`, `P_ADMIN`,
+`P_GRPC`, `R_HTTP`, `R_ADMIN` (`regress.sh`). Both scripts refuse to start
+when a port is already in use, because a stale server answering the health
+check would silently invalidate the run.
diff --git a/libsql-server/scripts/namespace-live-checks/compare.py b/libsql-server/scripts/namespace-live-checks/compare.py
new file mode 100755
index 0000000000..8afbf6db13
--- /dev/null
+++ b/libsql-server/scripts/namespace-live-checks/compare.py
@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+"""Print two regress.sh result files side by side and mark the lines that differ.
+
+Usage: compare.py <a/results.tsv> <b/results.tsv> [label-a] [label-b]
+"""
+import csv
+import sys
+
+
+def load(path):
+    out = {}
+    for row in csv.reader(open(path), delimiter="\t"):
+        if len(row) == 3:
+            out[(row[0], row[1])] = row[2]
+    return out
+
+
+def short(v, n):
+    v = v.replace('{"error":', "").replace('"}', "").replace('"', "")
+    return v[: n - 1] + "…" if len(v) > n else v
+
+
+a, b = load(sys.argv[1]), load(sys.argv[2])
+la = sys.argv[3] if len(sys.argv) > 3 else sys.argv[1]
+lb = sys.argv[4] if len(sys.argv) > 4 else sys.argv[2]
+keys = list(dict.fromkeys(list(a.keys()) + list(b.keys())))
+w = 60
+print(f"{'scenario / check':52s} | {la:{w}s} | {lb:{w}s} |")
+print("-" * 52 + "-+-" + "-" * w + "-+-" + "-" * w + "-+")
+differ = 0
+for k in keys:
+    va, vb = a.get(k, "-"), b.get(k, "-")
+    mark = "" if va == vb else " *"
+    differ += bool(mark)
+    print(f"{(k[0] + ' ' + k[1])[:52]:52s} | {short(va, w):{w}s} | {short(vb, w):{w}s} |{mark}")
+print(f"\n{differ} of {len(keys)} checks differ (marked *). Values with ids or counters are expected to differ.")
diff --git a/libsql-server/scripts/namespace-live-checks/load.py b/libsql-server/scripts/namespace-live-checks/load.py
new file mode 100755
index 0000000000..8f3c442d4d
--- /dev/null
+++ b/libsql-server/scripts/namespace-live-checks/load.py
@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+"""Drive a running sqld over Hrana HTTP from several keep-alive clients and
+tally the status codes it answers.
+
+Each thread loops over `POST /v2/pipeline` with one statement, selecting the
+namespace with the `x-namespace` header (`ns0`..`nsN-1`, or a fixed `--ns`).
+An optional action splits the tally into a "before" and an "after" phase:
+`--sigterm-after S --pid PID` sends SIGTERM to a server, `--at S --run CMD`
+runs a shell command. The report lists, per phase, how many responses had
+each status, when each status was first seen after the action, and a few
+distinct bodies for non-200 answers.
+
+Examples:
+  load.py --port 18080 --namespaces 6 --threads 24 --duration 20
+  load.py --port 18080 --namespaces 20 --threads 32 --duration 12 \
+          --sigterm-after 3 --pid $(pgrep -n sqld)
+  load.py --port 18080 --ns ns0 --sql "insert into t values (1)" \
+          --threads 8 --duration 4 --at 1.5 --run "curl -X DELETE ..."
+"""
+import argparse
+import collections
+import http.client
+import json
+import os
+import signal
+import subprocess
+import threading
+import time
+
+p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+p.add_argument("--port", type=int, required=True, help="sqld user HTTP port on 127.0.0.1")
+p.add_argument("--namespaces", type=int, default=1, help="cycle over ns0..nsN-1")
+p.add_argument("--ns", type=str, default=None, help="fixed namespace instead of ns0..nsN-1")
+p.add_argument("--sql", type=str, default="select 1")
+p.add_argument("--threads", type=int, default=16)
+p.add_argument("--duration", type=float, default=10)
+p.add_argument("--sigterm-after", type=float, default=None, help="seconds after start to SIGTERM --pid")
+p.add_argument("--pid", type=int, default=None)
+p.add_argument("--at", type=float, default=None, help="seconds after start to run --run")
+p.add_argument("--run", type=str, default=None, help="shell command to run at --at")
+p.add_argument("--quiet", action="store_true", help="one-line summary only")
+args = p.parse_args()
+
+BODY = json.dumps({"requests": [{"type": "execute", "stmt": {"sql": args.sql}}, {"type": "close"}]})
+HEADERS = {"content-type": "application/json"}
+
+lock = threading.Lock()
+tally = collections.defaultdict(collections.Counter)  # phase -> status -> count
+bodies = collections.defaultdict(set)  # status -> distinct bodies (truncated)
+first_seen = {}  # (phase, status) -> seconds after the action
+action_at = [None]
+stop = threading.Event()
+t0 = time.monotonic()
+
+
+def phase():
+    return "after" if action_at[0] is not None and time.monotonic() >= action_at[0] else "before"
+
+
+def since_action():
+    return None if action_at[0] is None else round(time.monotonic() - action_at[0], 3)
+
+
+def record(status, body=None):
+    ph = phase()
+    with lock:
+        tally[ph][status] += 1
+        first_seen.setdefault((ph, status), since_action())
+        if body is not None and len(bodies[status]) < 3:
+            bodies[status].add(body[:160])
+
+
+def worker(i):
+    conn = None
+    consecutive_conn_errors = 0
+    n = 0
+    while not stop.is_set():
+        ns = args.ns or f"ns{(i + n) % args.namespaces}"
+        n += 1
+        try:
+            if conn is None:
+                conn = http.client.HTTPConnection("127.0.0.1", args.port, timeout=5)
+            conn.request("POST", "/v2/pipeline", body=BODY, headers=dict(HEADERS, **{"x-namespace": ns}))
+            resp = conn.getresponse()
+            data = resp.read()
+            consecutive_conn_errors = 0
+            if resp.status == 200:
+                record(200)
+            else:
+                record(resp.status, data.decode("utf-8", "replace"))
+            if resp.getheader("connection", "").lower() == "close":
+                conn.close()
+                conn = None
+        except Exception as e:  # connection refused, reset, timeout
+            record("conn_error", type(e).__name__)
+            consecutive_conn_errors += 1
+            try:
+                if conn:
+                    conn.close()
+            except Exception:
+                pass
+            conn = None
+            # once the server is gone after the action, stop instead of spinning
+            if phase() == "after" and consecutive_conn_errors >= 20:
+                return
+            time.sleep(0.01)
+
+
+threads = [threading.Thread(target=worker, args=(i,), daemon=True) for i in range(args.threads)]
+for t in threads:
+    t.start()
+
+if args.sigterm_after is not None:
+    time.sleep(args.sigterm_after)
+    action_at[0] = time.monotonic()
+    os.kill(args.pid, signal.SIGTERM)
+    if not args.quiet:
+        print(f"sent SIGTERM to {args.pid} at t={round(action_at[0] - t0, 3)}s", flush=True)
+elif args.at is not None:
+    time.sleep(args.at)
+    action_at[0] = time.monotonic()
+    r = subprocess.run(args.run, shell=True, capture_output=True, text=True)
+    if not args.quiet:
+        print(f"ran action at t={round(action_at[0] - t0, 3)}s -> {r.stdout.strip()[:120]}", flush=True)
+
+deadline = t0 + args.duration
+while time.monotonic() < deadline and any(t.is_alive() for t in threads):
+    time.sleep(0.05)
+stop.set()
+for t in threads:
+    t.join(timeout=2)
+
+
+def fmt(counter):
+    return ", ".join(f"{k}:{v}" for k, v in sorted(counter.items(), key=lambda kv: str(kv[0])))
+
+
+if args.quiet:
+    print(" ".join(f"{ph}{{{fmt(tally[ph])}}}" for ph in ("before", "after") if tally[ph]))
+    raise SystemExit(0)
+
+print("\n=== tally (phase -> status: count) ===")
+for ph in ("before", "after"):
+    if tally[ph]:
+        print(f"{ph:7s} {fmt(tally[ph])}")
+if any(v is not None for v in first_seen.values()):
+    print("\n=== first seen (seconds after the action) ===")
+    for (ph, st), t in sorted(first_seen.items(), key=lambda kv: (kv[1] is None, kv[1] or 0)):
+        if ph == "after":
+            print(f"  {st}: +{t}s")
+print("\n=== distinct bodies for non-200 ===")
+for st, bs in bodies.items():
+    for b in bs:
+        print(f"  {st}: {b}")
diff --git a/libsql-server/scripts/namespace-live-checks/regress.sh b/libsql-server/scripts/namespace-live-checks/regress.sh
new file mode 100755
index 0000000000..208644dfed
--- /dev/null
+++ b/libsql-server/scripts/namespace-live-checks/regress.sh
@@ -0,0 +1,232 @@
+#!/bin/bash
+# Regression matrix for the namespace store, run against a sqld binary.
+#
+# Usage: regress.sh <path/to/sqld> [label] [path/to/other/sqld]
+#
+# Starts servers (one primary; later one primary plus one replica), drives
+# them through the admin and user HTTP APIs, and records one line per check
+# in $OUT/<label>/results.tsv: scenario, check, observed value. Run it once
+# per binary and diff the two results with compare.py.
+#
+# If a second binary is given, S10 pairs this binary as the primary with the
+# other as the replica (mixed versions). Otherwise S10 is skipped.
+#
+# Scenarios: S1 lifecycle and real 404s; S2 checkpoint; S3 fork; S4 create
+# from a dump; S5 shared schema; S6 delete under load; S7 persistence across
+# SIGTERM with and without --snapshot-at-shutdown; S8 SIGTERM under load with
+# namespaces disabled; S9 primary plus replica; S10 mixed versions.
+#
+# Works with bash 3.2. Needs curl, python3 and lsof.
+set -uo pipefail
+HERE=$(cd "$(dirname "$0")" && pwd)
+BIN=$1; LABEL=${2:-$(basename "$1")}; BIN_OTHER=${3:-}
+OTHER=$([ -n "$BIN_OTHER" ] && basename "$BIN_OTHER" || echo none)
+OUT=${OUT:-./namespace-live-checks-out}
+ROOT=$OUT/$LABEL; rm -rf "$ROOT"; mkdir -p "$ROOT"
+RESULTS=$ROOT/results.tsv; : > "$RESULTS"
+P_HTTP=${P_HTTP:-18080}; P_ADMIN=${P_ADMIN:-19090}; P_GRPC=${P_GRPC:-15001}
+R_HTTP=${R_HTTP:-18081}; R_ADMIN=${R_ADMIN:-19091}
+LOAD=$HERE/load.py
+export RUST_LOG=${RUST_LOG:-info}
+
+rec() { printf '%s\t%s\t%s\n' "$1" "$2" "$3" | tee -a "$RESULTS"; }
+
+# ---- server control -------------------------------------------------------
+for port in $P_HTTP $P_ADMIN $P_GRPC $R_HTTP $R_ADMIN; do
+  if lsof -nP -iTCP:$port -sTCP:LISTEN >/dev/null 2>&1; then echo "!! port $port is busy; kill the stale process first"; lsof -nP -iTCP:$port -sTCP:LISTEN; exit 1; fi
+done
+pid_of() { cat "$ROOT/$1.pid" 2>/dev/null; }
+forget() { rm -f "$ROOT/$1.pid"; }
+start() { # start NAME BIN DIR HTTP ADMIN [extra args...]
+  local name=$1 bin=$2 dir=$3 http=$4 admin=$5; shift 5
+  mkdir -p "$dir"
+  "$bin" --db-path "$dir" --http-listen-addr 127.0.0.1:$http --admin-listen-addr 127.0.0.1:$admin --no-welcome "$@" \
+    >>"$ROOT/$name.log" 2>&1 &
+  local pid=$!; echo $pid > "$ROOT/$name.pid"
+  for _ in $(seq 1 200); do
+    kill -0 $pid 2>/dev/null || { echo "!! $name exited at start"; tail -3 "$ROOT/$name.log"; return 1; }
+    curl -sf --max-time 2 -o /dev/null "http://127.0.0.1:$http/health" && return 0; sleep 0.05
+  done
+  echo "!! $name did not come up"; tail -5 "$ROOT/$name.log"; return 1
+}
+stop() { # stop NAME  (SIGTERM, wait)
+  local pid; pid=$(pid_of "$1"); [ -z "$pid" ] && return 0
+  kill "$pid" 2>/dev/null; for _ in $(seq 1 100); do kill -0 "$pid" 2>/dev/null || break; sleep 0.05; done
+  kill -9 "$pid" 2>/dev/null; forget "$1"; sleep 0.2
+}
+stop_all() { for f in "$ROOT"/*.pid; do [ -f "$f" ] && stop "$(basename "$f" .pid)"; done; }
+trap stop_all EXIT
+
+# ---- API helpers (print "STATUS BODY") -------------------------------------
+api() { # api METHOD URL [json]
+  local m=$1 u=$2 d=${3:-}
+  if [ -n "$d" ]; then curl -s --max-time 10 -o "$ROOT/body" -w '%{http_code}' -X "$m" -H 'content-type: application/json' -d "$d" "$u"
+  else curl -s --max-time 10 -o "$ROOT/body" -w '%{http_code}' -X "$m" "$u"; fi
+  printf ' %s' "$(tr -d '\n' < "$ROOT/body" | cut -c1-100)"
+}
+ns_create() { local d=${2:-"{}"}; api POST "http://127.0.0.1:${3:-$P_ADMIN}/v1/namespaces/$1/create" "$d"; }
+ns_delete() { api DELETE "http://127.0.0.1:${2:-$P_ADMIN}/v1/namespaces/$1"; }
+ns_fork()   { api POST "http://127.0.0.1:${3:-$P_ADMIN}/v1/namespaces/$1/fork/$2"; }
+ns_ckpt()   { api POST "http://127.0.0.1:${2:-$P_ADMIN}/v1/namespaces/$1/checkpoint"; }
+ns_config() { api GET  "http://127.0.0.1:${2:-$P_ADMIN}/v1/namespaces/$1/config"; }
+ns_stats()  { api GET  "http://127.0.0.1:${2:-$P_ADMIN}/v1/namespaces/$1/stats"; }
+q() { # q NS SQL [http port]  -> "STATUS firstvalue-or-error"
+  local ns=$1 sql=$2 port=${3:-$P_HTTP}
+  local body; body=$(python3 -c 'import json,sys; print(json.dumps({"requests":[{"type":"execute","stmt":{"sql":sys.argv[1]}},{"type":"close"}]}))' "$sql")
+  local code; code=$(curl -s --max-time 10 -o "$ROOT/body" -w '%{http_code}' -X POST -H 'content-type: application/json' -H "x-namespace: $ns" -d "$body" "http://127.0.0.1:$port/v2/pipeline")
+  local v; v=$(python3 -c '
+import json,sys
+try:
+    r=json.load(open(sys.argv[1]))
+    if "results" in r:
+        x=r["results"][0]
+        if x.get("type")=="ok":
+            rows=x["response"].get("result",{}).get("rows",[])
+            print(rows[0][0]["value"] if rows and rows[0] else "ok")
+        else: print("err:"+x.get("error",{}).get("message","?")[:60])
+    else: print(r.get("error","?")[:60])
+except Exception as e: print("?")' "$ROOT/body")
+  printf '%s %s' "$code" "$v"
+}
+q_code() { q "$@" | cut -d' ' -f1; }
+seed() { q "$1" "create table t(x)" "${2:-$P_HTTP}" >/dev/null; q "$1" "insert into t select value from generate_series(1,${3:-2000})" "${2:-$P_HTTP}" >/dev/null; }
+count() { q "$1" "select count(*) from t" "${2:-$P_HTTP}" | cut -d' ' -f2; }
+wait_count() { # wait_count NS PORT EXPECTED -> prints final count
+  for _ in $(seq 1 40); do c=$(count "$1" "$2"); [ "$c" = "$3" ] && break; sleep 0.2; done; echo "$c"
+}
+load() { python3 $LOAD --quiet --port "$@"; }
+
+echo "##### $LABEL: regression matrix"
+
+# ===========================================================================
+# S1 lifecycle and real 404s
+start p "$BIN" "$ROOT/p" $P_HTTP $P_ADMIN --enable-namespaces --max-active-namespaces 100 || exit 1
+rec S1 "create ns1" "$(ns_create ns1)"
+rec S1 "create ns1 again" "$(ns_create ns1)"
+seed ns1
+rec S1 "query ns1 count" "$(q ns1 'select count(*) from t')"
+rec S1 "config ns1" "$(ns_config ns1 | cut -c1-40)"
+rec S1 "stats ns1" "$(ns_stats ns1 | cut -c1-40)"
+rec S1 "query unknown" "$(q nope 'select 1')"
+rec S1 "config unknown" "$(ns_config nope)"
+rec S1 "stats unknown" "$(ns_stats nope)"
+rec S1 "delete ns1" "$(ns_delete ns1)"
+rec S1 "query deleted" "$(q ns1 'select 1')"
+rec S1 "config deleted" "$(ns_config ns1)"
+rec S1 "dir deleted" "$([ -d "$ROOT/p/dbs/ns1" ] && echo present || echo gone)"
+rec S1 "delete unknown" "$(ns_delete nope)"
+
+# S2 checkpoint: loaded, unloaded (after restart), unknown
+ns_create ns2 >/dev/null; seed ns2
+rec S2 "checkpoint loaded" "$(ns_ckpt ns2)"
+stop p; start p "$BIN" "$ROOT/p" $P_HTTP $P_ADMIN --enable-namespaces --max-active-namespaces 100 || exit 1
+rec S2 "checkpoint unloaded (after restart)" "$(ns_ckpt ns2)"
+rec S2 "query after unloaded checkpoint" "$(q ns2 'select count(*) from t')"
+rec S2 "checkpoint unknown" "$(ns_ckpt nope)"
+rec S2 "query unknown after its checkpoint" "$(q nope 'select 1')"
+
+# S3 fork: hot source, cold source, unknown source, existing destination
+ns_create src >/dev/null; seed src
+rec S3 "fork hot src->dst" "$(ns_fork src dst)"
+rec S3 "dst count" "$(q dst 'select count(*) from t')"
+stop p; start p "$BIN" "$ROOT/p" $P_HTTP $P_ADMIN --enable-namespaces --max-active-namespaces 100 || exit 1
+rec S3 "fork cold src->dst2" "$(ns_fork src dst2)"
+rec S3 "dst2 count" "$(q dst2 'select count(*) from t')"
+rec S3 "fork unknown->x" "$(ns_fork nope x)"
+rec S3 "fork src->dst2 (dst2 loaded)" "$(ns_fork src dst2)"
+stop p; start p "$BIN" "$ROOT/p" $P_HTTP $P_ADMIN --enable-namespaces --max-active-namespaces 100 || exit 1
+rec S3 "fork src->dst (dst exists, unloaded) [pre-existing check]" "$(ns_fork src dst)"
+rec S3 "dst after that: query" "$(q dst 'select count(*) from t')"
+rec S3 "dst after that: config" "$(ns_config dst | cut -c1-40)"
+rec S3 "dst after that: dir" "$([ -d "$ROOT/p/dbs/dst" ] && echo present || echo gone)"
+
+# S4 dump restore via create
+printf 'CREATE TABLE d(x);\nINSERT INTO d VALUES (1),(2),(3);\n' > "$ROOT/dump.sql"
+rec S4 "create from dump" "$(ns_create nsd "{\"dump_url\":\"file://$ROOT/dump.sql\"}")"
+rec S4 "dump rows" "$(q nsd 'select count(*) from d')"
+
+# S5 shared schema (create goes through fork)
+rec S5 "create schema" "$(ns_create schema '{"shared_schema":true}')"
+rec S5 "schema ddl" "$(q schema 'create table s(x)')"
+rec S5 "create linked1 (hot schema)" "$(ns_create linked1 '{"shared_schema_name":"schema"}')"
+rec S5 "linked1 query" "$(q linked1 'select count(*) from s')"
+stop p; start p "$BIN" "$ROOT/p" $P_HTTP $P_ADMIN --enable-namespaces --max-active-namespaces 100 || exit 1
+rec S5 "create linked2 (cold schema)" "$(ns_create linked2 '{"shared_schema_name":"schema"}')"
+rec S5 "linked2 query" "$(q linked2 'select count(*) from s')"
+rec S5 "delete schema with linked dbs" "$(ns_delete schema)"
+
+# S6 delete under load: must be 404 after, never 503, no resurrection
+# the load tool names namespaces ns0..; create ns0 for it instead
+ns_create ns0 >/dev/null; seed ns0
+rec S6 "load tally (ns0)" "$(load $P_HTTP --namespaces 1 --threads 8 --duration 4 --at 1.5 --run "curl -s -o /dev/null -w '%{http_code}' -X DELETE http://127.0.0.1:$P_ADMIN/v1/namespaces/ns0")"
+sleep 0.5
+rec S6 "config after delete-under-load" "$(ns_config ns0)"
+rec S6 "query after delete-under-load" "$(q ns0 'select 1')"
+rec S6 "dir after delete-under-load" "$([ -d "$ROOT/p/dbs/ns0" ] && echo present || echo gone)"
+
+# S7 shutdown persistence (plain, and with --snapshot-at-shutdown)
+for i in 1 2 3; do ns_create pers$i >/dev/null; seed pers$i "$P_HTTP" 1500; done
+stop p; start p "$BIN" "$ROOT/p" $P_HTTP $P_ADMIN --enable-namespaces --max-active-namespaces 100 --snapshot-at-shutdown || exit 1
+rec S7 "counts after restart" "$(count pers1) $(count pers2) $(count pers3)"
+q pers1 "insert into t values (0)" >/dev/null
+stop p; start p "$BIN" "$ROOT/p" $P_HTTP $P_ADMIN --enable-namespaces --max-active-namespaces 100 || exit 1
+rec S7 "counts after snapshot-at-shutdown restart" "$(count pers1) $(count pers2) $(count pers3)"
+rec S7 "shutdown errors in log" "$(grep -c -E 'ERROR.*(shutdown|shutting)' "$ROOT/p.log")"
+stop p
+
+# S8 namespaces disabled: SIGTERM under load on the default namespace
+start d "$BIN" "$ROOT/d" $P_HTTP $P_ADMIN --max-active-namespaces 100 || exit 1
+q default "create table t(x)" >/dev/null
+rec S8 "default ns sigterm tally" "$(python3 $LOAD --quiet --port $P_HTTP --namespaces 1 --threads 16 --duration 6 --sigterm-after 2 --pid $(pid_of d) | sed 's/ns0/default/')"
+forget d
+rec S8 "false 404 in log" "$(grep -c '404 Not Found' "$ROOT/d.log")"
+
+# ===========================================================================
+# S9 primary + replica
+start p "$BIN" "$ROOT/p9" $P_HTTP $P_ADMIN --enable-namespaces --grpc-listen-addr 127.0.0.1:$P_GRPC || exit 1
+start r "$BIN" "$ROOT/r9" $R_HTTP $R_ADMIN --enable-namespaces --primary-grpc-url http://127.0.0.1:$P_GRPC || exit 1
+ns_create rep >/dev/null; seed rep "$P_HTTP" 2000
+rec S9 "replica catches up" "$(wait_count rep $R_HTTP 2000)"
+rec S9 "proxied write via replica" "$(q rep 'insert into t values (0)' $R_HTTP)"
+rec S9 "primary sees proxied write" "$(wait_count rep $P_HTTP 2001)"
+rec S9 "replica sees it" "$(wait_count rep $R_HTTP 2001)"
+rec S9 "replica unknown ns" "$(q nope 'select 1' $R_HTTP)"
+# replica shutdown under read load
+ns_create ns0 >/dev/null; seed ns0 "$P_HTTP" 100; wait_count ns0 $R_HTTP 100 >/dev/null
+rec S9 "replica SIGTERM under read load" "$(python3 $LOAD --quiet --port $R_HTTP --namespaces 1 --threads 16 --duration 5 --sigterm-after 1.5 --pid $(pid_of r))"
+forget r
+start r "$BIN" "$ROOT/r9" $R_HTTP $R_ADMIN --enable-namespaces --primary-grpc-url http://127.0.0.1:$P_GRPC || exit 1
+rec S9 "replica after restart" "$(wait_count rep $R_HTTP 2001)"
+# primary shutdown under proxied-write load through the replica
+rec S9 "primary SIGTERM under proxied writes (replica statuses)" "$(python3 $LOAD --quiet --port $R_HTTP --namespaces 1 --ns rep --sql "insert into t values (9)" --threads 8 --duration 8 --sigterm-after 2 --pid $(pid_of p))"
+forget p
+start p "$BIN" "$ROOT/p9" $P_HTTP $P_ADMIN --enable-namespaces --grpc-listen-addr 127.0.0.1:$P_GRPC || exit 1
+pc=$(count rep $P_HTTP)
+rec S9 "primary count after restart" "$pc"
+q rep "insert into t values (1)" >/dev/null
+rec S9 "replication resumes after primary restart" "$(wait_count rep $R_HTTP $((pc+1)))"
+rec S9 "proxied write after primary restart" "$(q rep 'insert into t values (2)' $R_HTTP)"
+rec S9 "replica count after that" "$(wait_count rep $R_HTTP $((pc+2)))"
+# destroy on primary while replica replicates: the replica must still recognise the message
+ns_delete rep >/dev/null; sleep 2
+rec S9 "replica log after primary destroy" "$(grep -c "doesn't exist, trying to replicate again" "$ROOT/r.log")"
+rec S9 "replica log: non-fatal/unavailable lines" "$(grep -c -i -E 'unavailable|non-fatal' "$ROOT/r.log")"
+stop r; stop p
+
+# S10 mixed versions: this binary as primary, the other as replica
+if [ -n "$BIN_OTHER" ]; then
+start p "$BIN" "$ROOT/p10" $P_HTTP $P_ADMIN --enable-namespaces --grpc-listen-addr 127.0.0.1:$P_GRPC || exit 1
+start r "$BIN_OTHER" "$ROOT/r10" $R_HTTP $R_ADMIN --enable-namespaces --primary-grpc-url http://127.0.0.1:$P_GRPC || exit 1
+ns_create mix >/dev/null; seed mix "$P_HTTP" 500
+rec S10 "($OTHER replica) catches up" "$(wait_count mix $R_HTTP 500)"
+rec S10 "($OTHER replica) primary SIGTERM under proxied writes" "$(python3 $LOAD --quiet --port $R_HTTP --namespaces 1 --ns mix --sql "insert into t values (9)" --threads 8 --duration 8 --sigterm-after 2 --pid $(pid_of p))"
+forget p
+start p "$BIN" "$ROOT/p10" $P_HTTP $P_ADMIN --enable-namespaces --grpc-listen-addr 127.0.0.1:$P_GRPC || exit 1
+pc=$(count mix $P_HTTP); q mix "insert into t values (1)" >/dev/null
+rec S10 "($OTHER replica) replication resumes" "$(wait_count mix $R_HTTP $((pc+1)))"
+rec S10 "($OTHER replica) proxied write after restart" "$(q mix 'insert into t values (2)' $R_HTTP)"
+rec S10 "($OTHER replica) log unavailable/non-fatal/doesnt-exist" "$(grep -c -i -E 'unavailable|non-fatal' "$ROOT/r.log")/$(grep -c "doesn't exist" "$ROOT/r.log")"
+stop r; stop p
+fi
+
+echo "##### done: $RESULTS"
diff --git a/libsql-server/scripts/namespace-live-checks/scenario.sh b/libsql-server/scripts/namespace-live-checks/scenario.sh
new file mode 100755
index 0000000000..5a8c1cd418
--- /dev/null
+++ b/libsql-server/scripts/namespace-live-checks/scenario.sh
@@ -0,0 +1,67 @@
+#!/bin/bash
+# Run one focused scenario against a sqld binary and report the status codes
+# clients saw, the namespace-store counters, and the store's log lines.
+#
+# Usage: scenario.sh <path/to/sqld> <evict|sigterm> [label]
+#
+#   evict    cache capacity far below the hot set, so entries are evicted
+#            under requests all the time (--max-active-namespaces 2, 6
+#            namespaces, 24 clients, 20 s)
+#   sigterm  20 namespaces under 32 clients; SIGTERM the server at 3 s and
+#            keep requesting until the port closes
+#
+# Works with bash 3.2. Needs curl and python3.
+set -euo pipefail
+HERE=$(cd "$(dirname "$0")" && pwd)
+BIN=$1; SCENARIO=$2; LABEL=${3:-$(basename "$BIN")}
+PORT=${PORT:-18080}; ADMIN=${ADMIN:-19090}
+OUT=${OUT:-./namespace-live-checks-out}; mkdir -p "$OUT"
+DIR=$OUT/$LABEL-$SCENARIO; LOG=$DIR.log
+rm -rf "$DIR" "$LOG"; mkdir -p "$DIR"
+
+case $SCENARIO in
+  evict)   MAX_ACTIVE=2;   NS=6;  THREADS=24; DURATION=20; ACTION="" ;;
+  sigterm) MAX_ACTIVE=100; NS=20; THREADS=32; DURATION=12; ACTION="--sigterm-after 3" ;;
+  *) echo "unknown scenario: $SCENARIO (evict|sigterm)"; exit 2 ;;
+esac
+
+for port in $PORT $ADMIN; do
+  if lsof -nP -iTCP:"$port" -sTCP:LISTEN >/dev/null 2>&1; then
+    echo "port $port is busy; stop the process that holds it first"; exit 1
+  fi
+done
+
+export RUST_LOG=${RUST_LOG:-info,libsql_server::namespace::store=debug}
+"$BIN" --db-path "$DIR" --http-listen-addr 127.0.0.1:"$PORT" --admin-listen-addr 127.0.0.1:"$ADMIN" \
+  --enable-namespaces --max-active-namespaces "$MAX_ACTIVE" --no-welcome >"$LOG" 2>&1 &
+PID=$!
+trap 'kill $PID 2>/dev/null || true' EXIT
+
+for _ in $(seq 1 100); do
+  kill -0 $PID 2>/dev/null || { echo "sqld exited at start:"; tail -3 "$LOG"; exit 1; }
+  curl -sf --max-time 2 -o /dev/null "http://127.0.0.1:$PORT/health" && break; sleep 0.1
+done
+echo "sqld ($LABEL) pid $PID up"
+
+for i in $(seq 0 $((NS - 1))); do
+  curl -sf --max-time 10 -X POST -H 'content-type: application/json' -d '{}' \
+    "http://127.0.0.1:$ADMIN/v1/namespaces/ns$i/create" >/dev/null
+  curl -sf --max-time 10 -X POST -H 'content-type: application/json' -H "x-namespace: ns$i" \
+    -d '{"requests":[{"type":"execute","stmt":{"sql":"create table t(x)"}},{"type":"execute","stmt":{"sql":"insert into t select value from generate_series(1,2000)"}},{"type":"close"}]}' \
+    "http://127.0.0.1:$PORT/v2/pipeline" >/dev/null
+done
+echo "created $NS namespaces"
+
+if [ "$SCENARIO" = sigterm ]; then ACTION="$ACTION --pid $PID"; fi
+# shellcheck disable=SC2086
+python3 "$HERE/load.py" --port "$PORT" --namespaces "$NS" --threads "$THREADS" --duration "$DURATION" $ACTION
+
+echo
+echo "=== namespace-store counters (absent = zero) ==="
+curl -s --max-time 2 "http://127.0.0.1:$ADMIN/metrics" 2>/dev/null \
+  | grep -E "^libsql_server_namespace_(emptied|left_empty)|^libsql_server_user_http_(fault|response)" || echo "(admin port closed)"
+echo
+echo "=== store log lines (grouped) ==="
+grep -E "WARN|ERROR|emptied|left empty|giving up|dropped a different" "$LOG" \
+  | sed 's/^.*Z  *//; s/req_id=[0-9a-f-]*//' | sort | uniq -c | sort -rn | head -12
+echo "log: $LOG"

From 162259a748617d795c70ccc3a316faa312b3032e Mon Sep 17 00:00:00 2001
From: Tomasz Szymczyszyn <tomasz.szymczyszyn@shopify.com>
Date: Fri, 16 Oct 2026 11:58:56 +0200
Subject: [PATCH 2/2] Add the embedded-replica restart check and the
 shutdown-window measurement

`replica-sigterm.sh` runs eight embedded replicas (the `libsql` crate, via
the new `replica_sync_loop` example) against a primary that is stopped and
started again, and prints every distinct `sync()` error with the time it
was first seen, plus the replicator's own log lines. It shows what a client
built on the crate sees across a primary restart: a server that reports an
existing namespace as missing produces `Requested namespace doesn't exist`
at the signal, which the client does not retry; a server that answers
UNAVAILABLE produces the client's own retry line, and the replicas sync
again once the server is back.

`window.sh` loads N namespaces, each with a WAL, and measures the shutdown
window: how long clients get 503 before the port closes. The window grows
with the number of loaded namespaces, and it is what client retry budgets
have to cover.
---
 .../scripts/namespace-live-checks/README.md   | 21 +++++
 .../namespace-live-checks/replica-sigterm.sh  | 69 +++++++++++++++
 .../scripts/namespace-live-checks/window.sh   | 65 ++++++++++++++
 libsql/examples/replica_sync_loop.rs          | 85 +++++++++++++++++++
 4 files changed, 240 insertions(+)
 create mode 100755 libsql-server/scripts/namespace-live-checks/replica-sigterm.sh
 create mode 100755 libsql-server/scripts/namespace-live-checks/window.sh
 create mode 100644 libsql/examples/replica_sync_loop.rs

diff --git a/libsql-server/scripts/namespace-live-checks/README.md b/libsql-server/scripts/namespace-live-checks/README.md
index 843fcc6241..8be6f87d5f 100644
--- a/libsql-server/scripts/namespace-live-checks/README.md
+++ b/libsql-server/scripts/namespace-live-checks/README.md
@@ -16,6 +16,7 @@ widens the race windows, which is what you want.
 
 ```
 cargo build -p libsql-server --bin sqld
+cargo build -p libsql --example replica_sync_loop   # for replica-sigterm.sh
 ```
 
 ## Scripts
@@ -26,6 +27,8 @@ cargo build -p libsql-server --bin sqld
 | `scenario.sh` | One focused scenario: `evict` (cache capacity far below the hot set) or `sigterm` (signal under load). Prints the tally, the namespace-store counters and the grouped store log lines. |
 | `regress.sh` | A 65-check matrix across the admin and user APIs, one primary and one primary plus replica, optionally mixed versions. Writes `results.tsv`. |
 | `compare.py` | Side-by-side diff of two `results.tsv` files. |
+| `replica-sigterm.sh` | What embedded replicas (the `libsql` crate, `libsql/examples/replica_sync_loop.rs`) see when their primary is stopped and started again: every distinct `sync()` error with the time it was first seen, and the replicator's own retry log lines. |
+| `window.sh` | How long the shutdown window (503s until the port closes) is for N loaded namespaces, each with a WAL. Compare with your clients' retry budgets. |
 
 ## Comparing two binaries
 
@@ -37,6 +40,9 @@ $S/scenario.sh target/debug/sqld sigterm          # expect 503s after the signal
 $S/regress.sh /path/to/old/sqld old /path/to/new/sqld
 $S/regress.sh /path/to/new/sqld new /path/to/old/sqld
 $S/compare.py namespace-live-checks-out/old/results.tsv namespace-live-checks-out/new/results.tsv old new
+
+$S/replica-sigterm.sh target/debug/sqld           # expect no "doesn't exist"; see below
+$S/window.sh target/debug/sqld 2000               # setup takes about 45 s per 1,000 namespaces
 ```
 
 `regress.sh` writes server logs next to the results (`<label>/p.log`,
@@ -52,6 +58,18 @@ $S/compare.py namespace-live-checks-out/old/results.tsv namespace-live-checks-ou
 - `regress.sh`: unknown namespaces answer 404 everywhere; deleted namespaces
   answer 404 and their directory is gone; data survives a restart; the
   replica catches up, proxies writes, and resumes after a primary restart.
+- `replica-sigterm.sh`: no `sync()` error says `Requested namespace doesn't
+  exist` (a server that reports an existing namespace as missing produces
+  exactly that, at the moment of the signal, and the client does not retry
+  it). What a healthy server produces instead is the replicator's own retry
+  line, `error connecting to primary. retrying. error: status: Unavailable,
+  message: "Namespace store has shutdown"`, and, if the server stays down
+  longer than the client's handshake retries, `Timeout performing handshake
+  with primary`. All replicas sync again once the server is back.
+- `window.sh`: 503 `Namespace store has shutdown` from the first millisecond
+  after the signal until the port closes, no 404. The window grows with the
+  number of loaded namespaces (a debug build on a laptop: about 0.2 to
+  0.3 ms per namespace). It is the time your clients must retry through.
 
 ## Reading the output
 
@@ -62,6 +80,9 @@ $S/compare.py namespace-live-checks-out/old/results.tsv namespace-live-checks-ou
   server closed the connection after an error answer.
 - The admin port is gone after a SIGTERM; `scenario.sh` reads `/metrics`
   only when the server is still up.
+- `window.sh` creates namespaces with a few threads only. More trips sqld's
+  connection-creation throttle (`429 DbCreateTimeout`) during setup; the
+  namespace is still loaded, so the measurement is unaffected.
 - Under eviction churn the server logs `unable to remove .sentinel file`
   for namespaces reopened while the evicted instance is still shutting
   down. That is independent of the namespace store and shows on every
diff --git a/libsql-server/scripts/namespace-live-checks/replica-sigterm.sh b/libsql-server/scripts/namespace-live-checks/replica-sigterm.sh
new file mode 100755
index 0000000000..7f84794218
--- /dev/null
+++ b/libsql-server/scripts/namespace-live-checks/replica-sigterm.sh
@@ -0,0 +1,69 @@
+#!/bin/bash
+# What embedded replicas (the `libsql` crate) see when their primary restarts.
+#
+# Usage: replica-sigterm.sh <path/to/sqld> [label]
+#
+# Starts sqld with 20 namespaces loaded, runs 8 embedded replicas (one per
+# namespace, syncing in a tight loop; see libsql/examples/replica_sync_loop.rs),
+# sends SIGTERM at 4 s, starts the same server again at about 7 s, and prints
+# every distinct error the replicas saw with the time it was first seen.
+#
+# Needs the example built: cargo build -p libsql --example replica_sync_loop
+# Override its path with SYNC_LOOP. Works with bash 3.2. Needs curl.
+set -euo pipefail
+HERE=$(cd "$(dirname "$0")" && pwd)
+BIN=$1; LABEL=${2:-$(basename "$BIN")}
+PORT=${PORT:-18080}; ADMIN=${ADMIN:-19090}
+SYNC_LOOP=${SYNC_LOOP:-$HERE/../../../target/debug/examples/replica_sync_loop}
+OUT=${OUT:-./namespace-live-checks-out}; DIR=$OUT/$LABEL-replica-sigterm
+rm -rf "$DIR"; mkdir -p "$DIR"; LOG=$DIR/server.log
+
+[ -x "$SYNC_LOOP" ] || { echo "replica_sync_loop not found at $SYNC_LOOP; build it with: cargo build -p libsql --example replica_sync_loop"; exit 2; }
+for port in $PORT $ADMIN; do
+  if lsof -nP -iTCP:"$port" -sTCP:LISTEN >/dev/null 2>&1; then
+    echo "port $port is busy; stop the process that holds it first"; exit 1
+  fi
+done
+
+start_server() {
+  RUST_LOG=${RUST_LOG:-info} "$BIN" --db-path "$DIR/db" --http-listen-addr 127.0.0.1:"$PORT" --admin-listen-addr 127.0.0.1:"$ADMIN" \
+    --enable-namespaces --max-active-namespaces 100 --no-welcome >>"$LOG" 2>&1 &
+  echo $!
+}
+wait_up() {
+  for _ in $(seq 1 100); do
+    kill -0 "$1" 2>/dev/null || { echo "sqld exited at start:"; tail -3 "$LOG"; exit 1; }
+    curl -sf --max-time 2 -o /dev/null "http://127.0.0.1:$PORT/health" && return; sleep 0.1
+  done
+}
+PID=$(start_server); wait_up "$PID"
+trap 'kill $PID 2>/dev/null || true; [ -f "$DIR/pid2" ] && kill "$(cat "$DIR/pid2")" 2>/dev/null || true' EXIT
+
+for i in $(seq 0 19); do
+  curl -sf --max-time 10 -X POST -H 'content-type: application/json' -d '{}' \
+    "http://127.0.0.1:$ADMIN/v1/namespaces/ns$i/create" >/dev/null
+  curl -sf --max-time 10 -X POST -H 'content-type: application/json' -H "x-namespace: ns$i" \
+    -d '{"requests":[{"type":"execute","stmt":{"sql":"select 1"}},{"type":"close"}]}' \
+    "http://127.0.0.1:$PORT/v2/pipeline" >/dev/null
+done
+echo "sqld ($LABEL) pid $PID up with 20 namespaces loaded"
+
+(
+  sleep 4; echo "[+4s] SIGTERM $PID"; kill -TERM "$PID"; wait "$PID" 2>/dev/null || true
+  echo "[+$(( SECONDS ))s] server exited"; sleep 2.5
+  PID2=$(start_server); echo "$PID2" > "$DIR/pid2"; echo "[+$(( SECONDS ))s] server started again (pid $PID2)"
+) &
+TIMER=$!
+
+echo
+echo "=== what the replicas saw ==="
+PORT=$PORT REPLICAS=${REPLICAS:-8} DURATION=${DURATION:-12} "$SYNC_LOOP" 2>"$DIR/client.log"
+wait $TIMER
+
+echo
+echo "=== server: shutdown timing ==="
+sed "s/$(printf '\033')\[[0-9;]*m//g" "$LOG" | grep -E "Got SIGTERM|shutdown gracefully" | sed -E 's/^([^ ]+) .*(Got SIGTERM|shutdown gracefully).*/\1 \2/' | head -4
+echo
+echo "=== replicator log lines ==="
+sed "s/$(printf '\033')\[[0-9;]*m//g" "$DIR/client.log" | grep -E "WARN|ERROR" | grep -v "Unusually long" | sed -E 's/^.*Z  *//; s/ns[0-9]+/nsN/g' | sort | uniq -c | sort -rn | head -6 || true
+echo "logs: $LOG, $DIR/client.log"
diff --git a/libsql-server/scripts/namespace-live-checks/window.sh b/libsql-server/scripts/namespace-live-checks/window.sh
new file mode 100755
index 0000000000..c3c9183d2b
--- /dev/null
+++ b/libsql-server/scripts/namespace-live-checks/window.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+# How long the shutdown window is, against the number of loaded namespaces.
+#
+# Usage: window.sh <path/to/sqld> <namespaces> [label]
+#
+# Creates and loads N namespaces (each with a table and a row, so each has a
+# WAL), then runs load.py against all of them with a SIGTERM at 3 s. The
+# window is the time from the first 503 to the port closing; compare it with
+# your clients' retry budgets. Setup takes about 45 s per 1,000 namespaces.
+#
+# Works with bash 3.2. Needs curl and python3.
+set -euo pipefail
+HERE=$(cd "$(dirname "$0")" && pwd)
+BIN=$1; N=$2; LABEL=${3:-$(basename "$BIN")-$N}
+PORT=${PORT:-18080}; ADMIN=${ADMIN:-19090}
+OUT=${OUT:-./namespace-live-checks-out}; DIR=$OUT/$LABEL-window
+rm -rf "$DIR"; mkdir -p "$DIR"; LOG=$DIR/server.log
+
+for port in $PORT $ADMIN; do
+  if lsof -nP -iTCP:"$port" -sTCP:LISTEN >/dev/null 2>&1; then
+    echo "port $port is busy; stop the process that holds it first"; exit 1
+  fi
+done
+
+RUST_LOG=${RUST_LOG:-info} "$BIN" --db-path "$DIR/db" --http-listen-addr 127.0.0.1:"$PORT" --admin-listen-addr 127.0.0.1:"$ADMIN" \
+  --enable-namespaces --max-active-namespaces $(( N * 2 )) --no-welcome >"$LOG" 2>&1 &
+PID=$!
+trap 'kill $PID 2>/dev/null || true' EXIT
+for _ in $(seq 1 100); do
+  kill -0 $PID 2>/dev/null || { echo "sqld exited at start:"; tail -3 "$LOG"; exit 1; }
+  curl -sf --max-time 2 -o /dev/null "http://127.0.0.1:$PORT/health" && break; sleep 0.1
+done
+
+T0=$SECONDS
+# A few threads only: more trips sqld's connection-creation throttle (429
+# DbCreateTimeout) while namespaces are being created.
+python3 - "$N" "$PORT" "$ADMIN" <<'PY'
+import sys, json, http.client, concurrent.futures
+n, port, admin = int(sys.argv[1]), int(sys.argv[2]), int(sys.argv[3])
+def one(i):
+    c = http.client.HTTPConnection("127.0.0.1", admin, timeout=30)
+    c.request("POST", f"/v1/namespaces/ns{i}/create", body="{}", headers={"content-type": "application/json"})
+    c.getresponse().read(); c.close()
+    u = http.client.HTTPConnection("127.0.0.1", port, timeout=30)
+    body = json.dumps({"requests": [
+        {"type": "execute", "stmt": {"sql": "create table if not exists t(x)"}},
+        {"type": "execute", "stmt": {"sql": "insert into t values (1)"}},
+        {"type": "close"}]})
+    u.request("POST", "/v2/pipeline", body=body, headers={"content-type": "application/json", "x-namespace": f"ns{i}"})
+    r = u.getresponse(); r.read(); u.close()
+    return r.status
+with concurrent.futures.ThreadPoolExecutor(6) as ex:
+    statuses = list(ex.map(one, range(n)))
+bad = sum(1 for s in statuses if s != 200)
+print(f"loaded {n} namespaces" + (f"; {bad} setup writes were not 200 (usually 429 DbCreateTimeout; the namespace is still loaded)" if bad else ""))
+PY
+echo "setup took $(( SECONDS - T0 ))s; sqld rss $(ps -o rss= -p $PID | awk '{printf "%.0f MB", $1/1024}')"
+
+python3 "$HERE/load.py" --port "$PORT" --namespaces "$N" --threads 32 --duration 25 --sigterm-after 3 --pid $PID
+wait $PID 2>/dev/null || true
+echo
+echo "=== server: shutdown timing ==="
+sed "s/$(printf '\033')\[[0-9;]*m//g" "$LOG" | grep -E "Got SIGTERM|shutdown gracefully" | sed -E 's/^([^ ]+) .*(Got SIGTERM|shutdown gracefully).*/\1 \2/'
+echo "The window is the time from the first 503 to the first conn_error above."
+echo "log: $LOG"
diff --git a/libsql/examples/replica_sync_loop.rs b/libsql/examples/replica_sync_loop.rs
new file mode 100644
index 0000000000..7b21a17cfa
--- /dev/null
+++ b/libsql/examples/replica_sync_loop.rs
@@ -0,0 +1,85 @@
+//! Live check for the namespace store: N embedded replicas, one per namespace
+//! (`ns0..nsN-1`, reached as `nsK.localhost`), syncing in a tight loop against
+//! a local `sqld`. Prints how many syncs succeeded and every distinct error
+//! message with the time it was first and last seen, so that what a client
+//! sees across a server restart can be compared between two builds.
+//!
+//! Driven by `libsql-server/scripts/namespace-live-checks/replica-sigterm.sh`.
+//!
+//! Environment: `PORT` (default 18080), `REPLICAS` (default 8), `DURATION`
+//! seconds (default 12). Log lines (warn and above, which includes the
+//! replicator's retry lines) go to stderr.
+use libsql::Builder;
+use std::collections::BTreeMap;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::{Arc, Mutex};
+use std::time::{Duration, Instant};
+
+fn env_or<T: std::str::FromStr>(name: &str, default: T) -> T {
+    std::env::var(name)
+        .ok()
+        .and_then(|v| v.parse().ok())
+        .unwrap_or(default)
+}
+
+#[tokio::main]
+async fn main() {
+    tracing_subscriber::fmt()
+        .with_writer(std::io::stderr)
+        .with_max_level(tracing::Level::WARN)
+        .init();
+    let port: u16 = env_or("PORT", 18080);
+    let replicas: usize = env_or("REPLICAS", 8);
+    let duration: f64 = env_or("DURATION", 12.0);
+
+    let start = Instant::now();
+    // message -> (count, first seen, last seen), in seconds since start
+    let errors: Arc<Mutex<BTreeMap<String, (usize, f64, f64)>>> = Default::default();
+    let ok = Arc::new(AtomicUsize::new(0));
+    let dir = tempfile::tempdir().unwrap();
+
+    let mut tasks = Vec::new();
+    for i in 0..replicas {
+        let path = dir.path().join(format!("replica{i}.db"));
+        let url = format!("http://ns{i}.localhost:{port}");
+        let errors = errors.clone();
+        let ok = ok.clone();
+        tasks.push(tokio::spawn(async move {
+            let db = Builder::new_remote_replica(&path, url, String::new())
+                .build()
+                .await
+                .expect("open the embedded replica");
+            // A proxied write, so that every sync has frames to pull.
+            if let Ok(conn) = db.connect() {
+                let _ = conn
+                    .execute("CREATE TABLE IF NOT EXISTS t (x INTEGER)", ())
+                    .await;
+                let _ = conn.execute("INSERT INTO t VALUES (1)", ()).await;
+            }
+            while start.elapsed().as_secs_f64() < duration {
+                match db.sync().await {
+                    Ok(_) => {
+                        ok.fetch_add(1, Ordering::Relaxed);
+                    }
+                    Err(e) => {
+                        let now = start.elapsed().as_secs_f64();
+                        let mut errors = errors.lock().unwrap();
+                        let entry = errors.entry(e.to_string()).or_insert((0, now, now));
+                        entry.0 += 1;
+                        entry.2 = now;
+                    }
+                }
+                tokio::time::sleep(Duration::from_millis(2)).await;
+            }
+        }));
+    }
+    for task in tasks {
+        let _ = task.await;
+    }
+
+    println!("ok syncs: {}", ok.load(Ordering::Relaxed));
+    println!("errors (count, first seen, last seen):");
+    for (message, (count, first, last)) in errors.lock().unwrap().iter() {
+        println!("  {count:6}  +{first:7.3}s .. +{last:7.3}s  {message}");
+    }
+}