#!/usr/bin/env bash
#
# Z-Access self-host installer — stands up a full control plane on THIS server.
#
#   curl -fsSL https://get.zpoa.com | sudo bash
#   # or unattended (cloud-init / Ansible):
#   ZPOA_DOMAIN=access.acme.com ZPOA_ROUTES=10.0.0.0/16 \
#     curl -fsSL https://get.zpoa.com | sudo bash
#
# No copy-paste from any UI: the installer generates its own secrets, writes
# the compose stack (control plane + in-process DERP relay + optional
# subnet-router gateway + Postgres, Caddy auto-TLS), brings it up, and runs
# `zpoa-access bootstrap` itself — then prints the URL, admin token, and the
# one command to enroll an endpoint. Day-2 is CLI/API on this box:
#   docker compose exec control zpoa-access authkey create --tenant-slug ...
#
# Everything is overridable via env (white-label): ZPOA_BINARY_BASE, ZPOA_IMAGE.
#
set -euo pipefail

# ---------------------------------------------------------------------------
# Configuration (env-driven; interactive prompts fill the gaps on a TTY)
# ---------------------------------------------------------------------------
ZPOA_DOMAIN="${ZPOA_DOMAIN:-}"                 # public hostname clients connect to (required)
ZPOA_ADMIN_EMAIL="${ZPOA_ADMIN_EMAIL:-}"       # Let's Encrypt account email (optional)
ZPOA_ROLE="${ZPOA_ROLE:-server}"              # server | gateway
ZPOA_TENANT="${ZPOA_TENANT:-}"                 # tenant slug (default: derived from domain)
ZPOA_TENANT_NAME="${ZPOA_TENANT_NAME:-}"       # human tenant name (default: slug)
ZPOA_ROUTES="${ZPOA_ROUTES:-}"                 # corp CIDR(s) to advertise -> co-locates a gateway
ZPOA_GATEWAY_SNAT="${ZPOA_GATEWAY_SNAT:-true}" # false = preserve client source IPs into the LAN
# Corp DNS to PUSH to clients (split-DNS) so internal names resolve over the VPN
# — the OpenVPN `push "dhcp-option DNS/DOMAIN"` equivalent. Both must be set to
# take effect. Empty = no push (clients keep MagicDNS + their own resolvers).
ZPOA_DNS_SEARCH_DOMAINS="${ZPOA_DNS_SEARCH_DOMAINS:-}" # e.g. corp.local,acme.internal
ZPOA_DNS_CORP_RESOLVERS="${ZPOA_DNS_CORP_RESOLVERS:-}" # e.g. 10.0.0.53,10.0.0.54 (must be in a routed CIDR)
ZPOA_MESH_CIDR="${ZPOA_MESH_CIDR:-100.64.0.0/16}"
ZPOA_DB_URL="${ZPOA_DB_URL:-}"                 # external Postgres URL (empty = bundled postgres)
ZPOA_ADMIN_TOKEN="${ZPOA_ADMIN_TOKEN:-}"       # preset admin bearer (empty = generated)
ZPOA_VIEWER_TOKEN="${ZPOA_VIEWER_TOKEN:-}"     # read-only viewer bearer (Tier B: Workspace auto-connect; empty = disabled)
ZPOA_KEYCLOAK_ISSUER_URL="${ZPOA_KEYCLOAK_ISSUER_URL:-}"  # OIDC issuer. Empty = auto-filled from the ZPOA Cloud exchange (oidc_issuer_url), so the CP trusts Workspace LOGINS out of the box — no admin-token paste. An explicit value wins.
ZPOA_VAULT_KEY="${ZPOA_VAULT_KEY:-}"           # at-rest KEK (empty = generated)
# Employee VPN SSO (OIDC issuer/audience) is NOT configured at install time — the
# client is zero-touch and the operator sets the IdP later via the Workspace
# "Identity Provider" panel, which writes issuer/audience to the DB (read at CP
# startup). A fresh deploy boots token-only (CYBER_ADMIN_TOKEN); no OIDC env here.
# Comma-separated IdP hostnames the VENDOR operates. If a tenant's VPN SSO issuer
# resolves to one of these, the console shows a "vendor-hosted" warning so a
# bring-your-own-IdP customer never silently routes employee sign-in through the
# vendor. Empty = no check (pure BYO). Never hardcoded — set per white-label.
ZPOA_VENDOR_IDP_HOSTS="${ZPOA_VENDOR_IDP_HOSTS:-}"
# Cloud-linked self-host: a bootstrap token minted in ZPOA Cloud for a specific
# tenant. When set, the installer EXCHANGES it for the authoritative tenant slug
# + name + issuer from the SaaS (proves the tenant exists — no typo, no free
# text). Empty = standalone self-host (slug is prompted/validated locally).
ZPOA_CLOUD_TOKEN="${ZPOA_CLOUD_TOKEN:-}"
# SaaS API base (the zpoa-api that serves /install.sh + /api/v1). Prod default;
# override for stage/dev (https://api-stage.zpoa.com | https://api-dev.zpoa.com)
# or a white-label deployment.
ZPOA_CLOUD_API="${ZPOA_CLOUD_API:-https://api.zpoa.com}"
ZPOA_DERP_RELAYS="${ZPOA_DERP_RELAYS:-}"       # override; default <domain>:443:3478
ZPOA_TLS="${ZPOA_TLS:-auto}"                   # auto (Let's Encrypt) | internal (self-signed) | off
ZPOA_WG_PORT="${ZPOA_WG_PORT:-51820}"
ZPOA_DIR="${ZPOA_DIR:-/opt/zpoa-access}"
ZPOA_YES="${ZPOA_YES:-}"                        # any value = non-interactive, assume defaults
ZPOA_OPEN_FIREWALL="${ZPOA_OPEN_FIREWALL:-true}"

# Distribution. Default = build a local image from the PUBLIC prebuilt binary
# (works today, no registry login). Set ZPOA_IMAGE to a pullable image
# (post Phase-0 public registry, or your own mirror) to pull instead.
ZPOA_BINARY_BASE="${ZPOA_BINARY_BASE:-https://get.zpoa.com}"
ZPOA_VERSION="${ZPOA_VERSION:-latest}"          # pin in prod; SHA256SUMS verified when present
ZPOA_IMAGE="${ZPOA_IMAGE:-}"                    # e.g. ghcr.io/zpoasec/zpoa-access (pull mode)
ZPOA_IMAGE_TAG="${ZPOA_IMAGE_TAG:-v0.3.19}"     # pinned (reproducible); override for a different release

# Gateway mode only (a box that is NOT the CP — no DB here; CP + identity live
# on the control plane it joins). ZPOA_CLOUD_CP is the default CP a gateway
# joins when the operator doesn't name one — ZPOA Cloud. Override for dev
# (https://access-dev.zpoa.com) or to join your own self-hosted CP.
ZPOA_CONTROL_URL="${ZPOA_CONTROL_URL:-}"
ZPOA_AUTH_KEY="${ZPOA_AUTH_KEY:-}"
ZPOA_CLOUD_CP="${ZPOA_CLOUD_CP:-https://access.zpoa.com}"

# Backup control-plane mode (role=backup) — a WARM STANDBY on a SECOND host that
# streams the primary's Postgres (read-only replica) and can be promoted on a
# primary loss. See docs/high-availability.md. The data plane already survives a
# CP outage (clients cache the netmap 24h + fail over across control_plane_urls),
# so a backup is about fast recovery of NEW enrolment / admin writes, not keeping
# existing tunnels up. Stand the primary up first, run `zpoa ha prepare-primary`
# on it, then paste the block it prints here.
ZPOA_PRIMARY_URL="${ZPOA_PRIMARY_URL:-}"          # primary CP URL this backup watches + follows (https://access.acme.com)
ZPOA_PRIMARY_DB_HOST="${ZPOA_PRIMARY_DB_HOST:-}"  # primary Postgres host[:port] reachable from here (default: host of PRIMARY_URL :5432)
ZPOA_REPL_USER="${ZPOA_REPL_USER:-zpoa_replicator}"  # replication role (created on the primary by prepare-primary)
ZPOA_REPL_PASSWORD="${ZPOA_REPL_PASSWORD:-}"      # replication password (from prepare-primary)
ZPOA_REPL_SLOT="${ZPOA_REPL_SLOT:-}"              # optional physical replication slot name on the primary
ZPOA_PG_PASSWORD="${ZPOA_PG_PASSWORD:-}"          # the PRIMARY's `zpoa` DB password — the replicated role carries it (required)
ZPOA_AUTO_PROMOTE="${ZPOA_AUTO_PROMOTE:-false}"   # true = Tier 0.5 fenced auto-failover (EXACTLY 2 nodes); false = Tier 0 one-click promote
# NOTE: a backup must reuse the primary's ZPOA_VAULT_KEY (rows are AES-256-GCM
# sealed with it and replicated) and ZPOA_ADMIN_TOKEN (so the admin API keeps
# working after a promote). prepare-primary prints both.

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
c_reset=$'\033[0m'; c_bold=$'\033[1m'; c_grn=$'\033[32m'; c_yel=$'\033[33m'; c_red=$'\033[31m'; c_cyn=$'\033[36m'
log()  { printf '%s==>%s %s\n' "$c_cyn" "$c_reset" "$*"; }
ok()   { printf '%s ✓ %s%s\n' "$c_grn" "$*" "$c_reset"; }
warn() { printf '%s !  %s%s\n' "$c_yel" "$*" "$c_reset"; }
die()  { printf '%s ✗ %s%s\n' "$c_red" "$*" "$c_reset" >&2; exit 1; }

# Interactive when NOT forced non-interactive, stdout is a terminal, and the
# controlling tty is readable. Testing [ -t 1 ] (not [ -t 0 ]) matters for the
# `curl … | sudo bash` case: there stdin is the pipe (never a tty), so the old
# `[ -t 0 ]` made every piped install skip prompts. prompt() reads /dev/tty.
on_tty() { [ -z "$ZPOA_YES" ] && [ -t 1 ] && [ -r /dev/tty ]; }
prompt() { # prompt VAR "question" "default"
  local __var="$1" __q="$2" __def="${3:-}" __ans=""
  if ! on_tty; then printf -v "$__var" '%s' "$__def"; return; fi
  if [ -n "$__def" ]; then read -r -p "$__q [$__def]: " __ans </dev/tty || true; else read -r -p "$__q: " __ans </dev/tty || true; fi
  printf -v "$__var" '%s' "${__ans:-$__def}"
}
gen_secret() { openssl rand -hex 32 2>/dev/null || head -c32 /dev/urandom | od -An -tx1 | tr -d ' \n'; }

# --- input normalizers + validators ----------------------------------------
_lc_trim()   { local s="$1"; s="${s#"${s%%[![:space:]]*}"}"; s="${s%"${s##*[![:space:]]}"}"; printf '%s' "$s" | tr '[:upper:]' '[:lower:]'; }
_strip_ws()  { printf '%s' "$1" | tr -d '[:space:]'; }
# _norm_routes normalizes the advertise-routes answer. Same whitespace strip as
# _strip_ws, but an explicit skip token (none/skip/no/-) becomes EMPTY so an
# operator can decline the pre-filled suggestion with one word instead of having
# to blank the field — keeps the suggestion a suggestion, never enforced.
_norm_routes() {
  local v; v="$(_strip_ws "$1")"
  case "$v" in none|None|NONE|skip|Skip|no|No|-|'') printf '' ;; *) printf '%s' "$v" ;; esac
}
_is_uint()   { case "$1" in ''|*[!0-9]*) return 1 ;; esac; return 0; }

is_valid_ipv4() { # dotted quad, each octet 0-255, exactly 4 octets
  local ip="$1" o cnt=0 IFS=.
  # shellcheck disable=SC2086
  set -- $ip
  [ "$#" -eq 4 ] || return 1
  for o in "$@"; do _is_uint "$o" || return 1; [ "$o" -le 255 ] || return 1; cnt=$((cnt+1)); done
  [ "$cnt" -eq 4 ]
}
is_valid_cidr() { # a.b.c.d/nn  (nn 0-32)
  local c="$1" ip mask
  case "$c" in */*) ;; *) return 1 ;; esac
  ip="${c%/*}"; mask="${c#*/}"
  case "$c" in */*/*) return 1 ;; esac        # more than one slash
  _is_uint "$mask" || return 1
  [ "$mask" -ge 0 ] && [ "$mask" -le 32 ] || return 1
  is_valid_ipv4 "$ip"
}
is_valid_routes() { # comma-separated CIDR list; empty is allowed (no routes)
  local list one IFS=,
  list="$(_strip_ws "$1")"
  [ -z "$list" ] && return 0
  case "$list" in ,*|*,|*,,*) return 1 ;; esac # no leading/trailing/empty elements
  # shellcheck disable=SC2086
  for one in $list; do is_valid_cidr "$one" || return 1; done
  return 0
}
is_nonempty_routes() { # like is_valid_routes but at least one CIDR (gateway needs one)
  [ -n "$(_strip_ws "$1")" ] || return 1
  is_valid_routes "$1"
}
# detect_local_cidrs prints this server's DIRECTLY-CONNECTED private subnets
# (comma-separated) as a SUGGESTED default for the advertise prompt. It reads
# the kernel's scope-link routes — subnets the box can genuinely reach — and
# keeps only RFC1918 ranges, dropping loopback, docker/bridge/veth, the WG mesh
# interface, and non-private (public) NICs. This is a suggestion the operator
# confirms or WIDENS (a /24 box may want the whole corp /16); it never advertises
# anything on its own. Empty output = nothing safe to suggest (prompt stays blank).
detect_local_cidrs() {
  command -v ip >/dev/null 2>&1 || return 0
  local line cidr dev out=""
  while IFS= read -r line; do
    [ -n "$line" ] || continue
    cidr="${line%% *}"
    dev="$(printf '%s\n' "$line" | sed -n 's/.* dev \([^ ]*\).*/\1/p')"
    case "$dev" in lo|docker*|br-*|veth*|zpoa[0-9]*|tun[0-9]*|wg[0-9]*|tailscale*|cni*|flannel*|cali*|kube*) continue ;; esac
    case "$cidr" in
      10.*|192.168.*|172.1[6-9].*|172.2[0-9].*|172.3[0-1].*) : ;;
      *) continue ;;
    esac
    [ "$cidr" = "$ZPOA_MESH_CIDR" ] && continue
    case ",$out," in *",$cidr,"*) continue ;; esac   # dedup
    out="${out:+$out,}$cidr"
  done <<EOF
$(ip -o -f inet route show scope link 2>/dev/null)
EOF
  printf '%s' "$out"
}
# detect_local_dns prints "domains|resolvers" — the server's own search domain(s)
# and PRIVATE resolver(s) from /etc/resolv.conf, as a suggested corp-DNS push.
# Only RFC1918 nameservers qualify: a public resolver (8.8.8.8) is not corp DNS
# and must never be pushed to clients. Either side empty ⇒ nothing to suggest.
detect_local_dns() {
  local rc=/etc/resolv.conf dom="" res="" line w
  [ -r "$rc" ] || { printf '|'; return 0; }
  while IFS= read -r line; do
    case "$line" in
      search\ *|domain\ *)
        for w in ${line#* }; do
          case "$w" in
            ''|localdomain|local|lan) : ;;
            *) case ",$dom," in *",$w,"*) : ;; *) dom="${dom:+$dom,}$w" ;; esac ;;
          esac
        done ;;
      nameserver\ *)
        w="$(_strip_ws "${line#nameserver }")"
        case "$w" in
          10.*|192.168.*|172.1[6-9].*|172.2[0-9].*|172.3[0-1].*)
            case ",$res," in *",$w,"*) : ;; *) res="${res:+$res,}$w" ;; esac ;;
        esac ;;
    esac
  done < "$rc"
  printf '%s|%s' "$dom" "$res"
}
is_valid_slug() { # 1-63 chars, [a-z0-9-], no leading/trailing hyphen
  local s="$1"
  [ -n "$s" ] && [ "${#s}" -le 63 ] || return 1
  case "$s" in -*|*-|*[!a-z0-9-]*) return 1 ;; esac
  return 0
}
is_valid_hostname() { # dotted, labels [a-z0-9-], at least one dot, <=253 chars
  local h="$1"
  [ -n "$h" ] && [ "${#h}" -le 253 ] || return 1
  case "$h" in
    *.*) ;; *) return 1 ;;                     # require at least one dot
  esac
  case "$h" in .*|*.|*..*|*[!a-z0-9.-]*) return 1 ;; esac
  return 0
}
is_pg_url() { # postgres://[user[:pass]@]host[:port]/db[?params] — lenient (real check is at connect time)
  case "$1" in postgres://*|postgresql://*) ;; *) return 1 ;; esac
  # require a host and a non-empty /database after the scheme
  printf '%s' "$1" | grep -qiE '^postgres(ql)?://[^/[:space:]]+/[^/?[:space:]]+' || return 1
  return 0
}

# prompt_valid VAR "question" "default" validator "hint" [normalizer]
# TTY: re-prompt until the (optionally normalized) answer passes the validator.
# non-TTY (unattended): normalize+validate the env/default value once, die on bad input.
prompt_valid() {
  local __var="$1" __q="$2" __def="$3" __val="$4" __hint="$5" __norm="${6:-}" __cur
  if ! on_tty; then
    __cur="$__def"
    [ -n "$__norm" ] && __cur="$("$__norm" "$__cur")"
    printf -v "$__var" '%s' "$__cur"
    "$__val" "$__cur" || die "invalid ${__var}='${__cur:-<empty>}' — $__hint"
    return
  fi
  while :; do
    prompt "$__var" "$__q" "$__def"
    __cur="${!__var}"
    if [ -n "$__norm" ]; then __cur="$("$__norm" "$__cur")"; printf -v "$__var" '%s' "$__cur"; fi
    "$__val" "$__cur" && return
    warn "$__hint"
  done
}

# json_str '<json>' <key> -> the string value of a top-level "key":"..." field.
# Prefers python3/jq; falls back to sed for a flat object. Always exits 0.
json_str() {
  local j="$1" k="$2"
  if command -v python3 >/dev/null 2>&1; then
    printf '%s' "$j" | python3 -c 'import sys,json
try:
    d=json.load(sys.stdin); v=d.get(sys.argv[1]); print(v if isinstance(v,str) else "")
except Exception:
    pass' "$k" 2>/dev/null || true
    return
  fi
  if command -v jq >/dev/null 2>&1; then
    printf '%s' "$j" | jq -r --arg k "$k" '.[$k] // empty' 2>/dev/null || true
    return
  fi
  printf '%s' "$j" | sed -n 's/.*"'"$k"'"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | head -1 || true
}

# cloud_exchange: trade a ZPOA Cloud bootstrap token for the AUTHORITATIVE tenant
# identity (slug/name/issuer) from the SaaS. A valid token proves the tenant
# exists — this replaces the typo-prone free-text slug. Fails closed.
cloud_exchange() {
  local api body slug name regcp issuer tok="$ZPOA_CLOUD_TOKEN"
  # Self-describing token: the Workspace may embed the API endpoint IN the token
  # as "<token>.<base64(api)>" so the install command needn't show a URL. When a
  # trailing ".<base64-of-an-https-url>" is present, use it as the API and strip
  # it back to the raw token. A plain token (no dot) falls back to ZPOA_CLOUD_API.
  case "$tok" in
    *.*)
      local __enc="${tok##*.}" __raw="${tok%.*}" __dec
      __dec="$(printf '%s' "$__enc" | base64 -d 2>/dev/null || true)"
      case "$__dec" in https://*|http://*) ZPOA_CLOUD_API="$__dec"; tok="$__raw" ;; esac
      ;;
  esac
  api="${ZPOA_CLOUD_API%/}"
  log "Verifying tenant with ZPOA Cloud…"
  # Cloud registration (Tier A): pass this CP's public URL alongside the token.
  # The token proves tenant ownership, so the SaaS records control_plane_url on
  # the tenant — the tenant's Workspace then auto-shows this self-host CP with no
  # manual URL entry. ZPOA_DOMAIN is already set + validated above.
  body="$(curl -fsS --max-time 30 -X POST -H 'Content-Type: application/json' \
      -d "{\"token\":\"${tok}\",\"control_plane_url\":\"https://${ZPOA_DOMAIN}\"}" \
      "${api}/api/v1/selfhost/bootstrap/exchange" 2>/dev/null)" \
    || die "ZPOA Cloud rejected the bootstrap token (invalid, expired, or already used) — or ${api} is unreachable. Fix ZPOA_CLOUD_TOKEN / ZPOA_CLOUD_API, or omit it to self-host standalone."
  slug="$(json_str "$body" tenant_slug)"
  name="$(json_str "$body" tenant_name)"
  regcp="$(json_str "$body" control_plane_url)"
  issuer="$(json_str "$body" oidc_issuer_url)"
  [ -n "$slug" ] || die "ZPOA Cloud returned no tenant for that token (invalid/expired/used)."
  is_valid_slug "$slug" || die "ZPOA Cloud returned an unexpected tenant slug '$slug' — contact support."
  ZPOA_TENANT="$slug"
  [ -n "$name" ] && ZPOA_TENANT_NAME="$name"
  # Auto-configure SSO from the exchange: adopt the tenant's OIDC issuer so the CP
  # trusts Workspace LOGINS out of the box — the tenant's Workspace lists Devices
  # with NO admin-token paste, and it survives redeploys (the login, not a rotating
  # token, is the credential). An explicit ZPOA_KEYCLOAK_ISSUER_URL always wins.
  if [ -z "$ZPOA_KEYCLOAK_ISSUER_URL" ] && [ -n "$issuer" ]; then
    ZPOA_KEYCLOAK_ISSUER_URL="$issuer"
  fi
  ok "linked to ZPOA Cloud tenant '${ZPOA_TENANT}'$( [ -n "${ZPOA_TENANT_NAME:-}" ] && printf ' (%s)' "$ZPOA_TENANT_NAME" )"
  [ -n "$regcp" ] && ok "registered this control plane (${regcp}) — the tenant's Workspace will auto-show it"
  [ -n "$ZPOA_KEYCLOAK_ISSUER_URL" ] && ok "SSO auto-configured (issuer ${ZPOA_KEYCLOAK_ISSUER_URL}) — the tenant's Workspace signs in with no token"
  # Hand the AUTHORITATIVE result to the control plane (written into .env by
  # write_env → CYBER_CLOUD_* env). The server persists the SAME cloud_link row a
  # Helm/GitOps deploy self-registers, so `zpoa-access cloud-link` status is
  # identical across install paths. The single-use token was already spent above —
  # the server records this result, it NEVER re-exchanges. Set only on a real link.
  ZPOA_CLOUD_TENANT_SLUG="$slug"
  ZPOA_CLOUD_TENANT_NAME="$name"
  ZPOA_CLOUD_OIDC_ISSUER="$issuer"
  ZPOA_CLOUD_CONTROL_PLANE_URL="$regcp"
}

need_root() { [ "$(id -u)" -eq 0 ] || die "run as root (sudo). e.g.  curl -fsSL <url>/install.sh | sudo bash"; }

detect_arch() {
  case "$(uname -m)" in
    x86_64|amd64) ARCH=amd64 ;;
    aarch64|arm64) ARCH=arm64 ;;
    *) die "unsupported architecture: $(uname -m)" ;;
  esac
}

ensure_docker() {
  if command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then
    ok "Docker + compose present ($(docker --version | awk '{print $3}' | tr -d ,))"; return
  fi
  log "Installing Docker Engine + compose plugin..."
  curl -fsSL https://get.docker.com | sh >/dev/null 2>&1 || die "Docker install failed — install Docker manually and re-run"
  systemctl enable --now docker >/dev/null 2>&1 || true
  docker compose version >/dev/null 2>&1 || die "docker compose plugin missing after install"
  ok "Docker installed"
}

# ---------------------------------------------------------------------------
# Gather config
# ---------------------------------------------------------------------------
gather_config() {
  # Public hostname (required). On an unattended run with nothing set, fail
  # early with the specific message; otherwise validate + re-prompt on a TTY.
  if [ -z "$ZPOA_DOMAIN" ] && ! on_tty; then
    die "ZPOA_DOMAIN is required (public hostname clients connect to)"
  fi
  prompt_valid ZPOA_DOMAIN "Public hostname for this control plane (e.g. access.acme.com)" \
    "$ZPOA_DOMAIN" is_valid_hostname "enter a valid public hostname, e.g. access.acme.com (letters, digits, dots, hyphens)" _lc_trim

  # Tenant identity. Two modes:
  #   • Cloud-linked — a ZPOA Cloud bootstrap token (env or pasted) is exchanged
  #     for the AUTHORITATIVE slug/name/issuer from the SaaS. The token proves
  #     the tenant exists, so there is no typo and no shared-realm fallback.
  #   • Standalone — no token: the slug is prompted (default = first DNS label),
  #     validated locally, and bootstrap CREATES that tenant on this box.
  # Offer to link when interactive and nothing was pre-specified.
  if [ -z "$ZPOA_CLOUD_TOKEN" ] && [ -z "$ZPOA_TENANT" ] && on_tty; then
    prompt ZPOA_CLOUD_TOKEN "ZPOA Cloud bootstrap token to link an existing tenant (blank = standalone)" ""
  fi
  if [ -n "$ZPOA_CLOUD_TOKEN" ]; then
    cloud_exchange
  else
    local __derived
    __derived="$(printf '%s' "$ZPOA_DOMAIN" | cut -d. -f1 | tr -cd 'a-z0-9-')"
    [ -z "$__derived" ] && __derived="default"
    if [ -n "$ZPOA_TENANT" ]; then
      ZPOA_TENANT="$(_lc_trim "$ZPOA_TENANT")"
      is_valid_slug "$ZPOA_TENANT" || die "invalid ZPOA_TENANT='$ZPOA_TENANT' — lowercase letters, digits, hyphens only (1-63 chars, no leading/trailing '-')"
    else
      prompt_valid ZPOA_TENANT "Tenant / organization slug (lowercase a-z 0-9 -)" \
        "$__derived" is_valid_slug "lowercase letters, digits, hyphens only — no spaces, no leading/trailing '-'" _lc_trim
    fi
  fi
  [ -z "$ZPOA_TENANT_NAME" ] && ZPOA_TENANT_NAME="$ZPOA_TENANT"

  # Admin email (optional — Let's Encrypt just wants a contact; no strict check).
  on_tty && prompt ZPOA_ADMIN_EMAIL "Admin email (Let's Encrypt notices, optional)" "$ZPOA_ADMIN_EMAIL"

  # Database — bundled (co-located Postgres) vs external / HA Postgres.
  #   ZPOA_DB_URL (env/flag) wins for unattended runs; otherwise ASK on a TTY.
  #   'External' points the CP at your own managed/HA Postgres. Either way the
  #   control plane creates its OWN schema on first boot (embedded migrations run
  #   on every start), so there is never a manual table-deployment step. Pointing
  #   at an HA endpoint (managed multi-AZ, or a VIP over primary+standby) is what
  #   removes the DB single-point-of-failure — a single DB, on-box or external,
  #   is still a SPOF.
  if [ -z "$ZPOA_DB_URL" ] && on_tty; then
    printf '\n%sDatabase for this control plane:%s\n' "$c_bold" "$c_reset"
    printf '  %s1)%s Bundled   — run Postgres in a container on THIS box (simplest; single box = SPOF).\n' "$c_cyn" "$c_reset"
    printf '  %s2)%s External  — connect to your own managed / HA Postgres (recommended for production).\n' "$c_cyn" "$c_reset"
    printf '             The server creates its own schema on first boot — no manual tables.\n'
    local __dbans=""
    read -r -p "Choose [1/2] (default 1): " __dbans </dev/tty || true
    if [ "$__dbans" = "2" ]; then
      prompt_valid ZPOA_DB_URL "External Postgres URL (postgres://user:pass@host:5432/db?sslmode=require)" \
        "" is_pg_url "must be a postgres:// or postgresql:// URL including a host and /database" _strip_ws
      ok "external DB set — the bundled Postgres container will not be deployed"
    fi
  fi

  # Corp routes (optional). Pre-fill the prompt with the server's directly-
  # connected private subnet(s) as a SUGGESTION — the operator confirms with
  # Enter or widens to their full corp range. Interactive-only: a scripted run
  # (no TTY / --yes) never auto-advertises a detected subnet; it stays blank
  # unless ZPOA_ROUTES was passed explicitly.
  local __routes_q="Corp CIDR(s) to make reachable via VPN, comma-separated (blank = none)"
  if on_tty && [ -z "$ZPOA_ROUTES" ]; then
    ZPOA_ROUTES="$(detect_local_cidrs)"
    if [ -n "$ZPOA_ROUTES" ]; then
      log "Detected this server's directly-connected subnet(s): ${c_bold}${ZPOA_ROUTES}${c_reset}"
      log "  Enter = advertise it · type a wider range (e.g. 10.0.0.0/16) to override · type 'none' to skip."
      __routes_q="Corp CIDR(s) to advertise (Enter = detected, or type your own; 'none' to skip)"
    fi
  fi
  # Validate as a comma-separated CIDR list and re-prompt on anything malformed
  # (e.g. "10.dujdhued.89" or a missing /mask). _norm_routes lets 'none' clear a
  # pre-filled suggestion, so the detected value is never forced on the operator.
  prompt_valid ZPOA_ROUTES "$__routes_q" \
    "$ZPOA_ROUTES" is_valid_routes "each entry must be a CIDR like 10.0.0.0/16 (comma-separated; 'none' or blank = skip)" _norm_routes

  # Corp DNS push (split-DNS) — only meaningful once a subnet is advertised (the
  # corp resolver must be reachable over a routed CIDR). Suggest the server's own
  # search domain + private resolver from /etc/resolv.conf; the operator confirms
  # or skips with 'none'. Interactive-only, never auto-pushed in scripted runs.
  if on_tty && [ -n "$ZPOA_ROUTES" ] && [ -z "${ZPOA_DNS_SEARCH_DOMAINS}${ZPOA_DNS_CORP_RESOLVERS}" ]; then
    local __dns __dom __res
    __dns="$(detect_local_dns)"; __dom="${__dns%%|*}"; __res="${__dns#*|}"
    if [ -n "$__dom" ] && [ -n "$__res" ]; then
      log "Detected corp DNS — search domain(s): ${c_bold}${__dom}${c_reset}  resolver(s): ${c_bold}${__res}${c_reset}"
      log "  Push it so clients resolve internal names (e.g. fileserver.${__dom%%,*}) over the VPN? Enter = yes · 'none' = skip."
      prompt ZPOA_DNS_SEARCH_DOMAINS "Search domain(s) to push, comma-separated ('none' to skip)" "$__dom"
      ZPOA_DNS_SEARCH_DOMAINS="$(_norm_routes "$ZPOA_DNS_SEARCH_DOMAINS")"
      if [ -n "$ZPOA_DNS_SEARCH_DOMAINS" ]; then
        prompt ZPOA_DNS_CORP_RESOLVERS "Corp DNS resolver IP(s) for those domains, comma-separated" "$__res"
        ZPOA_DNS_CORP_RESOLVERS="$(_strip_ws "$ZPOA_DNS_CORP_RESOLVERS")"
      fi
    fi
  fi

  # Mesh pool (defaulted; validate in case it was overridden with junk).
  is_valid_cidr "$ZPOA_MESH_CIDR" || die "invalid ZPOA_MESH_CIDR='$ZPOA_MESH_CIDR' — expected a CIDR like 100.64.0.0/16"

  # Secrets — generated on THIS box, never in a UI.
  [ -z "$ZPOA_ADMIN_TOKEN" ] && ZPOA_ADMIN_TOKEN="$(gen_secret)"
  [ -z "$ZPOA_VAULT_KEY" ]   && ZPOA_VAULT_KEY="$(gen_secret)"
  PG_PASSWORD="$(gen_secret)"
  # DERP relay this node advertises to clients (in the netmap). A WARM STANDBY
  # (role=backup, i.e. ZPOA_PRIMARY_URL set) MUST advertise the PRIMARY's relay,
  # NOT its own hostname: this build has no DERP meshing, so a backup advertising
  # itself forms a SECOND, un-meshed relay island and cross-relay peer traffic is
  # silently black-holed (sender sees no error, retries forever). `zpoa ha promote`
  # flips this to our own domain on takeover. A primary/gateway (no PRIMARY_URL)
  # advertises itself, as before.
  if [ -z "$ZPOA_DERP_RELAYS" ]; then
    if [ -n "$ZPOA_PRIMARY_URL" ]; then
      __ph="${ZPOA_PRIMARY_URL#https://}"; __ph="${__ph#http://}"; __ph="${__ph%%/*}"; __ph="${__ph%%:*}"
      ZPOA_DERP_RELAYS="${__ph}:443:3478"
    else
      ZPOA_DERP_RELAYS="${ZPOA_DOMAIN}:443:3478"
    fi
  fi

  # A gateway is co-located when routes are given (or role explicitly gateway).
  WITH_GATEWAY=false
  { [ -n "$ZPOA_ROUTES" ] || [ "$ZPOA_ROLE" = "gateway" ]; } && WITH_GATEWAY=true

  # Image mode
  if [ -n "$ZPOA_IMAGE" ]; then BUILD_LOCAL=false; else BUILD_LOCAL=true; ZPOA_IMAGE="zpoa-access"; ZPOA_IMAGE_TAG="local"; fi

  case "$ZPOA_TLS" in
    auto)     CADDY_TLS_DIRECTIVE="" ;;
    internal) CADDY_TLS_DIRECTIVE="tls internal" ;;
    off)      CADDY_TLS_DIRECTIVE="" ;;
    *) die "ZPOA_TLS must be auto|internal|off" ;;
  esac
}

# ---------------------------------------------------------------------------
# Fetch + verify the prebuilt binary (build-local mode)
# ---------------------------------------------------------------------------
fetch_binary() {
  local base="${ZPOA_BINARY_BASE}/${ZPOA_VERSION}"
  local url="${base}/zpoa-access-linux-${ARCH}"
  log "Downloading zpoa-access (${ZPOA_VERSION}, ${ARCH})..."
  curl -fSL --connect-timeout 30 --max-time 600 -o "$ZPOA_DIR/zpoa-access" "$url" \
    || die "failed to download binary from $url"
  # Verify the checksum. The release pipeline publishes a per-arch bare-hash
  # file next to each binary (zpoa-access-linux-<arch>.sha256, just the 64-hex
  # digest); prefer that. Fall back to a combined SHA256SUMS if a mirror ships
  # that format instead.
  local got want=""
  got="$(sha256sum "$ZPOA_DIR/zpoa-access" | awk '{print $1}')"
  if curl -fsSL --max-time 30 -o "$ZPOA_DIR/zpoa-access.sha256" "${url}.sha256" 2>/dev/null; then
    want="$(grep -oiE '[0-9a-f]{64}' "$ZPOA_DIR/zpoa-access.sha256" | head -1)"
  elif curl -fsSL --max-time 30 -o "$ZPOA_DIR/SHA256SUMS" "${base}/SHA256SUMS" 2>/dev/null; then
    want="$(grep "zpoa-access-linux-${ARCH}" "$ZPOA_DIR/SHA256SUMS" | awk '{print $1}' | head -1)"
  fi
  if [ -n "$want" ]; then
    [ "$want" = "$got" ] || die "checksum mismatch for the binary (want $want got $got) — aborting"
    ok "binary checksum verified"
  else
    warn "no checksum published for ${ZPOA_VERSION} — skipping verify (pin ZPOA_VERSION in prod)"
  fi
  chmod +x "$ZPOA_DIR/zpoa-access"
}

# ---------------------------------------------------------------------------
# Emit the deployment files into $ZPOA_DIR
# ---------------------------------------------------------------------------
write_env() {
  cat > "$ZPOA_DIR/.env" <<EOF
# Generated by install.sh on $(uname -n) — do not commit; contains secrets.
ZPOA_IMAGE=${ZPOA_IMAGE}
IMAGE_TAG=${ZPOA_IMAGE_TAG}
HOSTNAME=${ZPOA_DOMAIN}
TENANT_ID=${ZPOA_TENANT}
TENANT_NAME=${ZPOA_TENANT_NAME}
ADMIN_TOKEN=${ZPOA_ADMIN_TOKEN}
ADMIN_EMAIL=${ZPOA_ADMIN_EMAIL}
VIEWER_TOKEN=${ZPOA_VIEWER_TOKEN}
KEYCLOAK_ISSUER_URL=${ZPOA_KEYCLOAK_ISSUER_URL}
VAULT_KEY=${ZPOA_VAULT_KEY}
POSTGRES_PASSWORD=${PG_PASSWORD}
MESH_CIDR=${ZPOA_MESH_CIDR}
# Pre-exchanged ZPOA Cloud link (set only when a bootstrap token was used). The
# control plane records these as its cloud_link row on first boot — same DB state
# a Helm/GitOps deploy self-registers, no re-exchange of the spent token.
CLOUD_TENANT_SLUG=${ZPOA_CLOUD_TENANT_SLUG:-}
CLOUD_TENANT_NAME=${ZPOA_CLOUD_TENANT_NAME:-}
CLOUD_OIDC_ISSUER=${ZPOA_CLOUD_OIDC_ISSUER:-}
CLOUD_CONTROL_PLANE_URL=${ZPOA_CLOUD_CONTROL_PLANE_URL:-}
GATEWAY_ROUTES=${ZPOA_ROUTES}
GATEWAY_SNAT=${ZPOA_GATEWAY_SNAT}
# VPN→SIEM shipper (optional; env-driven, empty = disabled). Ships VPN
# security events over HTTPS to a SIEM ingest API to drive detection +
# response. Set ZPOA_SIEM_INGEST_URL / ZPOA_SIEM_INGEST_KEY to enable.
SIEM_INGEST_URL=${ZPOA_SIEM_INGEST_URL:-}
SIEM_INGEST_KEY=${ZPOA_SIEM_INGEST_KEY:-}
WG_PORT=${ZPOA_WG_PORT}
DERP_RELAYS=${ZPOA_DERP_RELAYS}
VENDOR_IDP_HOSTS=${ZPOA_VENDOR_IDP_HOSTS}
DNS_SEARCH_DOMAINS=${ZPOA_DNS_SEARCH_DOMAINS}
DNS_CORP_RESOLVERS=${ZPOA_DNS_CORP_RESOLVERS}
CADDY_HOSTNAME=${ZPOA_DOMAIN}
CADDY_TLS=${CADDY_TLS_DIRECTIVE}
EOF
  # External DB overrides the bundled one.
  if [ -n "$ZPOA_DB_URL" ]; then echo "DATABASE_URL=${ZPOA_DB_URL}" >> "$ZPOA_DIR/.env"; fi
  chmod 600 "$ZPOA_DIR/.env"
}

write_dockerfile() {
  cat > "$ZPOA_DIR/Dockerfile" <<'EOF'
# Wraps the public prebuilt zpoa-access binary; iptables/iproute2 are needed
# for gateway (subnet-router) mode.
FROM debian:stable-slim
RUN apt-get update && apt-get install -y --no-install-recommends \
      ca-certificates iptables iproute2 wget && rm -rf /var/lib/apt/lists/*
COPY zpoa-access /usr/local/bin/zpoa-access
RUN chmod +x /usr/local/bin/zpoa-access
EXPOSE 8090 8081 3478/udp 51820/udp
ENTRYPOINT ["/usr/local/bin/zpoa-access"]
EOF
}

write_caddyfile() {
  # Routes /derp* (WebSocket, no buffering) to the in-process DERP relay on
  # :8081; everything else to the control API on :8090. Caddy auto-provisions
  # a Let's Encrypt cert for the hostname (or `tls internal` when air-gapped).
  cat > "$ZPOA_DIR/Caddyfile" <<'EOF'
{$CADDY_HOSTNAME} {
    encode gzip
    {$CADDY_TLS}
    @derp path /derp*
    handle @derp {
        reverse_proxy control:8081 {
            flush_interval -1
        }
    }
    handle {
        reverse_proxy control:8090 {
            header_up Host {host}
            header_up X-Forwarded-Proto {scheme}
            header_up X-Forwarded-For {remote}
        }
    }
    log {
        output stdout
        format console
    }
}
EOF
}

write_compose() {
  local db_url image_line build_line pg_service redis_addr
  image_line="    image: \${ZPOA_IMAGE:-zpoa-access}:\${IMAGE_TAG:-local}"
  if [ "$BUILD_LOCAL" = true ]; then
    build_line="    build:
      context: .
      dockerfile: Dockerfile"
  else
    build_line=""
  fi
  if [ -n "$ZPOA_DB_URL" ]; then
    db_url="\${DATABASE_URL}"; pg_service=""; DEPENDS=""
  else
    db_url="postgres://zpoa:\${POSTGRES_PASSWORD}@postgres:5432/zpoa?sslmode=disable"
    DEPENDS="    depends_on:
      postgres:
        condition: service_healthy"
    pg_service="  postgres:
    image: postgres:16-alpine
    container_name: zpoa-access-postgres
    restart: unless-stopped
    environment:
      POSTGRES_USER: zpoa
      POSTGRES_PASSWORD: \${POSTGRES_PASSWORD}
      POSTGRES_DB: zpoa
    volumes:
      - postgres-data:/var/lib/postgresql/data
    healthcheck:
      test: [\"CMD-SHELL\", \"pg_isready -U zpoa -d zpoa\"]
      interval: 5s
      timeout: 3s
      retries: 10
    networks: [zpoa]
"
  fi

  cat > "$ZPOA_DIR/docker-compose.yml" <<EOF
# Generated by install.sh — control plane + in-process DERP relay (+ Postgres).
# Layer docker-compose.gateway.yml to co-locate the WireGuard subnet-router.
services:
  control:
$( [ "$BUILD_LOCAL" = true ] && echo "$build_line" )
$image_line
    container_name: zpoa-access-control
    restart: unless-stopped
$DEPENDS
    environment:
      CYBER_MODE: "server"
      CYBER_SERVER_HOST: "0.0.0.0"
      CYBER_SERVER_PORT: "8090"
      CYBER_HOSTNAME: \${HOSTNAME}
      CYBER_TENANT_MODE: "single"
      CYBER_TENANT_ID: \${TENANT_ID:-default}
      CYBER_DATABASE_URL: "$db_url"
      CYBER_VAULT_KEY: \${VAULT_KEY:-}
      CYBER_ADMIN_TOKEN: \${ADMIN_TOKEN:-}
      CYBER_ADMIN_EMAIL: \${ADMIN_EMAIL:-}
      CYBER_VIEWER_TOKEN: \${VIEWER_TOKEN:-}
      CYBER_KEYCLOAK_ISSUER_URL: \${KEYCLOAK_ISSUER_URL:-}
      # Pre-exchanged ZPOA Cloud link (from install.sh cloud_exchange). The server
      # records these as its cloud_link row on first boot — no token re-exchange.
      CYBER_CLOUD_TENANT_SLUG: "\${CLOUD_TENANT_SLUG:-}"
      CYBER_CLOUD_TENANT_NAME: "\${CLOUD_TENANT_NAME:-}"
      CYBER_CLOUD_OIDC_ISSUER: "\${CLOUD_OIDC_ISSUER:-}"
      CYBER_CLOUD_CONTROL_PLANE_URL: "\${CLOUD_CONTROL_PLANE_URL:-}"
      CYBER_VENDOR_IDP_HOSTS: "\${VENDOR_IDP_HOSTS:-}"
      # Corp DNS pushed to clients as split-DNS (resolve internal names over the
      # VPN). Both empty = no push. OpenVPN push "dhcp-option DNS/DOMAIN" analog.
      CYBER_DNS_SEARCH_DOMAINS: "\${DNS_SEARCH_DOMAINS:-}"
      CYBER_DNS_CORP_RESOLVERS: "\${DNS_CORP_RESOLVERS:-}"
      CYBER_VPN_NETWORK_CIDR: \${MESH_CIDR:-100.64.0.0/16}
      # In-process NAT-traversal relay (no separate container).
      CYBER_DERP_RELAY_ENABLED: "true"
      CYBER_DERP_RELAYS: \${DERP_RELAYS:-}
      CYBER_DISCOVERY_CONTROL_PLANE_URL: "https://\${HOSTNAME}"
      CYBER_DISCOVERY_TENANT_ID: \${TENANT_ID:-default}
      CYBER_DISCOVERY_TENANT_NAME: \${TENANT_NAME:-default}
      CYBER_DISCOVERY_DERP_RELAYS: \${DERP_RELAYS:-}
    ports:
      - "3478:3478/udp"          # STUN (DERP relay)
    volumes:
      - control-data:/var/lib/zpoa
    healthcheck:
      test: ["CMD", "wget", "-qO-", "http://localhost:8090/healthz"]
      interval: 10s
      timeout: 3s
      retries: 6
      start_period: 20s
    networks: [zpoa]

$pg_service
  caddy:
    image: caddy:2-alpine
    container_name: zpoa-access-caddy
    restart: unless-stopped
    depends_on: [control]
    ports:
      - "80:80"
      - "443:443"
    environment:
      CADDY_HOSTNAME: \${CADDY_HOSTNAME}
      CADDY_TLS: \${CADDY_TLS:-}
    volumes:
      - ./Caddyfile:/etc/caddy/Caddyfile:ro
      - caddy-data:/data
      - caddy-config:/config
    networks: [zpoa]

volumes:
  control-data:
$( [ -z "$ZPOA_DB_URL" ] && echo "  postgres-data:" )
  caddy-data:
  caddy-config:

networks:
  zpoa:
    driver: bridge
EOF

  # Gateway override (co-located subnet-router) — same image, privileged bits.
  cat > "$ZPOA_DIR/docker-compose.gateway.yml" <<'EOF'
services:
  control:
    environment:
      CYBER_GATEWAY_ENABLED: "true"
      CYBER_GATEWAY_INTERFACE: "zpoa0"
      CYBER_GATEWAY_LISTEN_PORT: "51820"
      CYBER_GATEWAY_ADVERTISED_ROUTES: ${GATEWAY_ROUTES:-}
      # SNAT on by default (corp LAN needs no return route). Set ZPOA_GATEWAY_SNAT=false
      # to preserve client mesh source IPs into the LAN (then route 100.64.0.0/10 back
      # via this box; not for exit-node gateways). Tailscale --snat-subnet-routes.
      CYBER_GATEWAY_SNAT: "${GATEWAY_SNAT:-true}"
    ports:
      - "${WG_PORT:-51820}:51820/udp"
    cap_add: [NET_ADMIN]
    sysctls:
      net.ipv4.ip_forward: "1"
    devices:
      - /dev/net/tun:/dev/net/tun
EOF
}

# ---------------------------------------------------------------------------
# Day-2 control: a GLOBAL `zpoa` command (on PATH for every user) + a small
# manifest it reads to find this install. Fronts `docker compose exec …
# zpoa-access` so day-2 ops don't require cd-ing into $ZPOA_DIR or knowing the
# compose invocation — the native feel of `tailscale` / `cloudflared`.
# ---------------------------------------------------------------------------
write_install_conf() {
  mkdir -p /etc/zpoa
  cat > /etc/zpoa/install.conf <<EOF
# Z-Access install manifest — read by /usr/local/bin/zpoa. Generated by install.sh.
ZPOA_DIR=${ZPOA_DIR}
ZPOA_TENANT=${ZPOA_TENANT}
ZPOA_DOMAIN=${ZPOA_DOMAIN}
ZPOA_ROLE=${ZPOA_ROLE:-server}
ZPOA_PRIMARY_URL=${ZPOA_PRIMARY_URL:-}
ZPOA_WITH_GATEWAY=${WITH_GATEWAY}
ZPOA_HA_PRIMARY=false
ZPOA_BUILD_LOCAL=${BUILD_LOCAL}
EOF
  chmod 644 /etc/zpoa/install.conf
}

write_zpoa_cli() {
  # Quoted heredoc: the wrapper is fully static — it sources install.conf at
  # runtime for the install-specific values (dir, tenant, domain, gateway).
  cat > /usr/local/bin/zpoa <<'ZPOA_CLI'
#!/usr/bin/env bash
# zpoa — day-2 control for a self-hosted Z-Access control plane.
# Installed by install.sh; reads /etc/zpoa/install.conf. Run from any directory.
set -euo pipefail

CONF="${ZPOA_CONF:-/etc/zpoa/install.conf}"
c_reset=$'\033[0m'; c_bold=$'\033[1m'; c_grn=$'\033[32m'; c_yel=$'\033[33m'; c_red=$'\033[31m'; c_cyn=$'\033[36m'
log()  { printf '%s==>%s %s\n' "$c_cyn" "$c_reset" "$*"; }
die()  { printf '%s ✗ %s%s\n' "$c_red" "$*" "$c_reset" >&2; exit 1; }
ok()   { printf '%s ✓ %s%s\n' "$c_grn" "$*" "$c_reset"; }

[ -r "$CONF" ] || die "$CONF not found — run this on the control-plane host (installed by install.sh)."
# shellcheck disable=SC1090
. "$CONF"
DIR="${ZPOA_DIR:-/opt/zpoa-access}"
TENANT="${ZPOA_TENANT:-default}"
[ -d "$DIR" ] || die "install dir $DIR is missing (per $CONF)"

rebuild_cargs() {
  cargs=(-f docker-compose.yml)
  # Include the subnet-router overlay when routes are advertised. Check the
  # PERSISTED GATEWAY_ROUTES (.env = source of truth) as well as the
  # ZPOA_WITH_GATEWAY flag, so `zpoa dc`/recreate (and any wrapper-driven `up`)
  # never drop the gateway's NET_ADMIN + TUN — which would break LAN forwarding.
  # (grep inline, not env_val: this runs before env_val is defined.)
  local routes=""
  [ -f "$DIR/.env" ] && routes="$(grep -E '^GATEWAY_ROUTES=' "$DIR/.env" 2>/dev/null | head -1 | cut -d= -f2-)"
  if [ "${ZPOA_WITH_GATEWAY:-false}" = "true" ] || [ -n "$routes" ]; then cargs+=(-f docker-compose.gateway.yml); fi
  if [ "${ZPOA_HA_PRIMARY:-false}" = "true" ] && [ -f "$DIR/docker-compose.ha-primary.yml" ]; then cargs+=(-f docker-compose.ha-primary.yml); fi
  return 0
}
rebuild_cargs
dc() { ( cd "$DIR" && docker compose "${cargs[@]}" "$@" ); }
za() { dc exec -T control zpoa-access "$@"; }          # admin CLI inside the container
psql_super() { dc exec -T postgres psql -U zpoa -d "${1:-zpoa}" -v ON_ERROR_STOP=1; }  # stdin SQL
recreate() { rebuild_cargs; dc up -d; }

env_val()     { grep "^${1}=" "$DIR/.env" 2>/dev/null | head -1 | cut -d= -f2- ; }
admin_token() { env_val ADMIN_TOKEN; }
gen_pw()      { openssl rand -hex 24 2>/dev/null || head -c24 /dev/urandom | od -An -tx1 | tr -d ' \n'; }
this_host_ip(){ hostname -I 2>/dev/null | awk '{print $1}'; }

set_env()  { local k="$1" v="$2" f="$DIR/.env";  if grep -q "^${k}=" "$f" 2>/dev/null; then sed -i "s|^${k}=.*|${k}=${v}|" "$f"; else printf '%s=%s\n' "$k" "$v" >> "$f"; fi; }
set_conf() { local k="$1" v="$2";                 if grep -q "^${k}=" "$CONF" 2>/dev/null; then sed -i "s|^${k}=.*|${k}=${v}|" "$CONF"; else printf '%s=%s\n' "$k" "$v" >> "$CONF"; fi; }

# CIDR validation (mirrors the installer) — reject malformed routes before they
# reach the gateway config, where a bad value just breaks the container.
_wsq()      { printf '%s' "$1" | tr -d '[:space:]'; }
valid_cidr()  { local c="$1" ip m o IFS; case "$c" in */*/*|*/|/*) return 1 ;; */*) ;; *) return 1 ;; esac
  ip="${c%/*}"; m="${c#*/}"; case "$m" in ''|*[!0-9]*) return 1 ;; esac; [ "$m" -le 32 ] || return 1
  IFS=.; set -- $ip; [ "$#" -eq 4 ] || return 1
  for o in "$@"; do case "$o" in ''|*[!0-9]*) return 1 ;; esac; [ "$o" -le 255 ] || return 1; done; return 0; }
valid_cidrs() { local l s IFS=,; s="$(_wsq "$1")"; [ -z "$s" ] && return 1; case "$s" in ,*|*,|*,,*) return 1 ;; esac; for l in $s; do valid_cidr "$l" || return 1; done; return 0; }

# Pass through to the container admin CLI, auto-injecting --tenant-slug.
passthru() {
  local cmd="$1"; shift
  local a=("$@")
  case " ${a[*]:-} " in *" --tenant-slug "*) : ;; *) a+=(--tenant-slug "$TENANT") ;; esac
  za "$cmd" "${a[@]}"
}

usage() {
  cat <<USAGE
${c_bold}zpoa${c_reset} — Z-Access control-plane day-2 CLI  (tenant: ${TENANT}, dir: ${DIR})

  ${c_bold}zpoa status${c_reset}                       stack health + mesh reachability
  ${c_bold}zpoa routes advertise${c_reset} <CIDR,..>   make corp subnet(s) reachable (co-locates a gateway)
  ${c_bold}zpoa routes list${c_reset}                  advertised routes + approval state
  ${c_bold}zpoa routes approve${c_reset} <ROUTE-ID>    approve + enable a route
  ${c_bold}zpoa authkey create${c_reset} [--kind endpoint|mdm|gateway] [--reusable] [--expiry-days N]
  ${c_bold}zpoa authkey list|revoke${c_reset} [--id <ID>]
  ${c_bold}zpoa device list|approve|deny${c_reset} [--id <ID>] [--reason <text>]
  ${c_bold}zpoa device delete${c_reset} --device-id <DEVICE-ID>
  ${c_bold}zpoa config show|set${c_reset} [--device-approval-required true|false] [--magic-dns-enabled ..] ...
  ${c_bold}zpoa set cp-url${c_reset} <hostname>         change control-plane URL (clients auto-follow)
  ${c_bold}zpoa set mesh-cidr${c_reset} <CIDR>          change the mesh address pool
  ${c_bold}zpoa ha status${c_reset}                     replication role + cluster + streaming health
  ${c_bold}zpoa ha prepare-primary${c_reset}            (on PRIMARY) make it replication-ready for a backup
  ${c_bold}zpoa ha add-backup${c_reset} <backup-url>    (on PRIMARY) register a backup so clients fail over to it
  ${c_bold}zpoa ha promote${c_reset}                    (on BACKUP) promote this standby → primary
  ${c_bold}zpoa logs${c_reset} [service]                follow logs (default: control)
  ${c_bold}zpoa restart${c_reset} [service]             restart the stack (or one service)
  ${c_bold}zpoa upgrade${c_reset}                       pull a newer image + recreate
  ${c_bold}zpoa uninstall${c_reset}                     stop + remove containers and volumes
USAGE
}

cmd="${1:-status}"; [ $# -gt 0 ] && shift || true
case "$cmd" in
  status)
    printf '%sZ-Access%s @ %shttps://%s%s  (tenant: %s)\n\n' "$c_bold" "$c_reset" "$c_bold" "${ZPOA_DOMAIN:-?}" "$c_reset" "$TENANT"
    dc ps
    echo
    za mesh-status --tenant-slug "$TENANT" 2>/dev/null || true
    ;;
  routes|route)
    sub="${1:-list}"; [ $# -gt 0 ] && shift || true
    case "$sub" in
      advertise)
        cidrs="$(_wsq "${1:-}")"; [ -n "$cidrs" ] || die "usage: zpoa routes advertise <CIDR[,CIDR...]>"
        valid_cidrs "$cidrs" || die "not a valid CIDR list: '$cidrs'  (expected e.g. 10.0.0.0/16,192.168.5.0/24)"
        set_env GATEWAY_ROUTES "$cidrs"
        ZPOA_WITH_GATEWAY=true; set_conf ZPOA_WITH_GATEWAY true
        log "advertising ${cidrs} (recreating with subnet-router)…"
        recreate
        printf 'Approve when it appears:  %szpoa routes list%s  →  %szpoa routes approve <ROUTE-ID>%s\n' "$c_bold" "$c_reset" "$c_bold" "$c_reset"
        ;;
      list)    za route list --tenant-slug "$TENANT" ;;
      approve) id="${1:-}"; [ -n "$id" ] || die "usage: zpoa routes approve <ROUTE-ID>"; za route approve --tenant-slug "$TENANT" --id "$id" ;;
      *) die "usage: zpoa routes advertise|list|approve …" ;;
    esac
    ;;
  authkey|device|config|mesh-status|gateway-install) passthru "$cmd" "$@" ;;
  ha)
    sub="${1:-status}"; [ $# -gt 0 ] && shift || true
    DOMAIN="${ZPOA_DOMAIN:-}"; ROLE="${ZPOA_ROLE:-server}"
    case "$sub" in
      status)
        printf '%sZ-Access HA%s  (this node: %s%s%s, %shttps://%s%s)\n\n' "$c_bold" "$c_reset" "$c_bold" "$ROLE" "$c_reset" "$c_bold" "${DOMAIN:-?}" "$c_reset"
        # Local replication role straight from Postgres (authoritative).
        role_local="$(printf 'SELECT CASE WHEN pg_is_in_recovery() THEN '\''standby'\'' ELSE '\''primary'\'' END;' | psql_super 2>/dev/null | sed -n 3p | tr -d '[:space:]')"
        printf '  DB role (pg_is_in_recovery) : %s%s%s\n' "$c_bold" "${role_local:-unknown}" "$c_reset"
        if [ "$role_local" = standby ]; then
          # On a standby: is the stream live + how far behind?
          lag="$(printf "SELECT COALESCE(EXTRACT(EPOCH FROM (now()-pg_last_xact_replay_timestamp()))::int::text,'n/a');" | psql_super 2>/dev/null | sed -n 3p | tr -d '[:space:]')"
          conn="$(printf 'SELECT count(*) FROM pg_stat_wal_receiver;' | psql_super 2>/dev/null | sed -n 3p | tr -d '[:space:]')"
          printf '  streaming from primary     : %s\n' "$( [ "${conn:-0}" -ge 1 ] 2>/dev/null && printf '%syes%s' "$c_grn" "$c_reset" || printf '%sNO (link down)%s' "$c_red" "$c_reset" )"
          printf '  replay lag (seconds)       : %s\n' "${lag:-n/a}"
        else
          reps="$(printf 'SELECT count(*) FROM pg_stat_replication;' | psql_super 2>/dev/null | sed -n 3p | tr -d '[:space:]')"
          printf '  connected standbys         : %s\n' "${reps:-0}"
        fi
        # Cluster view via the admin API (self + probed peers) when a token exists.
        tok="$(admin_token)"
        if [ -n "$tok" ] && [ -n "$DOMAIN" ]; then
          echo; log "cluster (via https://${DOMAIN}/api/v1/admin/ha/cluster):"
          curl -fsS --max-time 10 -H "Authorization: Bearer $tok" "https://${DOMAIN}/api/v1/admin/ha/cluster" 2>/dev/null | sed 's/^/    /' || printf '    (unavailable)\n'
        fi
        ;;
      promote)
        [ "$ROLE" = backup ] || printf '%s !  this node is role=%s (expected backup) — promoting anyway if it is a standby%s\n' "$c_yel" "$ROLE" "$c_reset"
        tok="$(admin_token)"; [ -n "$tok" ] || die "no ADMIN_TOKEN in $DIR/.env"
        [ -n "$DOMAIN" ] || die "no ZPOA_DOMAIN known for this node"
        printf 'Promote %shttps://%s%s to PRIMARY? This runs pg_promote + makes it the preferred CP. [y/N] ' "$c_bold" "$DOMAIN" "$c_reset"
        { read -r a </dev/tty; } 2>/dev/null || a=""
        [ "$a" = y ] || [ "$a" = Y ] || { echo "aborted (no confirmation — run from a terminal, or promote at the DB: SELECT pg_promote();)"; exit 0; }
        log "promoting…"
        curl -fsS --max-time 30 -X POST -H "Authorization: Bearer $tok" "https://${DOMAIN}/api/v1/admin/ha/promote" | sed 's/^/    /' \
          || die "promote call failed — is this node reachable at https://${DOMAIN}? You can also promote at the DB: SELECT pg_promote();"
        # We are primary now: advertise OUR relay (not the old primary's) so clients
        # re-home their DERP here. A backup was installed with DERP_RELAYS pointing at
        # the old primary (so all clients shared one relay island); promotion MUST flip
        # it to this node, or clients keep dialing the dead relay and peer traffic that
        # falls back to relay is black-holed.
        want_relay="${DOMAIN}:443:3478"
        if [ "$(env_val DERP_RELAYS)" != "$want_relay" ]; then
          log "pointing DERP relay at this node (${want_relay}) + reloading control…"
          set_env DERP_RELAYS "$want_relay"
          recreate || printf '%s !  DERP_RELAYS set in .env but container reload failed — run: zpoa dc up -d%s\n' "$c_yel" "$c_reset"
        fi
        echo; ok "promotion requested — verify with: zpoa ha status"
        ;;
      prepare-primary)
        [ "$ROLE" = backup ] && die "run this on the PRIMARY, not a backup"
        rl="${ZPOA_REPL_USER:-zpoa_replicator}"
        rp="${ZPOA_REPL_PASSWORD:-$(gen_pw)}"
        log "creating replication role '${rl}' + tuning WAL retention on the primary DB…"
        printf "DO \$\$ BEGIN IF EXISTS (SELECT FROM pg_roles WHERE rolname='%s') THEN ALTER ROLE %s WITH REPLICATION LOGIN PASSWORD '%s'; ELSE CREATE ROLE %s WITH REPLICATION LOGIN PASSWORD '%s'; END IF; END \$\$;\nALTER SYSTEM SET wal_keep_size='512MB';\nSELECT pg_reload_conf();\n" \
          "$rl" "$rl" "$rp" "$rl" "$rp" | psql_super >/dev/null || die "failed to create replication role"
        # Allow replication connections from anywhere (SSL/scram-authenticated) — idempotent.
        dc exec -T postgres sh -c 'PGDATA="${PGDATA:-/var/lib/postgresql/data}"; grep -q "replication '"$rl"'" "$PGDATA/pg_hba.conf" 2>/dev/null || echo "host replication '"$rl"' 0.0.0.0/0 scram-sha-256" >> "$PGDATA/pg_hba.conf"' || die "failed to update pg_hba.conf"
        printf 'SELECT pg_reload_conf();\n' | psql_super >/dev/null || true
        # Publish Postgres :5432 on the host so a backup box can reach it.
        cat > "$DIR/docker-compose.ha-primary.yml" <<'YML'
# Generated by `zpoa ha prepare-primary` — publishes Postgres to the host so a
# backup control plane can pg_basebackup + stream from it. Layered automatically.
services:
  postgres:
    ports:
      - "5432:5432"
YML
        set_conf ZPOA_HA_PRIMARY true; ZPOA_HA_PRIMARY=true
        log "exposing Postgres :5432 on the host + reloading the stack…"
        recreate
        ip="$(this_host_ip)"; vk="$(env_val VAULT_KEY)"; pg="$(env_val POSTGRES_PASSWORD)"; at="$(env_val ADMIN_TOKEN)"
        cat <<DONE

$c_grn✓ primary is replication-ready.$c_reset  Replication role: ${c_bold}${rl}${c_reset}

${c_bold}On the BACKUP host, run the installer with this block${c_reset} (replace the
backup hostname; the DB host must be an address the backup can actually reach):

  ${c_bold}ZPOA_ROLE=backup \\
  ZPOA_DOMAIN=access-backup.example.com \\
  ZPOA_PRIMARY_URL=https://${DOMAIN:-<primary-host>} \\
  ZPOA_PRIMARY_DB_HOST=${ip:-<primary-host>}:5432 \\
  ZPOA_REPL_USER=${rl} \\
  ZPOA_REPL_PASSWORD=${rp} \\
  ZPOA_PG_PASSWORD=${pg} \\
  ZPOA_VAULT_KEY=${vk} \\
  ZPOA_ADMIN_TOKEN=${at} \\
    curl -fsSL <installer-url> | sudo bash${c_reset}

Then back here: ${c_bold}zpoa ha add-backup https://access-backup.example.com${c_reset}
(For fenced 2-node AUTO failover add ${c_bold}ZPOA_AUTO_PROMOTE=true${c_reset} on the backup.)
DONE
        ;;
      add-backup)
        [ "$ROLE" = backup ] && die "run this on the PRIMARY (the writable node)"
        url="${1:-}"; [ -n "$url" ] || die "usage: zpoa ha add-backup <https://backup-host>"
        case "$url" in https://*|http://*) ;; *) die "backup URL must start with https:// (e.g. https://access-backup.acme.com)" ;; esac
        self="https://${DOMAIN}"
        log "registering backup ${url} in control_plane_urls (clients auto-adopt via netmap)…"
        # Defensive: append to control_plane_urls wherever that column lives, keeping
        # the primary first + de-duping. No-op-safe if the row/col is absent.
        printf "DO \$\$
DECLARE r record;
BEGIN
  FOR r IN SELECT table_schema, table_name FROM information_schema.columns WHERE column_name='control_plane_urls' LOOP
    EXECUTE format('UPDATE %%I.%%I SET control_plane_urls = (SELECT array(SELECT DISTINCT e FROM unnest(ARRAY[%%L]::text[] || coalesce(control_plane_urls, ARRAY[]::text[]) || ARRAY[%%L]::text[]) e WHERE e <> '''' )) ', r.table_schema, r.table_name, '%s', '%s');
  END LOOP;
END \$\$;\n" "$self" "$url" | psql_super >/dev/null || die "failed to update control_plane_urls (is this the writable primary?)"
        ok "backup registered. Clients will learn https fail-over URLs on their next netmap."
        printf 'Verify:  zpoa ha status   (and check %s serves /health)\n' "$url"
        ;;
      *) die "usage: zpoa ha status|prepare-primary|add-backup <url>|promote" ;;
    esac
    ;;
  set)
    key="${1:-}"; val="${2:-}"
    case "$key" in
      cp-url)
        [ -n "$val" ] || die "usage: zpoa set cp-url <hostname>"
        host="${val#https://}"; host="${host#http://}"; host="${host%%/*}"
        set_env HOSTNAME "$host"; set_env CADDY_HOSTNAME "$host"; set_conf ZPOA_DOMAIN "$host"
        log "control-plane URL → https://${host} (recreating; clients auto-follow via netmap)…"
        recreate ;;
      mesh-cidr)
        val="$(_wsq "$val")"; [ -n "$val" ] || die "usage: zpoa set mesh-cidr <CIDR>"
        valid_cidr "$val" || die "not a valid CIDR: '$val'  (expected e.g. 100.64.0.0/16)"
        set_env MESH_CIDR "$val"; log "mesh CIDR → ${val} (recreating)…"; recreate ;;
      *) die "usage: zpoa set cp-url|mesh-cidr <value>" ;;
    esac
    ;;
  logs)    dc logs -f "${1:-control}" ;;
  restart) [ $# -gt 0 ] && dc restart "$@" || dc restart ;;
  upgrade)
    [ "${ZPOA_BUILD_LOCAL:-false}" = "true" ] && die "this install builds a local image — re-run install.sh with a newer ZPOA_VERSION to upgrade"
    log "pulling newer image…"; dc pull && recreate ;;
  uninstall)
    printf 'Remove the Z-Access stack + data volumes at %s? [y/N] ' "$DIR"; { read -r a </dev/tty; } 2>/dev/null || a=""
    [ "$a" = y ] || [ "$a" = Y ] || exit 0
    dc down -v || true
    printf 'Removed containers + volumes. Config remains: %s, /etc/zpoa, /usr/local/bin/zpoa\n' "$DIR" ;;
  help|-h|--help) usage ;;
  *) die "unknown command: $cmd  (run 'zpoa help')" ;;
esac
ZPOA_CLI
  chmod 0755 /usr/local/bin/zpoa
}

# ---------------------------------------------------------------------------
# Bring the stack up + self-bootstrap
# ---------------------------------------------------------------------------
compose_args() {
  local a=(-f docker-compose.yml)
  # Co-locate the subnet-router overlay whenever routes are advertised. Key off
  # the PERSISTED GATEWAY_ROUTES (.env = source of truth) as well as the
  # WITH_GATEWAY flag: the overlay grants the control container the NET_ADMIN +
  # /dev/net/tun it needs to actually forward + SNAT mesh traffic into the LAN,
  # so a base-only `up` (which would silently strip the gateway) must not happen
  # when routes exist.
  local routes=""
  [ -f "$ZPOA_DIR/.env" ] && routes="$(grep -E '^GATEWAY_ROUTES=' "$ZPOA_DIR/.env" 2>/dev/null | head -1 | cut -d= -f2-)"
  { [ "$WITH_GATEWAY" = true ] || [ -n "$routes" ]; } && a+=(-f docker-compose.gateway.yml)
  printf '%s\n' "${a[@]}"
}

bring_up() {
  cd "$ZPOA_DIR"
  mapfile -t CARGS < <(compose_args)
  if [ "$BUILD_LOCAL" = true ]; then log "Building local image from the prebuilt binary..."; docker build -q -t zpoa-access:local "$ZPOA_DIR" >/dev/null || die "image build failed"; ok "image zpoa-access:local built"; fi
  log "Starting the stack (docker compose up -d)..."
  docker compose "${CARGS[@]}" up -d || die "docker compose up failed"

  log "Waiting for the control plane to become healthy..."
  local i healthy=false
  for i in $(seq 1 30); do
    if docker compose "${CARGS[@]}" exec -T control wget -qO- http://localhost:8090/healthz >/dev/null 2>&1; then ok "control plane healthy"; healthy=true; break; fi
    sleep 2
  done
  [ "$healthy" = true ] || warn "control plane not healthy yet — check: (cd $ZPOA_DIR && docker compose logs control)"

  # Caddy (the public TLS front-end) — check it EXPLICITLY. A bad Caddyfile
  # crash-loops Caddy while the control plane stays healthy on :8090, so without
  # this we'd report a false "up" with no HTTPS reachable. We only assert the
  # process is running (not that an ACME cert is already issued — that can still
  # be in flight and needs public DNS + :80).
  local cstate=""
  for i in $(seq 1 10); do
    cstate="$(docker compose "${CARGS[@]}" ps caddy --format '{{.State}}' 2>/dev/null | head -1)"
    [ "$cstate" = running ] && break
    case "$cstate" in restarting|exited|dead) break ;; esac
    sleep 2
  done
  case "$cstate" in
    running) ok "caddy (TLS front-end) running" ;;
    "")      warn "caddy state unknown — check: (cd $ZPOA_DIR && docker compose logs caddy)" ;;
    *)       warn "caddy is '$cstate' — the HTTPS front-end is NOT serving (the control plane itself is fine). This usually means a Caddyfile or port-bind problem. Recent caddy logs:"
             docker compose "${CARGS[@]}" logs caddy --tail 10 2>&1 | sed 's/^/    /' ;;
  esac
}

bootstrap_tenant() {
  cd "$ZPOA_DIR"
  mapfile -t CARGS < <(compose_args)
  log "Bootstrapping first tenant '${ZPOA_TENANT}'..."
  local bargs=(bootstrap --tenant-name "$ZPOA_TENANT_NAME" --tenant-slug "$ZPOA_TENANT" --control-url "https://${ZPOA_DOMAIN}")
  [ -n "$ZPOA_ROUTES" ] && bargs+=(--gateway-routes "$ZPOA_ROUTES")
  BOOTSTRAP_OUT="$(docker compose "${CARGS[@]}" exec -T control zpoa-access "${bargs[@]}" 2>&1 || true)"
  printf '%s\n' "$BOOTSTRAP_OUT" | sed 's/^/    /'
}

open_firewall() {
  [ "$ZPOA_OPEN_FIREWALL" = true ] || return 0
  command -v ufw >/dev/null 2>&1 || return 0
  ufw status 2>/dev/null | grep -q "Status: active" || return 0
  log "Opening firewall (ufw): 80,443/tcp 3478/udp$( [ "$WITH_GATEWAY" = true ] && echo " ${ZPOA_WG_PORT}/udp" )"
  ufw allow 80/tcp   >/dev/null 2>&1 || true
  ufw allow 443/tcp  >/dev/null 2>&1 || true
  ufw allow 3478/udp >/dev/null 2>&1 || true
  [ "$WITH_GATEWAY" = true ] && ufw allow "${ZPOA_WG_PORT}/udp" >/dev/null 2>&1 || true
  [ "$WITH_GATEWAY" = true ] && open_gateway_hairpin
}

# _gw_bridge_subnet: prints the docker-bridge subnet the co-located gateway
# container runs on (e.g. 172.19.0.0/16). Finds it via the container publishing
# the WG port, else by the well-known control/gateway container name.
_gw_bridge_subnet() {
  command -v docker >/dev/null 2>&1 || return 1
  local c n
  c="$(docker ps --format '{{.Names}} {{.Ports}}' 2>/dev/null | awk -v p=":${ZPOA_WG_PORT}->" 'index($0,p){print $1; exit}')"
  [ -z "$c" ] && c="$(docker ps --format '{{.Names}}' 2>/dev/null | grep -E 'zpoa-access-(control|gateway)' | head -1)"
  [ -z "$c" ] && return 1
  n="$(docker inspect "$c" --format '{{range $k,$v := .NetworkSettings.Networks}}{{$k}} {{end}}' 2>/dev/null | awk '{print $1}')"
  [ -z "$n" ] && return 1
  docker network inspect "$n" --format '{{range .IPAM.Config}}{{.Subnet}}{{end}}' 2>/dev/null | grep -E '^[0-9]+\.' | head -1
}

# open_gateway_hairpin: when this box is BOTH the subnet-router gateway AND hosts
# services reachable over the mesh (RDP/SSH/etc.), mesh traffic to the box's OWN
# IP arrives masqueraded as the gateway container's docker-bridge source. The
# admin's LAN-scoped ufw rules (e.g. "3389 ALLOW 10.0.0.0/16") don't cover that
# source, so it is dropped — even though reaching OTHER hosts on the same
# advertised CIDR works (they get re-SNAT'd to this host's LAN IP). Fix: mirror
# every LAN/host-restricted inbound ALLOW port onto the gateway's bridge subnet,
# so mesh clients get exactly the access the LAN already has. Ports already open
# to Anywhere include the bridge, so we skip them. Idempotent; safe to re-run.
open_gateway_hairpin() {
  command -v ufw >/dev/null 2>&1 || return 0
  ufw status 2>/dev/null | grep -q "Status: active" || return 0
  local sub; sub="$(_gw_bridge_subnet)" || return 0
  [ -n "$sub" ] || return 0
  log "Mirroring LAN-restricted ufw ports to gateway subnet ${sub} (so mesh clients can reach THIS box's own services, e.g. RDP/SSH)"
  ufw status 2>/dev/null | grep -E '\bALLOW\b' | grep -vi 'v6' | while read -r _to _act _from _rest; do
    [ "$_act" = "ALLOW" ] || continue          # skip header / OUT rules
    [ "$_from" = "Anywhere" ] && continue        # already reachable from the bridge
    case "$_to" in [0-9]*) : ;; *) continue ;; esac   # numeric ports only
    if printf '%s' "$_to" | grep -q '/'; then
      ufw allow from "$sub" to any port "${_to%%/*}" proto "${_to##*/}" >/dev/null 2>&1 || true
    else
      ufw allow from "$sub" to any port "$_to" >/dev/null 2>&1 || true
    fi
  done
}

summary() {
  local enroll="curl -fsSL https://${ZPOA_DOMAIN}/install.sh?key=<ENROLL_KEY> | sudo bash"
  # Paste-once connection blob for the Workspace: base64({url,token,tenant}) with
  # a scheme prefix. Works for ANY install (minted or plain curl) — the admin
  # pastes it into the Workspace Control Plane URL field and it auto-fills the
  # URL + admin token, so the self-host CP is manageable without re-typing.
  local __blob
  __blob="zpoa-connect:$(printf '{"url":"https://%s","token":"%s","tenant":"%s"}' \
      "$ZPOA_DOMAIN" "$ZPOA_ADMIN_TOKEN" "$ZPOA_TENANT" | base64 2>/dev/null | tr -d '\n')"
  cat <<EOF

${c_grn}${c_bold}Z-Access control plane is up.${c_reset}

  Control plane URL : ${c_bold}https://${ZPOA_DOMAIN}${c_reset}
  Admin token       : ${c_bold}${ZPOA_ADMIN_TOKEN}${c_reset}
                      (Bearer for the admin API / staticToken for the Workspace GUI — store it safely)
  Install dir       : ${ZPOA_DIR}   (.env holds all secrets, chmod 600)
$( [ "$WITH_GATEWAY" = true ] && echo "  Advertised routes : ${ZPOA_ROUTES}" )

${c_bold}Day 2 — one global command, from anywhere on the box:${c_reset}
  • ${c_bold}zpoa status${c_reset}                              stack health + mesh reachability
  • ${c_bold}zpoa authkey create --kind endpoint${c_reset}      mint an enrollment key
  • ${c_bold}zpoa routes advertise <CIDR>${c_reset}             make a corp subnet reachable
  • ${c_bold}zpoa routes approve <ROUTE-ID>${c_reset}           approve it
  • ${c_bold}zpoa set cp-url <hostname>${c_reset}               move the control-plane URL
  • ${c_bold}zpoa help${c_reset}                                full command list
  (installed at /usr/local/bin/zpoa — no cd, no docker compose to remember)

  Or drive everything by API:
      curl -H "Authorization: Bearer ${ZPOA_ADMIN_TOKEN}" https://${ZPOA_DOMAIN}/api/v1/vpn/devices

${c_bold}Connect the Zpoa Workspace (manage this CP from the GUI — no re-typing):${c_reset}
  Paste this ONE line into  Settings → Secure Client → Control Plane URL:
      ${c_bold}${__blob}${c_reset}
  It auto-fills the URL + admin token. (Manual alternative: Control Plane URL →
  Self-host → https://${ZPOA_DOMAIN} ; staticToken = the admin token above.)

  DNS: point ${ZPOA_DOMAIN} at this host's public IP (needed for Let's Encrypt).
  Logs: ${c_bold}zpoa logs${c_reset}
EOF
}

# ---------------------------------------------------------------------------
# Remote gateway path (--role gateway on a box that is NOT the control plane)
# ---------------------------------------------------------------------------
run_remote_gateway() {
  [ -n "$ZPOA_CONTROL_URL" ] || die "--role gateway needs ZPOA_CONTROL_URL (your control plane URL)"
  case "$ZPOA_CONTROL_URL" in https://*|http://*) ;; *) die "invalid ZPOA_CONTROL_URL='$ZPOA_CONTROL_URL' — must start with https:// (e.g. https://access.acme.com)" ;; esac
  [ -n "$ZPOA_AUTH_KEY" ]    || die "--role gateway needs ZPOA_AUTH_KEY (mint via 'zpoa-access gateway-install' on the CP)"
  [ -n "$ZPOA_ROUTES" ]      || die "--role gateway needs ZPOA_ROUTES (CIDR to advertise)"
  ZPOA_ROUTES="$(_strip_ws "$ZPOA_ROUTES")"
  is_valid_routes "$ZPOA_ROUTES" || die "invalid ZPOA_ROUTES='$ZPOA_ROUTES' — expected CIDR(s) like 10.0.0.0/16 (comma-separated)"
  ensure_docker
  # Image resolution. Prefer an explicit ZPOA_IMAGE (registry/pull mode) when the
  # operator has one; OTHERWISE build locally from the PUBLIC prebuilt binary —
  # exactly like full self-host does — so gateway mode needs NO published
  # container image. (The old default pulled ghcr.io/zpoasec/zpoa-access, which
  # is NOT public, so `--role gateway` failed at `docker run` with an image-pull
  # error. Building from the S3 binary is the same public artifact self-host uses.)
  local img
  if [ -n "$ZPOA_IMAGE" ]; then
    img="${ZPOA_IMAGE}:${ZPOA_IMAGE_TAG:-latest}"
    log "Pulling gateway image ${img}..."
    docker pull "$img" || die "failed to pull gateway image ${img} — set ZPOA_IMAGE to a pullable image, or leave it unset to build locally from ${ZPOA_BINARY_BASE}"
  else
    mkdir -p "$ZPOA_DIR"
    write_dockerfile
    fetch_binary
    log "Building local gateway image from the prebuilt binary..."
    docker build -q -t zpoa-access:local "$ZPOA_DIR" >/dev/null || die "gateway image build failed"
    img="zpoa-access:local"
    ok "image zpoa-access:local built"
  fi
  log "Starting remote subnet-router gateway -> ${ZPOA_CONTROL_URL} (${ZPOA_ROUTES})"
  docker rm -f zpoa-gateway >/dev/null 2>&1 || true
  docker run -d --name zpoa-gateway --restart unless-stopped \
    --cap-add NET_ADMIN --device /dev/net/tun --sysctl net.ipv4.ip_forward=1 \
    -p "${ZPOA_WG_PORT}:51820/udp" -v zpoa-gateway:/var/lib/zpoa \
    -e CYBER_MODE=gateway -e CYBER_GATEWAY_ENABLED=true -e CYBER_GATEWAY_INTERFACE=zpoa0 \
    -e CYBER_GATEWAY_LISTEN_PORT=51820 -e CYBER_API_URL="$ZPOA_CONTROL_URL" \
    -e CYBER_AUTH_KEY="$ZPOA_AUTH_KEY" -e CYBER_GATEWAY_ADVERTISED_ROUTES="$ZPOA_ROUTES" \
    "$img" || die "gateway container failed to start"
  ok "gateway started (docker logs -f zpoa-gateway)"
}

# ---------------------------------------------------------------------------
# Deployment mode: gateway-only (join a CP, no DB) vs full self-host (CP+DB).
# Explicit flags/env win (unattended); otherwise ASK on a TTY.
# ---------------------------------------------------------------------------
select_mode() {
  # Already implied by env/flags? Backup FIRST (a backup also sets ZPOA_DOMAIN,
  # so its signal — a primary URL — must win over the plain-server check below).
  if [ "$ZPOA_ROLE" = "backup" ] || [ -n "$ZPOA_PRIMARY_URL" ]; then
    ZPOA_ROLE=backup
    # Backup mode is DB-level replication of the PRIMARY's bundled Postgres
    # (pg_basebackup + streaming). An External/HA DB has no local Postgres to
    # replicate, so the two are mutually exclusive. Fail loudly instead of
    # silently ignoring ZPOA_DB_URL.
    if [ -n "$ZPOA_DB_URL" ]; then
      die "backup mode replicates the PRIMARY's bundled Postgres and is incompatible with an External/HA DB (ZPOA_DB_URL is set). If your database is already HA (e.g. RDS Multi-AZ / a VIP over primary+standby) you do NOT need a backup CP for the DB — deploy a SECOND full self-host (role=server) pointing ZPOA_DB_URL at the same database for control-plane redundancy instead."
    fi
    return
  fi
  if [ "$ZPOA_ROLE" = "gateway" ] || [ -n "$ZPOA_CONTROL_URL" ] || [ -n "$ZPOA_AUTH_KEY" ]; then ZPOA_ROLE=gateway; return; fi
  if [ -n "$ZPOA_DOMAIN" ] || [ -n "$ZPOA_CLOUD_TOKEN" ]; then ZPOA_ROLE=server; return; fi
  if ! on_tty; then ZPOA_ROLE="${ZPOA_ROLE:-server}"; return; fi
  printf '\n%sHow do you want to deploy Z-Access on this server?%s\n' "$c_bold" "$c_reset"
  printf '  %s1)%s Gateway only    — bridge my network to a control plane (ZPOA Cloud or my own).\n' "$c_cyn" "$c_reset"
  printf '                    No database here; the control plane + identity stay on the CP.\n'
  printf '  %s2)%s Full self-host  — run the whole control plane here (CP + database + relay + gateway).\n' "$c_cyn" "$c_reset"
  printf '  %s3)%s Backup CP       — a warm standby of an EXISTING self-host primary (HA failover).\n' "$c_cyn" "$c_reset"
  printf '                    Streams the primary DB; promote it if the primary dies.\n'
  local ans=""
  read -r -p "Choose [1/2/3] (default 2): " ans </dev/tty || true
  case "$ans" in
    1) ZPOA_ROLE=gateway ;;
    3) ZPOA_ROLE=backup ;;
    *) ZPOA_ROLE=server ;;
  esac
}

# Self-describing gateway key: <key>.<base64(control_plane_url)>. The Workspace's
# "Mint gateway key" embeds the CP URL in the key so the install one-liner need
# NOT expose the control-plane URL. Decode it into ZPOA_CONTROL_URL (unless one
# was set explicitly) and reduce ZPOA_AUTH_KEY to the raw key. A plain key (no
# embedded URL) is left untouched. Mirrors the ZPOA_CLOUD_TOKEN format.
decode_selfdescribing_authkey() {
  case "$ZPOA_AUTH_KEY" in
    *.*)
      local __enc="${ZPOA_AUTH_KEY##*.}" __raw="${ZPOA_AUTH_KEY%.*}" __dec
      __dec="$(printf '%s' "$__enc" | base64 -d 2>/dev/null || true)"
      case "$__dec" in
        https://*|http://*) [ -z "$ZPOA_CONTROL_URL" ] && ZPOA_CONTROL_URL="$__dec"; ZPOA_AUTH_KEY="$__raw" ;;
      esac
      ;;
  esac
}

# Prompts for gateway mode: which CP to join (default ZPOA Cloud), the tenant's
# gateway enrollment key, and the routes to advertise.
gather_gateway_config() {
  decode_selfdescribing_authkey   # env-provided key from the Workspace one-liner
  if [ -z "$ZPOA_CONTROL_URL" ]; then
    if on_tty; then prompt ZPOA_CONTROL_URL "Control plane to join (Enter = ZPOA Cloud)" "$ZPOA_CLOUD_CP"
    else ZPOA_CONTROL_URL="$ZPOA_CLOUD_CP"; fi
  fi
  case "$ZPOA_CONTROL_URL" in https://*|http://*) ;; *) die "invalid control-plane URL '$ZPOA_CONTROL_URL' — must start with https:// (e.g. https://access.zpoa.com)" ;; esac
  if [ -z "$ZPOA_AUTH_KEY" ]; then
    on_tty || die "gateway mode needs ZPOA_AUTH_KEY (a gateway enrollment key minted in the control plane for your tenant)"
    prompt ZPOA_AUTH_KEY "Gateway enrollment key for your tenant (mint it in the CP: Sites & Gateways / API)" ""
    [ -n "$ZPOA_AUTH_KEY" ] || die "a gateway enrollment key is required"
    decode_selfdescribing_authkey   # allow pasting a self-describing key too
  fi
  prompt_valid ZPOA_ROUTES "Corp CIDR(s) this gateway makes reachable, comma-separated (e.g. 10.0.0.0/16)" \
    "$ZPOA_ROUTES" is_nonempty_routes "at least one CIDR is required, e.g. 10.0.0.0/16 (comma-separated)" _strip_ws
}

open_gateway_firewall() {
  [ "$ZPOA_OPEN_FIREWALL" = true ] || return 0
  command -v ufw >/dev/null 2>&1 || return 0
  ufw status 2>/dev/null | grep -q "Status: active" || return 0
  ufw allow "${ZPOA_WG_PORT}/udp" >/dev/null 2>&1 || true
  log "Opened firewall (ufw): ${ZPOA_WG_PORT}/udp"
  open_gateway_hairpin
}

gateway_summary() {
  cat <<EOF

${c_grn}${c_bold}Gateway is running.${c_reset}  (subnet router — no database on this box)

  Joined control plane : ${c_bold}${ZPOA_CONTROL_URL}${c_reset}
  Advertising routes    : ${c_bold}${ZPOA_ROUTES}${c_reset}
  Logs                  : docker logs -f zpoa-gateway

${c_bold}Finish on the control plane (${ZPOA_CONTROL_URL}):${c_reset}
  1. Approve this gateway's routes so clients start using them
       (admin → Routes & DNS → approve; self-host CP: zpoa routes approve <id>).
  2. Employees install the Secure Client from the Employee Portal → Downloads —
       it already points at ${ZPOA_CONTROL_URL}. Once the routes are approved,
       their traffic to ${ZPOA_ROUTES} flows through THIS gateway automatically.

  Note: this gateway and your employees must be in the SAME tenant — mint the
  gateway key above under that tenant.
EOF
}

# ---------------------------------------------------------------------------
# Backup control-plane path (--role backup) — a warm standby of an existing
# self-host primary. Seeds a Postgres streaming replica from the primary via
# pg_basebackup, then runs zpoa-access as a read-only standby with the failover
# watchdog wired up. Promote it (Tier 0 one-click, or Tier 0.5 fenced auto) when
# the primary dies. See docs/high-availability.md.
# ---------------------------------------------------------------------------
gather_backup_config() {
  # This backup's own public hostname (its slot in control_plane_urls).
  if [ -z "$ZPOA_DOMAIN" ] && ! on_tty; then
    die "backup mode needs ZPOA_DOMAIN (this backup's own public hostname, e.g. access-backup.acme.com)"
  fi
  prompt_valid ZPOA_DOMAIN "Public hostname for THIS backup control plane (e.g. access-backup.acme.com)" \
    "$ZPOA_DOMAIN" is_valid_hostname "enter a valid public hostname (letters, digits, dots, hyphens)" _lc_trim

  # The primary this backup follows.
  if [ -z "$ZPOA_PRIMARY_URL" ]; then
    on_tty || die "backup mode needs ZPOA_PRIMARY_URL (the primary control-plane URL, e.g. https://access.acme.com)"
    prompt ZPOA_PRIMARY_URL "Primary control-plane URL this backup follows (https://access.acme.com)" ""
  fi
  case "$ZPOA_PRIMARY_URL" in https://*|http://*) ;; *) die "invalid ZPOA_PRIMARY_URL='$ZPOA_PRIMARY_URL' — must start with https:// (e.g. https://access.acme.com)" ;; esac

  # Primary Postgres address reachable from here (for pg_basebackup + streaming +
  # the split-brain fence). Default = host part of the primary URL on :5432.
  if [ -z "$ZPOA_PRIMARY_DB_HOST" ]; then
    local __h="${ZPOA_PRIMARY_URL#https://}"; __h="${__h#http://}"; __h="${__h%%/*}"; __h="${__h%%:*}"
    if on_tty; then prompt ZPOA_PRIMARY_DB_HOST "Primary Postgres host[:port] reachable from here (Enter = ${__h}:5432)" "${__h}:5432"
    else ZPOA_PRIMARY_DB_HOST="${__h}:5432"; fi
  fi
  case "$ZPOA_PRIMARY_DB_HOST" in *:*) ;; *) ZPOA_PRIMARY_DB_HOST="${ZPOA_PRIMARY_DB_HOST}:5432" ;; esac
  PRIMARY_DB_HOSTONLY="${ZPOA_PRIMARY_DB_HOST%%:*}"
  PRIMARY_DB_PORT="${ZPOA_PRIMARY_DB_HOST##*:}"
  _is_uint "$PRIMARY_DB_PORT" || die "invalid port in ZPOA_PRIMARY_DB_HOST='$ZPOA_PRIMARY_DB_HOST'"

  # Replication credentials (created on the primary by `zpoa ha prepare-primary`).
  if [ -z "$ZPOA_REPL_PASSWORD" ]; then
    on_tty || die "backup mode needs ZPOA_REPL_PASSWORD (from 'zpoa ha prepare-primary' on the primary)"
    prompt ZPOA_REPL_PASSWORD "Replication password (from 'zpoa ha prepare-primary' on the primary)" ""
    [ -n "$ZPOA_REPL_PASSWORD" ] || die "a replication password is required"
  fi
  ZPOA_REPL_USER="$(_strip_ws "${ZPOA_REPL_USER:-zpoa_replicator}")"

  # The primary's `zpoa` DB password — the replicated role carries it, so the
  # control container's DSN must use the SAME password to connect to the standby.
  if [ -z "$ZPOA_PG_PASSWORD" ]; then
    on_tty || die "backup mode needs ZPOA_PG_PASSWORD (the primary's 'zpoa' DB password, from prepare-primary)"
    prompt ZPOA_PG_PASSWORD "Primary's 'zpoa' database password (from prepare-primary)" ""
    [ -n "$ZPOA_PG_PASSWORD" ] || die "the primary's zpoa DB password is required"
  fi
  PG_PASSWORD="$ZPOA_PG_PASSWORD"

  # Vault key MUST match the primary — replicated rows (gateway/DERP/mesh-CA keys,
  # webhook secrets) are AES-256-GCM sealed with it. A mismatch = undecryptable.
  if [ -z "$ZPOA_VAULT_KEY" ]; then
    on_tty || die "backup mode needs ZPOA_VAULT_KEY (the PRIMARY's vault key — replicated secrets are sealed with it)"
    prompt ZPOA_VAULT_KEY "Primary's ZPOA_VAULT_KEY (must match exactly — sealed rows are replicated)" ""
    [ -n "$ZPOA_VAULT_KEY" ] || die "the primary's vault key is required (a mismatch makes replicated secrets undecryptable)"
  fi

  # Admin token: reuse the primary's so the admin API keeps working post-promote.
  # Prompt on a TTY (blank = generate); unattended with none set = generate + warn.
  if [ -z "$ZPOA_ADMIN_TOKEN" ] && on_tty; then
    prompt ZPOA_ADMIN_TOKEN "Primary's admin token — for a seamless promote (Enter = generate a fresh one)" ""
  fi
  if [ -z "$ZPOA_ADMIN_TOKEN" ]; then
    ZPOA_ADMIN_TOKEN="$(gen_secret)"
    warn "no ZPOA_ADMIN_TOKEN given — generated a fresh one for this box. It will NOT match the primary's; set ZPOA_ADMIN_TOKEN=<primary token> for a seamless promote (prepare-primary prints it)."
  fi

  # Auto-promote (Tier 0.5) is opt-in and 2-node only; the DB-addr fence is required.
  ZPOA_AUTO_PROMOTE="$(_lc_trim "$ZPOA_AUTO_PROMOTE")"
  case "$ZPOA_AUTO_PROMOTE" in true|false) ;; *) die "ZPOA_AUTO_PROMOTE must be true or false" ;; esac

  # Mesh/relay defaults. The standby's own DERP relay stays DORMANT while it is a
  # standby — it advertises the PRIMARY's relay (see DERP_RELAYS default below) so
  # all clients share ONE relay island; `zpoa ha promote` activates this node's
  # relay on takeover.
  is_valid_cidr "$ZPOA_MESH_CIDR" || die "invalid ZPOA_MESH_CIDR='$ZPOA_MESH_CIDR'"
  # DERP relay this node advertises to clients (in the netmap). A WARM STANDBY
  # (role=backup, i.e. ZPOA_PRIMARY_URL set) MUST advertise the PRIMARY's relay,
  # NOT its own hostname: this build has no DERP meshing, so a backup advertising
  # itself forms a SECOND, un-meshed relay island and cross-relay peer traffic is
  # silently black-holed (sender sees no error, retries forever). `zpoa ha promote`
  # flips this to our own domain on takeover. A primary/gateway (no PRIMARY_URL)
  # advertises itself, as before.
  if [ -z "$ZPOA_DERP_RELAYS" ]; then
    if [ -n "$ZPOA_PRIMARY_URL" ]; then
      __ph="${ZPOA_PRIMARY_URL#https://}"; __ph="${__ph#http://}"; __ph="${__ph%%/*}"; __ph="${__ph%%:*}"
      ZPOA_DERP_RELAYS="${__ph}:443:3478"
    else
      ZPOA_DERP_RELAYS="${ZPOA_DOMAIN}:443:3478"
    fi
  fi

  # A backup is a CP standby only — no co-located subnet router (routes come from
  # the replicated DB; run a dedicated gateway box for data-plane redundancy).
  WITH_GATEWAY=false
  ZPOA_ROUTES=""

  if [ -n "$ZPOA_IMAGE" ]; then BUILD_LOCAL=false; else BUILD_LOCAL=true; ZPOA_IMAGE="zpoa-access"; ZPOA_IMAGE_TAG="local"; fi
  case "$ZPOA_TLS" in
    auto)     CADDY_TLS_DIRECTIVE="" ;;
    internal) CADDY_TLS_DIRECTIVE="tls internal" ;;
    off)      CADDY_TLS_DIRECTIVE="" ;;
    *) die "ZPOA_TLS must be auto|internal|off" ;;
  esac
  # The tenant slug isn't authoritative on a standby (the replicated DB owns it),
  # but the day-2 wrapper wants a value; derive a harmless one from the hostname.
  if [ -z "$ZPOA_TENANT" ]; then
    ZPOA_TENANT="$(printf '%s' "$ZPOA_DOMAIN" | cut -d. -f1 | tr -cd 'a-z0-9-')"
    [ -z "$ZPOA_TENANT" ] && ZPOA_TENANT="default"
  fi
  return 0   # never end on a short-circuited && (set -e would abort the installer)
}

write_backup_env() {
  cat > "$ZPOA_DIR/.env" <<EOF
# Generated by install.sh (BACKUP control plane) on $(uname -n) — secrets; do not commit.
ZPOA_IMAGE=${ZPOA_IMAGE}
IMAGE_TAG=${ZPOA_IMAGE_TAG}
HOSTNAME=${ZPOA_DOMAIN}
TENANT_ID=${ZPOA_TENANT}
TENANT_NAME=${ZPOA_TENANT}
ADMIN_TOKEN=${ZPOA_ADMIN_TOKEN}
ADMIN_EMAIL=${ZPOA_ADMIN_EMAIL}
VIEWER_TOKEN=${ZPOA_VIEWER_TOKEN}
KEYCLOAK_ISSUER_URL=${ZPOA_KEYCLOAK_ISSUER_URL}
VAULT_KEY=${ZPOA_VAULT_KEY}
POSTGRES_PASSWORD=${PG_PASSWORD}
MESH_CIDR=${ZPOA_MESH_CIDR}
DERP_RELAYS=${ZPOA_DERP_RELAYS}
VENDOR_IDP_HOSTS=${ZPOA_VENDOR_IDP_HOSTS}
DNS_SEARCH_DOMAINS=${ZPOA_DNS_SEARCH_DOMAINS}
DNS_CORP_RESOLVERS=${ZPOA_DNS_CORP_RESOLVERS}
CADDY_HOSTNAME=${ZPOA_DOMAIN}
CADDY_TLS=${CADDY_TLS_DIRECTIVE}
# --- replication (pg_basebackup seed + streaming) ---
PRIMARY_DB_HOST=${PRIMARY_DB_HOSTONLY}
PRIMARY_DB_PORT=${PRIMARY_DB_PORT}
REPL_USER=${ZPOA_REPL_USER}
REPL_PASSWORD=${ZPOA_REPL_PASSWORD}
REPL_SLOT=${ZPOA_REPL_SLOT}
# --- failover watchdog (Tier 0 manual by default; Tier 0.5 = AUTO_PROMOTE true) ---
FAILOVER_PRIMARY_URL=${ZPOA_PRIMARY_URL}
FAILOVER_SELF_URL=https://${ZPOA_DOMAIN}
FAILOVER_AUTO_PROMOTE=${ZPOA_AUTO_PROMOTE}
FAILOVER_PRIMARY_DB_ADDR=${ZPOA_PRIMARY_DB_HOST}
EOF
  chmod 600 "$ZPOA_DIR/.env"
}

# The standby seeder: on first boot (empty datadir) it pg_basebackups the primary
# into the postgres volume with -R, which writes primary_conninfo + standby.signal
# so Postgres then boots as a streaming read-only replica. Idempotent — a seeded
# datadir is left untouched.
write_standby_init() {
  cat > "$ZPOA_DIR/standby-init.sh" <<'EOF'
#!/bin/sh
# pg_basebackup seeder for a Z-Access backup control plane. Runs as root in a
# throwaway postgres:16 container BEFORE the real postgres service starts.
set -eu
DATADIR=/var/lib/postgresql/data
HOST="${PRIMARY_DB_HOST:?PRIMARY_DB_HOST required}"
PORT="${PRIMARY_DB_PORT:-5432}"
USER="${REPL_USER:-zpoa_replicator}"
PW="${REPL_PASSWORD:?REPL_PASSWORD required}"
SLOT="${REPL_SLOT:-}"

if [ -s "$DATADIR/PG_VERSION" ]; then
  echo "[standby-init] datadir already seeded (PG_VERSION present) — skipping pg_basebackup"
  exit 0
fi

echo "[standby-init] waiting for primary Postgres ${HOST}:${PORT} …"
i=0
until PGPASSWORD="$PW" pg_isready -h "$HOST" -p "$PORT" -U "$USER" >/dev/null 2>&1; do
  i=$((i+1)); [ "$i" -gt 90 ] && { echo "[standby-init] primary DB unreachable after ~3m — aborting"; exit 1; }
  sleep 2
done

echo "[standby-init] seeding standby via pg_basebackup from ${HOST}:${PORT} (user ${USER})…"
# Empty the mountpoint (fresh volume), then stream a base backup. -R writes
# primary_conninfo (with the password we connect with) + standby.signal.
rm -rf "${DATADIR:?}/"* 2>/dev/null || true
slot_arg=""
[ -n "$SLOT" ] && slot_arg="-S $SLOT"
# shellcheck disable=SC2086
PGPASSWORD="$PW" pg_basebackup \
  -h "$HOST" -p "$PORT" -U "$USER" -D "$DATADIR" \
  -R -X stream -c fast -P $slot_arg \
  || { echo "[standby-init] pg_basebackup FAILED — check repl creds / pg_hba / wal_keep_size on the primary"; exit 1; }

chown -R postgres:postgres "$DATADIR" 2>/dev/null || true
chmod 700 "$DATADIR" 2>/dev/null || true
echo "[standby-init] standby seeded; primary_conninfo + standby.signal written. Postgres will boot as a streaming replica."
EOF
  chmod 0755 "$ZPOA_DIR/standby-init.sh"
}

write_backup_compose() {
  local image_line build_line
  image_line="    image: \${ZPOA_IMAGE:-zpoa-access}:\${IMAGE_TAG:-local}"
  if [ "$BUILD_LOCAL" = true ]; then
    build_line="    build:
      context: .
      dockerfile: Dockerfile"
  else
    build_line=""
  fi

  cat > "$ZPOA_DIR/docker-compose.yml" <<EOF
# Generated by install.sh — BACKUP control plane (warm standby of a primary).
# postgres here is a STREAMING REPLICA seeded from the primary by standby-init.sh;
# the control plane runs read-only until promoted (zpoa ha promote).
services:
  pg-standby-init:
    image: postgres:16-alpine
    container_name: zpoa-access-standby-init
    restart: "no"
    entrypoint: ["/bin/sh", "/standby-init.sh"]
    environment:
      PRIMARY_DB_HOST: \${PRIMARY_DB_HOST}
      PRIMARY_DB_PORT: \${PRIMARY_DB_PORT:-5432}
      REPL_USER: \${REPL_USER:-zpoa_replicator}
      REPL_PASSWORD: \${REPL_PASSWORD}
      REPL_SLOT: \${REPL_SLOT:-}
    volumes:
      - postgres-data:/var/lib/postgresql/data
      - ./standby-init.sh:/standby-init.sh:ro
    networks: [zpoa]

  postgres:
    image: postgres:16-alpine
    container_name: zpoa-access-postgres
    restart: unless-stopped
    depends_on:
      pg-standby-init:
        condition: service_completed_successfully
    environment:
      # Only used if the datadir were empty (it isn't — seeded by init). The real
      # roles/passwords come from the replicated primary DB.
      POSTGRES_USER: zpoa
      POSTGRES_PASSWORD: \${POSTGRES_PASSWORD}
      POSTGRES_DB: zpoa
    volumes:
      - postgres-data:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U zpoa -d zpoa"]
      interval: 5s
      timeout: 3s
      retries: 20
    networks: [zpoa]

  control:
$( [ "$BUILD_LOCAL" = true ] && echo "$build_line" )
$image_line
    container_name: zpoa-access-control
    restart: unless-stopped
    depends_on:
      postgres:
        condition: service_healthy
    environment:
      CYBER_MODE: "server"
      CYBER_SERVER_HOST: "0.0.0.0"
      CYBER_SERVER_PORT: "8090"
      CYBER_HOSTNAME: \${HOSTNAME}
      CYBER_TENANT_MODE: "single"
      CYBER_TENANT_ID: \${TENANT_ID:-default}
      CYBER_DATABASE_URL: "postgres://zpoa:\${POSTGRES_PASSWORD}@postgres:5432/zpoa?sslmode=disable"
      CYBER_VAULT_KEY: \${VAULT_KEY:-}
      CYBER_ADMIN_TOKEN: \${ADMIN_TOKEN:-}
      CYBER_ADMIN_EMAIL: \${ADMIN_EMAIL:-}
      CYBER_VIEWER_TOKEN: \${VIEWER_TOKEN:-}
      CYBER_KEYCLOAK_ISSUER_URL: \${KEYCLOAK_ISSUER_URL:-}
      CYBER_VENDOR_IDP_HOSTS: "\${VENDOR_IDP_HOSTS:-}"
      # Corp DNS pushed to clients as split-DNS (resolve internal names over the
      # VPN). Both empty = no push. OpenVPN push "dhcp-option DNS/DOMAIN" analog.
      CYBER_DNS_SEARCH_DOMAINS: "\${DNS_SEARCH_DOMAINS:-}"
      CYBER_DNS_CORP_RESOLVERS: "\${DNS_CORP_RESOLVERS:-}"
      CYBER_VPN_NETWORK_CIDR: \${MESH_CIDR:-100.64.0.0/16}
      CYBER_DERP_RELAY_ENABLED: "true"
      CYBER_DERP_RELAYS: \${DERP_RELAYS:-}
      CYBER_DISCOVERY_CONTROL_PLANE_URL: "https://\${HOSTNAME}"
      CYBER_DISCOVERY_TENANT_ID: \${TENANT_ID:-default}
      CYBER_DISCOVERY_TENANT_NAME: \${TENANT_NAME:-default}
      CYBER_DISCOVERY_DERP_RELAYS: \${DERP_RELAYS:-}
      # --- failover watchdog: this node is a BACKUP ---
      CYBER_FAILOVER_PRIMARY_URL: \${FAILOVER_PRIMARY_URL}
      CYBER_FAILOVER_SELF_URL: \${FAILOVER_SELF_URL}
      CYBER_FAILOVER_AUTO_PROMOTE: \${FAILOVER_AUTO_PROMOTE:-false}
      CYBER_FAILOVER_PRIMARY_DB_ADDR: \${FAILOVER_PRIMARY_DB_ADDR:-}
    ports:
      - "3478:3478/udp"
    volumes:
      - control-data:/var/lib/zpoa
    healthcheck:
      test: ["CMD", "wget", "-qO-", "http://localhost:8090/healthz"]
      interval: 10s
      timeout: 3s
      retries: 6
      start_period: 20s
    networks: [zpoa]

  caddy:
    image: caddy:2-alpine
    container_name: zpoa-access-caddy
    restart: unless-stopped
    depends_on: [control]
    ports:
      - "80:80"
      - "443:443"
    environment:
      CADDY_HOSTNAME: \${CADDY_HOSTNAME}
      CADDY_TLS: \${CADDY_TLS:-}
    volumes:
      - ./Caddyfile:/etc/caddy/Caddyfile:ro
      - caddy-data:/data
      - caddy-config:/config
    networks: [zpoa]

volumes:
  control-data:
  postgres-data:
  caddy-data:
  caddy-config:

networks:
  zpoa:
    driver: bridge
EOF
}

bring_up_backup() {
  cd "$ZPOA_DIR"
  if [ "$BUILD_LOCAL" = true ]; then log "Building local image from the prebuilt binary..."; docker build -q -t zpoa-access:local "$ZPOA_DIR" >/dev/null || die "image build failed"; ok "image zpoa-access:local built"; fi
  log "Seeding the standby (pg_basebackup) + starting the backup control plane..."
  docker compose -f docker-compose.yml up -d || die "docker compose up failed — check: (cd $ZPOA_DIR && docker compose logs pg-standby-init)"
  log "Waiting for the standby control plane to become healthy..."
  local i
  for i in $(seq 1 60); do
    if docker compose -f docker-compose.yml exec -T control wget -qO- http://localhost:8090/healthz >/dev/null 2>&1; then ok "backup control plane healthy (streaming standby)"; return; fi
    sleep 2
  done
  warn "backup not healthy yet — the pg_basebackup seed can take a while on a large DB. Check: (cd $ZPOA_DIR && docker compose logs -f pg-standby-init postgres control)"
}

backup_summary() {
  local tier promote
  if [ "$ZPOA_AUTO_PROMOTE" = true ]; then tier="Tier 0.5 — fenced AUTO-failover (2-node)"; else tier="Tier 0 — manual one-click promote"; fi
  promote="zpoa ha promote"
  cat <<EOF

${c_grn}${c_bold}Backup control plane is up (warm standby).${c_reset}

  This backup URL   : ${c_bold}https://${ZPOA_DOMAIN}${c_reset}
  Following primary : ${c_bold}${ZPOA_PRIMARY_URL}${c_reset}
  Replicating DB    : ${ZPOA_PRIMARY_DB_HOST}  (user ${ZPOA_REPL_USER})
  Failover mode     : ${c_bold}${tier}${c_reset}
  Install dir       : ${ZPOA_DIR}

  Role right now    : ${c_bold}read-only standby${c_reset} — it serves netmaps to clients that fail
                      over, but new enrolment + admin writes go to the primary until promoted.

${c_bold}Two things to finish on the PRIMARY${c_reset} (so clients know to fail over here):
  1. Register this backup so clients learn the failover URL:
       ${c_bold}zpoa ha add-backup https://${ZPOA_DOMAIN}${c_reset}
  2. Confirm replication is streaming:
       ${c_bold}zpoa ha status${c_reset}   (on either node)

${c_bold}When the primary dies — take over from this box:${c_reset}
  • ${c_bold}${promote}${c_reset}   → pg_promote + become the preferred control plane$( [ "$ZPOA_AUTO_PROMOTE" = true ] && printf '\n  • (auto-promote is ON — it fires by itself once the primary DB is also confirmed dead)' )

  Health: ${c_bold}zpoa ha status${c_reset}   Logs: ${c_bold}zpoa logs${c_reset}
  DNS: point ${ZPOA_DOMAIN} at THIS host's public IP.
EOF
}

# ---------------------------------------------------------------------------
main() {
  # allow: install.sh --role gateway / --dir X / --yes
  while [ $# -gt 0 ]; do
    case "$1" in
      --role) ZPOA_ROLE="$2"; shift 2 ;;
      --dir)  ZPOA_DIR="$2"; shift 2 ;;
      --yes|-y) ZPOA_YES=1; shift ;;
      --no-firewall) ZPOA_OPEN_FIREWALL=false; shift ;;
      *) die "unknown argument: $1" ;;
    esac
  done

  need_root
  command -v curl >/dev/null 2>&1 || die "curl is required"
  command -v openssl >/dev/null 2>&1 || warn "openssl not found — falling back to /dev/urandom for secrets"
  detect_arch

  select_mode
  local __mode
  case "$ZPOA_ROLE" in gateway) __mode='gateway only' ;; backup) __mode='backup control plane' ;; *) __mode='full self-host' ;; esac
  printf '%s%sZ-Access installer%s (%s, %s)\n' "$c_bold" "$c_cyn" "$c_reset" "$__mode" "$ARCH"

  if [ "$ZPOA_ROLE" = "gateway" ]; then
    gather_gateway_config
    run_remote_gateway
    open_gateway_firewall
    gateway_summary
    exit 0
  fi

  if [ "$ZPOA_ROLE" = "backup" ]; then
    gather_backup_config
    ensure_docker
    mkdir -p "$ZPOA_DIR"
    [ "$BUILD_LOCAL" = true ] && { write_dockerfile; fetch_binary; }
    write_backup_env
    write_caddyfile
    write_standby_init
    write_backup_compose
    write_install_conf
    write_zpoa_cli
    bring_up_backup
    open_firewall
    backup_summary
    exit 0
  fi

  gather_config
  ensure_docker
  mkdir -p "$ZPOA_DIR"
  [ "$BUILD_LOCAL" = true ] && { write_dockerfile; fetch_binary; }
  write_env
  write_caddyfile
  write_compose
  write_install_conf
  write_zpoa_cli
  bring_up
  bootstrap_tenant
  open_firewall
  summary
}

# Allow sourcing for tests without executing (ZPOA_SOURCE_ONLY=1 source install.sh).
[ "${ZPOA_SOURCE_ONLY:-}" = "1" ] || main "$@"
