diff --git a/RELEASES.md b/RELEASES.md index 1247a68f2..2b95fd30b 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -161,6 +161,29 @@ and `:mainnet-release`. Promote with the retag workflows: Use `source-tag = 3.11.0` and `release-tag = testnet-release` or `mainnet-release`. +## Ops tooling + +[`scripts/ops/menu.sh`](./scripts/ops/menu.sh) is the entry point for the +scripted parts of a release. It offers two things: + +1. **release github code** — runs `prepare-github-release.sh` (step 1 above). +2. **migrate devnet cluster** — rolls a published release out to a NEAR One dev + cluster via [`scripts/ops/dev-cluster/dev-menu.sh`](./scripts/ops/dev-cluster/dev-menu.sh). + +Given a network and release version, the dev-cluster flow retags every +`mpc-node-*` Nomad job (plan, confirm, run), checks the nodes report the new +`release=`, and offers a test signature. Each job keeps its own image +repository; the MPC task is found by the `MPC_ACCOUNT_SK` it carries, never by +its image name. + +Every command is printed before it runs and every write is behind a +confirmation prompt. Addresses and credentials stay out of this repo: the Nomad +URL and credentials are typed in per run, node addresses come from Nomad, and +the member keys are read from the job definitions into the near-cli keystore — +never reaching the command line or the echoed output. The per-network +`NOMAD_ADDR_DEV_*`, `NOMAD_HTTP_AUTH_DEV_*`, and `MPC_NODE_ADDRS_DEV_*` +variables skip the matching step. + ## Re-running after a failure The workflow refuses to start if a release for the version already diff --git a/scripts/ops/common.sh b/scripts/ops/common.sh new file mode 100644 index 000000000..2d64802ee --- /dev/null +++ b/scripts/ops/common.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# +# common.sh — generic helpers shared by the ops scripts (source, don't run). +# Dev-cluster-specific helpers live in dev-cluster/dev-common.sh. +# + +# Only when both streams are terminals, so redirected output stays clean. +# NO_COLOR is honoured (https://no-color.org). +if [[ -t 1 && -t 2 && -z "${NO_COLOR:-}" ]]; then + C_RESET=$'\033[0m' C_CMD=$'\033[36m' C_OUT=$'\033[2m' + C_STEP=$'\033[1;34m' C_OK=$'\033[32m' C_WARN=$'\033[33m' C_ERR=$'\033[1;31m' +else + C_RESET="" C_CMD="" C_OUT="" C_STEP="" C_OK="" C_WARN="" C_ERR="" +fi + +die() { + printf '%sError: %s%s\n' "$C_ERR" "$1" "$C_RESET" >&2 + exit 1 +} + +step() { printf '\n%s%s%s\n' "$C_STEP" "$*" "$C_RESET"; } +ok() { printf '%s%s%s\n' "$C_OK" "$*" "$C_RESET"; } +warn() { printf '%s%s%s\n' "$C_WARN" "$*" "$C_RESET" >&2; } + +require_cmds() { + local missing=0 + for cmd in "$@"; do + command -v "$cmd" >/dev/null 2>&1 || { + printf 'Missing dependency: %s\n' "$cmd" >&2 + missing=1 + } + done + [[ "$missing" -eq 0 ]] || die "Please install the missing dependencies above (hint: run from within 'nix develop')." +} + +# Mirrors .github/workflows/release.yml, so release candidates work too. +check_version() { + [[ "$1" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.]+)?$ ]] \ + || die "'$1' is not valid semver (expected MAJOR.MINOR.PATCH[-SUFFIX])." +} + +# sha256sum is GNU-only; macOS ships shasum. +sha256_of() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | cut -d' ' -f1 + elif command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$1" | cut -d' ' -f1 + else + die "Need sha256sum or shasum to hash ${1}." + fi +} + +confirm() { + local reply + read -rp "$1 [y/N] " reply + [[ "$reply" == y || "$reply" == Y ]] +} + +# Keeps the printed line copy-pasteable: quote anything not shell-safe. +fmt_cmd() { + local out="" arg + for arg in "$@"; do + case "$arg" in + ''|*[!A-Za-z0-9_/.:=@%+,-]*) out+=" '${arg//\'/\'\\\'\'}'" ;; + *) out+=" $arg" ;; + esac + done + printf '%s' "${out# }" +} + +# To stderr, so it stays visible when the caller captures stdout. +show_cmd() { + printf '\n%s $ %s%s\n' "$C_CMD" "$(fmt_cmd "$@")" "$C_RESET" >&2 +} + +# Echoes a captured response, truncated — job definitions run to several KB. +show_output() { + local text=$1 limit=${2:-1500} + if (( ${#text} > limit )); then + printf '%s%s\n … (%d more characters)%s\n' \ + "$C_OUT" "${text:0:limit}" "$(( ${#text} - limit ))" "$C_RESET" >&2 + else + printf '%s%s%s\n' "$C_OUT" "$text" "$C_RESET" >&2 + fi +} + +run_cmd() { + show_cmd "$@" + "$@" +} + +# Subshell, so a die() inside ends the step rather than the menu around it. +run_step() { + ( "$@" ) +} diff --git a/scripts/ops/dev-cluster/dev-common.sh b/scripts/ops/dev-cluster/dev-common.sh new file mode 100644 index 000000000..eade61426 --- /dev/null +++ b/scripts/ops/dev-cluster/dev-common.sh @@ -0,0 +1,224 @@ +#!/usr/bin/env bash +# +# dev-common.sh — helpers specific to the NEAR One dev clusters, including the +# Nomad HTTP client (source, don't run). Generic helpers live in ../common.sh. +# +# MPC_SIGN_WITH overrides the signer (default sign-with-legacy-keychain, which +# reads the ~/.near-credentials files first-time-setup.sh writes). +# + +SIGN_WITH="${MPC_SIGN_WITH:-sign-with-legacy-keychain}" +# The node's web server (routes /metrics and /public_data) — static port in +# every mpc-node-* job spec, so it never needs discovering, only the host. +NODE_HTTP_PORT="${MPC_NODE_HTTP_PORT:-8080}" + +# curl -K parses values as quoted strings with backslash escapes. +curl_cfg_escape() { + local s=${1//\\/\\\\} + printf '%s' "${s//\"/\\\"}" +} + +# Echoes the request, and the response for mutations. Never the credentials. +nomad_curl() { + local method=$1 path=$2 data=${3:-} + local url="${NOMAD_ADDR%/}/v1${path}" + # The API ignores the NOMAD_NAMESPACE env var (a nomad-CLI feature). + if [[ -n "${NOMAD_NAMESPACE:-}" ]]; then + local sep="?"; [[ "$url" != *\?* ]] || sep="&" + url+="${sep}namespace=${NOMAD_NAMESPACE}" + fi + # --fail-with-body (curl >= 7.76): plain -f discards Nomad's error body. + local args=(-sS --fail-with-body --max-time 30 -X "$method" "$url") + [[ -z "$data" ]] || args+=(-H 'Content-Type: application/json' --data-binary @-) + + show_cmd curl -X "$method" "$url" ${data:+--data-binary @-} + + # Secrets not on argv: credentials via config stream, body via stdin. + local config="" + [[ -z "${NOMAD_HTTP_AUTH:-}" ]] || config+="user = \"$(curl_cfg_escape "$NOMAD_HTTP_AUTH")\""$'\n' + [[ -z "${NOMAD_TOKEN:-}" ]] || config+="header = \"X-Nomad-Token: $(curl_cfg_escape "$NOMAD_TOKEN")\""$'\n' + + local response status=0 + response=$(printf '%s' "$data" | curl -K <(printf '%s' "$config") "${args[@]}") || status=$? + if (( status != 0 )); then + [[ -z "$response" ]] || show_output "$response" + return "$status" + fi + + # GET bodies are whole job definitions — too noisy to echo. + [[ "$method" == GET ]] || show_output "$response" + printf '%s' "$response" +} + +# IDs of every mpc-node-* job on the cluster. +discover_job_ids() { + local ids + ids=$(nomad_curl GET "/jobs?prefix=mpc-node" | jq -r '.[].ID') \ + || die "Could not list jobs from ${NOMAD_ADDR}." + [[ -n "$ids" ]] || die "No mpc-node-* jobs found at ${NOMAD_ADDR}." + printf '%s' "$ids" +} + +# Sets CONTRACT, NEAR_NET, MEMBER_ACCOUNTS, SIGN_DEPOSIT and re-points endpoint +# vars from per-cluster exports; network choice drives every step. +resolve_dev_cluster() { + local suffix="${1^^}" var + case "$1" in + testnet) + CONTRACT="mpc-dev-contract.testnet" NEAR_NET="testnet" SIGN_DEPOSIT="1 NEAR" + MEMBER_ACCOUNTS="mpc-node-0-mpc-dev.testnet mpc-node-1-mpc-dev.testnet" ;; + mainnet) + CONTRACT="dev-contract.near" NEAR_NET="mainnet" SIGN_DEPOSIT="0.1 NEAR" + MEMBER_ACCOUNTS="mpc-0-dev-mainnet.dev-signer.near mpc-1-dev-mainnet.dev-signer.near" ;; + *) die "Unknown dev cluster '$1' (expected testnet|mainnet)." ;; + esac + var="NOMAD_ADDR_DEV_${suffix}"; [[ -z "${!var:-}" ]] || export NOMAD_ADDR="${!var}" + var="MPC_NODE_ADDRS_DEV_${suffix}"; [[ -z "${!var:-}" ]] || export MPC_NODE_ADDRS="${!var}" + # +set: an intentionally empty value still disables the prompt. + var="NOMAD_HTTP_AUTH_DEV_${suffix}"; [[ -z "${!var+set}" ]] || export NOMAD_HTTP_AUTH="${!var}" +} + +# Prompts per run; NOMAD_ADDR_DEV_ export skips it. Takes bare IP only. +prompt_nomad_ip() { + local label=${1:-target} input scheme + while [[ -z "${NOMAD_ADDR:-}" ]]; do + read -rp "Nomad URL for the ${label} dev cluster: " input + # Tolerate pasted URL; preserve HTTPS, never downgrade to HTTP. + scheme="http"; [[ "$input" != https://* ]] || scheme="https" + input="${input#http://}"; input="${input#https://}"; input="${input%%/*}" + if [[ ! "$input" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}(:[0-9]+)?$ ]]; then + echo " Expected an IPv4 address or URL (e.g. 10.0.0.1, 10.0.0.1:4646, http://10.0.0.1:4646)." + continue + fi + NOMAD_ADDR="${scheme}://${input}" + done + export NOMAD_ADDR +} + +prompt_http_auth() { + local input + while true; do + read -rp "Nomad credentials for ${NOMAD_ADDR} (user:password): " input + [[ "$input" != *:* ]] || break + echo " Expected user:password." + done + export NOMAD_HTTP_AUTH="$input" + # Base64 on the wire is still cleartext over plain HTTP. + [[ "${NOMAD_ADDR:-}" == https://* ]] \ + || warn "Note: these credentials will be sent over plain HTTP (${NOMAD_ADDR:-})." +} + +# host:port of a job's running allocation, for MPC_NODE_ADDRS. Prefers the +# platform's external IP — the client's own address only resolves inside the VPC. +discover_node_addr() { + local job_id=$1 alloc_id node_id ip + alloc_id=$(nomad_curl GET "/job/${job_id}/allocations" \ + | jq -r '[.[] | select(.ClientStatus == "running")] | sort_by(.CreateIndex) | last | .ID // empty') \ + || return 1 + [[ -n "$alloc_id" ]] || return 1 + node_id=$(nomad_curl GET "/allocation/${alloc_id}" | jq -r '.NodeID // empty') || return 1 + [[ -n "$node_id" ]] || return 1 + ip=$(nomad_curl GET "/node/${node_id}" | jq -r ' + ([(.Attributes // {}) | to_entries[] + | select(.key | test("external-ip")) | .value] | first) + // (.Attributes // {})["unique.network.ip-address"] + // (.HTTPAddr // "" | split(":")[0]) + // empty') || return 1 + [[ -n "$ip" ]] || return 1 + printf '%s:%s' "$ip" "$NODE_HTTP_PORT" +} + +# Tries Nomad auto-discovery first; only prompts if that finds nothing. +# MPC_NODE_ADDRS export (or MPC_NODE_ADDRS_DEV_) skips both. +prompt_node_addrs() { + local input job_ids job_id addr addrs=() + [[ -z "${MPC_NODE_ADDRS+set}" ]] || return 0 + + if job_ids=$(discover_job_ids); then + for job_id in $job_ids; do + addr=$(discover_node_addr "$job_id") && addrs+=("$addr") + done + fi + + if [[ ${#addrs[@]} -gt 0 ]]; then + export MPC_NODE_ADDRS="${addrs[*]}" + ok "Discovered node metrics addresses: ${MPC_NODE_ADDRS}" + return 0 + fi + + warn "Could not auto-discover node addresses from Nomad." + read -rp "Node metrics addresses, space-separated (blank to skip verification): " input + export MPC_NODE_ADDRS="$input" +} + +# Whether a credential is configured — never the credential itself. +nomad_auth_state() { + if [[ -n "${NOMAD_HTTP_AUTH:-}" ]]; then echo "(set)"; else echo "(none)"; fi +} + +# Probes the legacy ~/.near-credentials layout; first-time-setup.sh writes here. +have_signing_key() { + [[ -f "${HOME}/.near-credentials/${NEAR_NET}/${1}.json" ]] +} + +# Retries per node (warm-up delay after allocation starts). +verify_nodes() { + local version=$1 + require_cmds curl + [[ -n "${MPC_NODE_ADDRS:-}" ]] || die "MPC_NODE_ADDRS is not set (e.g. \"host:8080 host:8080\")." + + local addr info try fetch + local -a upgraded=() stale=() unreachable=() + for addr in ${MPC_NODE_ADDRS}; do + # Internal-only plain HTTP; no TLS endpoint exists. + # nosemgrep: trailofbits.generic.curl-unencrypted-url.curl-unencrypted-url + fetch=(curl -sf --max-time 5 "http://${addr}/metrics") + show_cmd "${fetch[@]}" + info="" + for try in 1 2 3; do + info=$("${fetch[@]}" | grep -o 'mpc_node_build_info{[^}]*}') || info="" + [[ "$info" != *"release=\"${version}\""* ]] || break + if (( try < 3 )); then sleep 5; fi + done + if [[ -z "$info" ]]; then echo " (unreachable)"; unreachable+=("$addr"); continue; fi + echo " $info" + if [[ "$info" == *"release=\"${version}\""* ]]; then + upgraded+=("$addr") + else + stale+=("$addr") + fi + done + + # Unreachable is a visibility problem, not an upgrade failure — the dev + # nodes sit on internal IPs, so report it separately from a version miss. + (( ! ${#stale[@]} )) || warn "Not yet on ${version}: ${stale[*]}" + (( ! ${#unreachable[@]} )) || \ + warn "Could not reach (upgrade state unknown): ${unreachable[*]}" + if (( ${#upgraded[@]} && ! ${#stale[@]} && ! ${#unreachable[@]} )); then + ok "All nodes report release=\"${version}\"." + elif (( ${#upgraded[@]} )); then + ok "${#upgraded[@]} node(s) report release=\"${version}\"." + fi +} + +# Test signature request against the cluster contract (on-chain txn). +test_sign() { + resolve_dev_cluster "$1" + require_cmds near + + local signer=${MEMBER_ACCOUNTS%% *} + local payload='[12,1,2,0,4,5,6,8,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,44]' + local cmd=(near contract call-function as-transaction "$CONTRACT" sign + json-args "{\"request\": {\"payload\": ${payload}, \"path\": \"test\", \"key_version\": 0}}" + prepaid-gas '300.0 Tgas' attached-deposit "$SIGN_DEPOSIT" + sign-as "$signer" network-config "$NEAR_NET" "$SIGN_WITH" send) + + echo "Test sign on ${CONTRACT} as ${signer} (deposit ${SIGN_DEPOSIT})." + show_cmd "${cmd[@]}" + confirm "Send it?" || return 0 + if "${cmd[@]}"; then + ok "Signature returned — the cluster is signing." + else + warn "Test sign failed — investigate before proceeding." + fi +} diff --git a/scripts/ops/dev-cluster/dev-menu.sh b/scripts/ops/dev-cluster/dev-menu.sh new file mode 100755 index 000000000..599e6f69a --- /dev/null +++ b/scripts/ops/dev-cluster/dev-menu.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# +# dev-menu.sh — entry point for dev-cluster work. Picks the network and +# version, then upgrades the cluster nodes and verifies them. +# +# Usage: ./scripts/ops/dev-cluster/dev-menu.sh [testnet|mainnet] [VERSION] +# Prompts for the Nomad URL and credentials; node metrics addresses are +# discovered from Nomad. Exporting NOMAD_ADDR_DEV_*, NOMAD_HTTP_AUTH_DEV_*, or +# MPC_NODE_ADDRS_DEV_* for the network skips the matching step. +# + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=../common.sh +source "${SCRIPT_DIR}/../common.sh" +# shellcheck source=dev-common.sh +source "${SCRIPT_DIR}/dev-common.sh" + +# Sets NETWORK: a valid CLI argument passes, an invalid one dies (so scripted +# use fails loudly), no argument prompts until valid. +resolve_network() { + NETWORK="${1:-}" + [[ -z "$NETWORK" ]] || case "$NETWORK" in + testnet|mainnet) return 0 ;; + *) die "Unknown network '${NETWORK}' (expected testnet|mainnet)." ;; + esac + while true; do + read -rp "Network (testnet|mainnet) [testnet]: " NETWORK + NETWORK="${NETWORK:-testnet}" + case "$NETWORK" in + testnet|mainnet) return 0 ;; + *) echo "Unknown network '${NETWORK}'." ;; + esac + done +} + +# Sets VERSION from the CLI argument or a prompt, then validates it. +resolve_version() { + VERSION="${1:-}" + [[ -n "$VERSION" ]] || read -rp "Release version (e.g. 3.14.0): " VERSION + check_version "$VERSION" +} + +resolve_network "${1:-}" +resolve_version "${2:-}" +resolve_dev_cluster "$NETWORK" + +prompt_nomad_ip "$NETWORK" +[[ -n "${NOMAD_HTTP_AUTH+set}" ]] || prompt_http_auth +prompt_node_addrs + +cat < storing key for ${account}" + MPC_IMPORT_SK="$sk" MPC_IMPORT_ACCOUNT="$account" \ + MPC_IMPORT_DIR="${HOME}/.near-credentials/${NEAR_NET}" \ + python3 - <<'PY' || { warn "Could not store key for ${account}."; return; } +import json, os, sys + +_B58 = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz" + +def b58decode(s): + n = 0 + for c in s: + n = n * 58 + _B58.index(c) + body = n.to_bytes((n.bit_length() + 7) // 8, "big") + return b"\x00" * (len(s) - len(s.lstrip("1"))) + body + +def b58encode(b): + n = int.from_bytes(b, "big") + out = "" + while n: + n, r = divmod(n, 58) + out = _B58[r] + out + return "1" * (len(b) - len(b.lstrip(b"\x00"))) + out + +sk = os.environ["MPC_IMPORT_SK"] +account = os.environ["MPC_IMPORT_ACCOUNT"] +base = os.environ["MPC_IMPORT_DIR"] + +prefix, _, body = sk.partition(":") +raw = b58decode(body) +if len(raw) != 64: + sys.stderr.write("unexpected ed25519 key length: %d bytes\n" % len(raw)) + sys.exit(1) +pub_b58 = b58encode(raw[32:]) +record = {"public_key": "%s:%s" % (prefix, pub_b58), "private_key": sk} + +def write(path): + os.makedirs(os.path.dirname(path), exist_ok=True) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w") as f: + json.dump(record, f) + +write(os.path.join(base, "%s.json" % account)) +write(os.path.join(base, account, "%s_%s.json" % (prefix, pub_b58))) +PY + ok "${account}: key stored in ~/.near-credentials/${NEAR_NET}." +} + +# Imports any missing member-account keys found in a job's definition. +ensure_job_keys() { + local job_id=$1 job creds account sk found=0 + job=$(nomad_curl GET "/job/${job_id}") || { warn "Could not fetch ${job_id}."; return; } + # Capture rather than pipe into the loop: a jq failure there is invisible. + creds=$(job_signing_creds <<<"$job") \ + || { warn "${job_id}: could not read task env (unexpected job JSON)."; return; } + + while read -r account sk; do + [[ -n "$sk" ]] || continue + found=1 + if [[ -z "$account" ]]; then + warn "${job_id}: a task has MPC_ACCOUNT_SK but no MPC_ACCOUNT_ID — skipping." + elif have_signing_key "$account"; then + ok "${account}: key already in ~/.near-credentials." + else + import_signing_key "$account" "$sk" + fi + done <<<"$creds" + + if (( ! found )); then + warn "${job_id}: no MPC_ACCOUNT_SK in the job definition. Tasks (env var names only):" + job_env_summary <<<"$job" >&2 || true + fi +} diff --git a/scripts/ops/dev-cluster/migrate-dev-nodes.sh b/scripts/ops/dev-cluster/migrate-dev-nodes.sh new file mode 100755 index 000000000..d6fdcecaf --- /dev/null +++ b/scripts/ops/dev-cluster/migrate-dev-nodes.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +# +# migrate-dev-nodes.sh — Step 1 of a dev-cluster upgrade: retag every +# mpc-node-* Nomad job to — plan, confirm, run. Each job keeps its +# own image repository. Verification is the caller's job (dev-menu.sh does it). +# +# Usage: ./scripts/ops/dev-cluster/migrate-dev-nodes.sh +# The Nomad URL and its basic-auth credentials are prompted for. Exporting +# NOMAD_ADDR / NOMAD_HTTP_AUTH="user:password" skips the matching prompt; +# NOMAD_TOKEN adds an ACL token header; NOMAD_NAMESPACE targets that namespace. +# Member-account keys found in the job definitions are imported into the local +# near-cli keystore if missing, so the later signing steps can run. +# + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=../common.sh +source "${SCRIPT_DIR}/../common.sh" +# shellcheck source=dev-common.sh +source "${SCRIPT_DIR}/dev-common.sh" +# shellcheck source=first-time-setup.sh +source "${SCRIPT_DIR}/first-time-setup.sh" + +wait_for_alloc() { + local job_id=$1 tries=36 job_version status + job_version=$(nomad_curl GET "/job/${job_id}" | jq -r '.Version') \ + || die "Could not read the new job version for ${job_id}." + printf ' waiting for allocation (job version %s) ' "$job_version" + while (( tries-- > 0 )); do + # A blip must not abandon the jobs queued behind this one. + status=$(nomad_curl GET "/job/${job_id}/allocations" \ + | jq -r --argjson v "$job_version" \ + '[.[] | select(.JobVersion == $v)] | sort_by(.CreateIndex) | last | .ClientStatus // "pending"') \ + || status="unreachable" + if [[ "$status" == "running" ]]; then + ok "running." + return + fi + printf '.' + sleep 5 + done + echo + warn " Allocation not running after 3m (last status: ${status}) — check the Nomad UI." + # Continuing with this node down could drop the cluster below threshold. + confirm " Continue to the next job anyway?" \ + || die "Stopping the rollout — ${job_id} never reported running." +} + +# The MPC node task is the one holding MPC_ACCOUNT_SK — the image repository +# varies by cluster (mpc-node vs mpc-node-gcp), so never match on its name. +JQ_MPC_TASKS='.TaskGroups[]?.Tasks[]? + | select((((.Env // .Config.env // {}).MPC_ACCOUNT_SK) // "") != "")' +# Drop a trailing :tag, leaving any registry:port/path prefix intact. +JQ_REPO_OF='def repo_of: if test(":[^:/]+$") then sub(":[^:/]+$"; "") else . end;' + +# The MPC image(s) a job runs; empty for an unrelated job. +image_in_job() { + jq -r "[${JQ_MPC_TASKS} | .Config.image // empty] | unique | join(\", \")" +} + +# Same repository the task already runs, retagged to . +target_image_for_job() { + jq -r "${JQ_REPO_OF} [${JQ_MPC_TASKS} | .Config.image // empty | repo_of] + | unique | .[0] // empty" \ + | sed "s|\$|:$1|" +} + +# Retargets only the MPC task; sidecars and every other field pass through. +job_with_image() { + jq --arg img "$1" "(${JQ_MPC_TASKS} | .Config.image) = \$img" +} + +# The runbook's Definition -> Edit -> Plan -> Run for one job, over the API. +# Registering is the only write; Nomad then restarts the task on the new image. +upgrade_nomad_job() { + local job_id=$1 version=$2 + local job current image updated + job=$(nomad_curl GET "/job/${job_id}") || die "Could not fetch job ${job_id}." + current=$(image_in_job <<<"$job") + + # The mpc-node prefix search also matches jobs that run no MPC task. + if [[ -z "$current" ]]; then + step "==> ${job_id}: no MPC node task found, skipping." + return + fi + image=$(target_image_for_job "$version" <<<"$job") + [[ -n "$image" ]] || die "Could not derive the target image for ${job_id}." + # Makes a re-run after a partial rollout a no-op. + if [[ "$current" == "$image" ]]; then + step "==> ${job_id}: already on ${image}, skipping." + return + fi + step "==> ${job_id}: ${current} -> ${image}" + check_image_exists "$image" + + updated=$(job_with_image "$image" <<<"$job") + + # FailedTGAllocs means Nomad can't place the new allocation. + local plan failed warnings + plan=$(nomad_curl POST "/job/${job_id}/plan" \ + "$(jq -n --argjson job "$updated" '{Job: $job, Diff: true}')") \ + || die "Plan failed for ${job_id}." + failed=$(jq -r '.FailedTGAllocs // {} | keys | join(", ")' <<<"$plan") + warnings=$(jq -r '.Warnings // empty' <<<"$plan") + [[ -z "$failed" ]] || die "Plan reports failed allocations for: ${failed}" + [[ -z "$warnings" ]] || warn " plan warnings: ${warnings}" + + confirm " Apply to ${job_id} on ${NOMAD_ADDR}?" || { echo " skipped."; return; } + nomad_curl POST "/job/${job_id}" "$(jq -n --argjson job "$updated" '{Job: $job}')" >/dev/null \ + || die "Job registration failed for ${job_id}." + echo + # The next job waits on this node coming back, unless the operator overrides. + wait_for_alloc "$job_id" +} + +# An unpublished tag would leave the swapped jobs unable to start. +check_image_exists() { + local image=$1 + if ! command -v skopeo >/dev/null 2>&1; then + warn "WARNING: skopeo not found, skipping existence check for ${image}." + return 0 + fi + skopeo inspect --no-creds --format '{{.Digest}}' "docker://${image}" >/dev/null \ + || die "${image} not found on Docker Hub — is the release published?" +} + +[[ $# -eq 2 ]] || die "Usage: $0 (e.g. testnet 3.14.0)" +NETWORK=$1 +VERSION=$2 +check_version "$VERSION" +require_cmds curl jq near +resolve_dev_cluster "$NETWORK" +prompt_nomad_ip +[[ -n "${NOMAD_HTTP_AUTH+set}" ]] || prompt_http_auth + +JOB_IDS=$(discover_job_ids) + +step "==> Local signing keys" +for job_id in $JOB_IDS; do + ensure_job_keys "$job_id" +done + +for job_id in $JOB_IDS; do + upgrade_nomad_job "$job_id" "$VERSION" +done diff --git a/scripts/ops/menu.sh b/scripts/ops/menu.sh new file mode 100755 index 000000000..85b983d52 --- /dev/null +++ b/scripts/ops/menu.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# +# menu.sh — entry point for the MPC release/ops tooling. +# Usage: ./scripts/ops/menu.sh +# + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +prepare_release() { + local version + read -rp "Version to release (e.g. 3.14.0): " version + check_version "$version" + "${SCRIPT_DIR}/prepare-github-release.sh" "$version" +} + +while true; do + cat < " choice || exit 0 + case "$choice" in + 1) run_step prepare_release || true ;; + 2) run_step "${SCRIPT_DIR}/dev-cluster/dev-menu.sh" || true ;; + q|Q) exit 0 ;; + *) echo "Unknown choice '${choice}'." ;; + esac +done diff --git a/scripts/ops/prepare-github-release.sh b/scripts/ops/prepare-github-release.sh index ecce08e67..e9c78f238 100755 --- a/scripts/ops/prepare-github-release.sh +++ b/scripts/ops/prepare-github-release.sh @@ -21,43 +21,14 @@ set -euo pipefail REPO_ROOT="$(git rev-parse --show-toplevel)" +# shellcheck source=common.sh +source "${REPO_ROOT}/scripts/ops/common.sh" # --- Argument parsing --- -usage() { - echo "Usage: $0 (e.g. 3.6.0)" - exit 1 -} - -if [[ $# -ne 1 ]]; then - echo "Error: Expected exactly one argument, got $#." - usage -fi - +[[ $# -eq 1 ]] || die "Usage: $0 (e.g. 3.6.0)" VERSION="$1" - -if ! [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "Error: '$VERSION' is not valid semver (expected MAJOR.MINOR.PATCH)." - exit 1 -fi - -# --- Helper functions --- - -die() { - printf 'Error: %s\n' "$1" >&2 - exit 1 -} - -require_cmds() { - local missing=0 - for cmd in "$@"; do - command -v "$cmd" >/dev/null 2>&1 || { - printf 'Missing dependency: %s\n' "$cmd" >&2 - missing=1 - } - done - [[ "${missing}" -eq 0 ]] || die "Please install the missing dependencies above (hint: run from within 'nix develop')." -} +check_version "$VERSION" # --- Dependency checks --- @@ -87,16 +58,8 @@ fi # --- Generate changelog --- -# Use --prepend so the new release section is added on top of CHANGELOG.md, -# preserving any manually authored sections (e.g. for releases whose tag does not -# live on main, like 3.9.1 on release/v3.9.1). When new sections need to be -# hand-written, append the relevant duplicate cherry-pick commits to .cliffignore -# so they don't reappear in the next auto-generated release block. -# -# git-cliff fetches PR/author metadata for the ref at the range head. The literal -# `HEAD` resolves to the default branch (main), missing PRs merged only into a -# release branch, so we pass an explicit range ending at a concrete SHA. The base -# is the latest semver tag reachable from HEAD; --match skips stray non-semver tags. +# Use --prepend (preserves manual sections) and concrete SHA for metadata so +# branch-only PRs appear with correct links; append cherry-picks to .cliffignore. echo "==> Generating changelog..." BASE_TAG=$(git describe --tags --abbrev=0 --match '[0-9]*.[0-9]*.[0-9]*' HEAD) \ || die "Could not find a previous semver tag reachable from HEAD." @@ -104,8 +67,7 @@ git-cliff --prepend CHANGELOG.md -t "$VERSION" "${BASE_TAG}..$(git rev-parse HEA # --- Bump workspace version in Cargo.toml --- -# `grep -P` (PCRE) and `sed -i` without a suffix are GNU-only; use POSIX -# forms so this works on both Linux and macOS (BSD userland). +# GNU sed/grep won't work on macOS — use POSIX forms. CARGO_TOML="${REPO_ROOT}/Cargo.toml" OLD_VERSION=$(awk -F'"' '/^version = "[0-9]+\.[0-9]+\.[0-9]+"/ {print $2; exit}' "$CARGO_TOML") @@ -116,8 +78,7 @@ sed -i.bak -E "s/^version = \"[0-9]+\.[0-9]+\.[0-9]+\"/version = \"${VERSION}\"/ # --- Verify contract ABI has changed --- -# The version bump should cause the ABI snapshot to differ. We expect -# the test to fail — if it passes, the ABI was not affected. +# Test fails if ABI was not affected by the version bump (the expected case). echo "==> Verifying contract ABI changed after version bump..." if cargo nextest run --cargo-profile=test-release -p mpc-contract abi_has_not_changed 2>/dev/null; then die "abi_has_not_changed test passed unexpectedly — ABI was not affected by version bump."