diff --git a/dev/.gitignore b/dev/.gitignore new file mode 100644 index 000000000..6053ae8d1 --- /dev/null +++ b/dev/.gitignore @@ -0,0 +1,3 @@ +/logs/ +/run/ +/data-store/ diff --git a/dev/Makefile b/dev/Makefile new file mode 100644 index 000000000..95c9b888c --- /dev/null +++ b/dev/Makefile @@ -0,0 +1,99 @@ +# dev/Makefile — local launch of seq-db (store) + seq-proxy. +# +# A Makefile equivalent of dev/dev.sh (the original sh script is kept unchanged). +# Processes run as a PLAIN binary (no headless-delve); debugging is done via +# VS Code: "Attach to Go process (pick)" lists store/proxy (dlv attach). +# +# make up build the debug binary and start store + proxy +# make down stop +# make status show process status and PIDs +# make logs tail both logs (or: make logs name=proxy) +# make build build the debug binary only +# make restart down + up +# +# Run from dev/ or via `make -C dev ` from the repo root. +# +# GNU Make 3.81 (macOS) without .ONESHELL passes each recipe line to a separate +# shell, but a backslash+newline joins lines into one. So bash function bodies +# (which need real newlines) live in dev/_helpers.sh and are loaded via `source`; +# everything linear is written straight in the recipe with `; \`. +# +SHELL := /bin/bash + +DEV := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) +ROOT := $(abspath $(DEV)/..) +RUN := $(DEV)/run +LOG := $(DEV)/logs +DATA := $(DEV)/data-store + +OS := $(shell uname -s | tr '[:upper:]' '[:lower:]') +ARCH := $(shell uname -m | tr '[:upper:]' '[:lower:]') +ifeq ($(ARCH), x86_64) +ARCH := amd64 +endif +ifeq ($(ARCH), aarch64) +ARCH := arm64 +endif +BIN := $(ROOT)/bin/$(OS)-$(ARCH)/seq-db + +STORE_CFG := $(DEV)/config.store.yaml +PROXY_CFG := $(DEV)/config.proxy.yaml +STORE_GRPC_PORT := 9104 + +# Export paths into the environment — the functions in _helpers.sh read them. +export DEV ROOT RUN LOG BIN + +name ?= + +.DEFAULT_GOAL := help + +.PHONY: help build up down status logs restart + +help: + @echo "usage: make {up|down|status|logs [name=store|proxy]|build|restart}" + +build: + @source "$$DEV/_helpers.sh"; \ + log "building debug binary -> $(BIN)"; \ + cd "$(ROOT)"; \ + CGO_ENABLED=0 go build -gcflags="all=-N -l" -o "$(BIN)" ./cmd/seq-db + +up: build + @source "$$DEV/_helpers.sh"; \ + cd "$(DEV)"; \ + mkdir -p "$(DATA)" "$(RUN)" "$(LOG)"; \ + start_one store "$(STORE_CFG)" store; \ + wait_port "$(STORE_GRPC_PORT)" store; \ + start_one proxy "$(PROXY_CFG)" proxy; \ + log "ready. Debug via VS Code: \"Attach to Go process (pick)\"."; \ + log " store pid=$$(cat "$(RUN)/store.pid") grpc=:$(STORE_GRPC_PORT) log=$(LOG)/store.log"; \ + log " proxy pid=$$(cat "$(RUN)/proxy.pid") http=:9002 grpc=:9004 log=$(LOG)/proxy.log"; \ + log " clients -> http://localhost:9002 grpc://localhost:9004"; \ + log " stop: make down" + +down: + @source "$$DEV/_helpers.sh"; \ + stop_one proxy; \ + stop_one store; \ + log "stopped." + +status: + @source "$$DEV/_helpers.sh"; \ + for name in store proxy; do \ + if is_running "$$name"; then \ + log "$$name: running (pid $$(cat "$(RUN)/$$name.pid"))"; \ + else \ + log "$$name: stopped"; \ + fi; \ + done + +logs: + @source "$$DEV/_helpers.sh"; \ + if [[ -z "$(name)" ]]; then \ + tail -n 200 "$(LOG)/store.log" "$(LOG)/proxy.log" 2>/dev/null || err "no logs"; \ + else \ + tail -n 200 -f "$(LOG)/$(name).log"; \ + fi + +restart: down up + @: diff --git a/dev/README.md b/dev/README.md new file mode 100644 index 000000000..906818ae6 --- /dev/null +++ b/dev/README.md @@ -0,0 +1,73 @@ +# dev/ — local debugging of seq-proxy + seq-db + +Runs two processes (seq-db `store` + seq-proxy `proxy`) locally for debugging in +VS Code. A single `cmd/seq-db` binary runs in two modes — like +`quickstart/docker-compose.yaml`, but locally. + +Controlled via `make` (targets are defined in `dev/Makefile`). + +Processes start as a **plain binary, without headless-delve**. Debugging is done +through VS Code: the "Attach to Go process (pick)" config shows a list of +Go processes; you pick store or proxy — under the hood it's `dlv attach `. + +## Ports + +| Process | HTTP | gRPC (clients) | gRPC (internal) | debug/pprof | +|---------|------|----------------|-----------------|-------------| +| seq-db store | :9102 | — | **:9104** | :9201 | +| seq-proxy | **:9002** | **:9004** | — | :9200 | + +Clients talk to the proxy: `http://localhost:9002` / `grpc://localhost:9004`. +The proxy talks to the store at `localhost:9104`. + +## Running + +Commands can be run either from the `dev/` directory or from the repository root +via `make -C dev …` (the `-C` flag points make at the directory with the +`Makefile`): + +```bash +make -C dev up # build the debug binary and start store + proxy +make -C dev status # show process status and PIDs +make -C dev logs # tail both logs (or: make -C dev logs name=proxy) +make -C dev down # stop +make -C dev restart # down + up +``` + +Build only — `make -C dev build`. + +Debugging requires `dlv` installed (VS Code attaches through it): +`go install github.com/go-delve/delve/cmd/dlv@latest`. + +## Debugging in VS Code + +1. `make -C dev up` +2. Set breakpoints in the code you need (store: `storeapi/`, `frac/`, `storage/`; proxy: `proxy/`, `proxyapi/`). +3. Run & Debug → **Attach to Go process (pick)**. +4. Pick a process from the list: store and proxy are distinguishable by name — + `seq-db-store` and `seq-db-proxy` (the name is set via `exec -a`; it's one binary). + +To debug both processes at once — open a second VS Code window and attach to the +other process. + +> Since the processes are started without headless-delve, you can attach to them +> by PID without debugger conflicts (on macOS only one debugger can trace a +> process at a time — a headless setup would block local-attach). + +## Files + +- `Makefile` — start/stop/logs/status targets (`up`, `down`, `status`, `logs`, `build`, `restart`) +- `_helpers.sh` — shared bash functions for the Makefile recipes (paths via env) +- `config.store.yaml` / `config.proxy.yaml` — local configs (ports are separated) +- `data-store/` — store data (created automatically, safe to delete) +- `logs/{store,proxy}.log` — process logs +- `run/{store,proxy}.pid` — pid files + +## Notes + +- The binary is built with `-gcflags="all=-N -l"` (no optimizations/inlining) — required for the debugger. +- The mapping is a local copy in `dev/mappings.yaml` (mirrors `quickstart/mappings.yaml`). Config paths are relative to the process CWD, which the `up` recipe sets to `dev/` — so `make -C dev up` works from anywhere. +- The topology matches prod: proxy and store are separate processes. For simplified single-process debugging, the root `Makefile` has `make debug` (mode `single`). +- The `Makefile` targets GNU Make 3.81 (macOS): bash function bodies live in + `_helpers.sh` and are loaded via `source`; the rest is written in recipes with + `; \`-continuations (`.ONESHELL` is unsupported in 3.81). diff --git a/dev/_helpers.sh b/dev/_helpers.sh new file mode 100644 index 000000000..455d8f5c7 --- /dev/null +++ b/dev/_helpers.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# +# dev/_helpers.sh — shared bash helpers for dev/Makefile. +# +# Not run on its own — only sourced from Makefile recipes. Reads paths from +# environment variables exported by the Makefile: +# RUN — pid-file directory (dev/run) +# LOG — process log directory (dev/logs) +# BIN — path to the built debug binary +# +# Processes are started WITHOUT dlv — attach later via `dlv attach` from the IDE. +# exec -a overrides argv[0] (process name): store/proxy are distinguishable in +# `ps` and in the VS Code process list (one binary, different names). +# + +set -euo pipefail + +log() { printf '\033[1;34m[dev]\033[0m %s\n' "$*"; } +err() { printf '\033[1;31m[dev]\033[0m %s\n' "$*" >&2; } + +is_running() { [[ -f "$RUN/$1.pid" ]] && kill -0 "$(cat "$RUN/$1.pid")" 2>/dev/null; } + +start_one() { + local name="$1" cfg="$2" mode="$3" + if is_running "$name"; then + log "$name already running (pid $(cat "$RUN/$name.pid"))" + return 0 + fi + log "starting $name (mode=$mode)" + mkdir -p "$RUN" "$LOG" + exec -a "seq-db-$name" "$BIN" --mode="$mode" --config="$cfg" >"$LOG/$name.log" 2>&1 & + echo $! >"$RUN/$name.pid" +} + +wait_port() { + local port="$1" name="$2" i=0 + while ! nc -z 127.0.0.1 "$port" 2>/dev/null; do + i=$((i+1)) + if [[ $i -ge 60 ]]; then + err "$name did not open port :$port within 60s. See $LOG/$name.log" + tail -n 40 "$LOG/$name.log" >&2 || true + exit 1 + fi + sleep 1 + done +} + +stop_one() { + local name="$1" + if is_running "$name"; then + local pid; pid="$(cat "$RUN/$name.pid")" + log "stopping $name (pid $pid)" + kill "$pid" 2>/dev/null || true + sleep 1 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$RUN/$name.pid" +} diff --git a/dev/config.proxy.yaml b/dev/config.proxy.yaml new file mode 100644 index 000000000..dfd615497 --- /dev/null +++ b/dev/config.proxy.yaml @@ -0,0 +1,19 @@ +# Local debug config for seq-proxy. +# Mode: --mode=proxy. Accepts HTTP/gRPC from clients, talks to store via gRPC. + +address: + http: :9002 # client HTTP API (gRPC-gateway) + grpc: :9004 # client gRPC API (SeqProxyApi) + debug: :9200 # pprof / debug server + +cluster: + hot_stores: + - localhost:9104 + +slow_logs: + bulk_threshold: 1s + search_threshold: 100ms + fetch_threshold: 100ms + +mapping: + path: mappings.yaml diff --git a/dev/config.store.yaml b/dev/config.store.yaml new file mode 100644 index 000000000..af0eb5e93 --- /dev/null +++ b/dev/config.store.yaml @@ -0,0 +1,20 @@ +# Local debug config for seq-db (store). +# Mode: --mode=store. Stores fractions, serves data to proxy via gRPC. + +address: + http: :9102 # not used in store, but occupies the port from config + grpc: :9104 # proxy connects here (cluster.hot_stores) + debug: :9201 # pprof / debug server + +storage: + data_dir: data-store + frac_size: 16MiB + total_size: 1GiB + +slow_logs: + bulk_threshold: 1s + search_threshold: 100ms + fetch_threshold: 100ms + +mapping: + path: mappings.yaml diff --git a/dev/mappings.yaml b/dev/mappings.yaml new file mode 100644 index 000000000..e8b74d7f4 --- /dev/null +++ b/dev/mappings.yaml @@ -0,0 +1,24 @@ +mapping-list: + - type: "keyword" + name: "k8s_pod" + - type: "keyword" + name: "k8s_namespace" + - type: "keyword" + name: "k8s_container" + - type: "text" + name: "request" + - type: "path" + name: "request_uri" + - name: "message" + types: + - type: "text" + - title: "keyword" + type: "keyword" + size: 18 + - type: "object" + name: "someobj" + mapping-list: + - type: "keyword" + name: "nested" + - type: "text" + name: "nestedtext"