Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions dev/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
/logs/
/run/
/data-store/
99 changes: 99 additions & 0 deletions dev/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
# dev/Makefile — local launch of seq-db (store) + seq-proxy.
#
# A Makefile equivalent of dev/dev.sh (the original sh script is kept unchanged).
# Processes run as a PLAIN binary (no headless-delve); debugging is done via
# VS Code: "Attach to Go process (pick)" lists store/proxy (dlv attach).
#
# make up build the debug binary and start store + proxy
# make down stop
# make status show process status and PIDs
# make logs tail both logs (or: make logs name=proxy)
# make build build the debug binary only
# make restart down + up
#
# Run from dev/ or via `make -C dev <target>` from the repo root.
#
# GNU Make 3.81 (macOS) without .ONESHELL passes each recipe line to a separate
# shell, but a backslash+newline joins lines into one. So bash function bodies
# (which need real newlines) live in dev/_helpers.sh and are loaded via `source`;
# everything linear is written straight in the recipe with `; \`.
#
SHELL := /bin/bash

DEV := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
ROOT := $(abspath $(DEV)/..)
RUN := $(DEV)/run
LOG := $(DEV)/logs
DATA := $(DEV)/data-store

OS := $(shell uname -s | tr '[:upper:]' '[:lower:]')
ARCH := $(shell uname -m | tr '[:upper:]' '[:lower:]')
ifeq ($(ARCH), x86_64)
ARCH := amd64
endif
ifeq ($(ARCH), aarch64)
ARCH := arm64
endif
BIN := $(ROOT)/bin/$(OS)-$(ARCH)/seq-db

STORE_CFG := $(DEV)/config.store.yaml
PROXY_CFG := $(DEV)/config.proxy.yaml
STORE_GRPC_PORT := 9104

# Export paths into the environment — the functions in _helpers.sh read them.
export DEV ROOT RUN LOG BIN

name ?=

.DEFAULT_GOAL := help

.PHONY: help build up down status logs restart

help:
@echo "usage: make {up|down|status|logs [name=store|proxy]|build|restart}"

build:
@source "$$DEV/_helpers.sh"; \
log "building debug binary -> $(BIN)"; \
cd "$(ROOT)"; \
CGO_ENABLED=0 go build -gcflags="all=-N -l" -o "$(BIN)" ./cmd/seq-db

up: build
@source "$$DEV/_helpers.sh"; \
cd "$(DEV)"; \
mkdir -p "$(DATA)" "$(RUN)" "$(LOG)"; \
start_one store "$(STORE_CFG)" store; \
wait_port "$(STORE_GRPC_PORT)" store; \
start_one proxy "$(PROXY_CFG)" proxy; \
log "ready. Debug via VS Code: \"Attach to Go process (pick)\"."; \
log " store pid=$$(cat "$(RUN)/store.pid") grpc=:$(STORE_GRPC_PORT) log=$(LOG)/store.log"; \
log " proxy pid=$$(cat "$(RUN)/proxy.pid") http=:9002 grpc=:9004 log=$(LOG)/proxy.log"; \
log " clients -> http://localhost:9002 grpc://localhost:9004"; \
log " stop: make down"

down:
@source "$$DEV/_helpers.sh"; \
stop_one proxy; \
stop_one store; \
log "stopped."

status:
@source "$$DEV/_helpers.sh"; \
for name in store proxy; do \
if is_running "$$name"; then \
log "$$name: running (pid $$(cat "$(RUN)/$$name.pid"))"; \
else \
log "$$name: stopped"; \
fi; \
done

logs:
@source "$$DEV/_helpers.sh"; \
if [[ -z "$(name)" ]]; then \
tail -n 200 "$(LOG)/store.log" "$(LOG)/proxy.log" 2>/dev/null || err "no logs"; \
else \
tail -n 200 -f "$(LOG)/$(name).log"; \
fi

restart: down up
@:
73 changes: 73 additions & 0 deletions dev/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
# dev/ — local debugging of seq-proxy + seq-db

Runs two processes (seq-db `store` + seq-proxy `proxy`) locally for debugging in
VS Code. A single `cmd/seq-db` binary runs in two modes — like
`quickstart/docker-compose.yaml`, but locally.

Controlled via `make` (targets are defined in `dev/Makefile`).

Processes start as a **plain binary, without headless-delve**. Debugging is done
through VS Code: the "Attach to Go process (pick)" config shows a list of
Go processes; you pick store or proxy — under the hood it's `dlv attach <PID>`.

## Ports

| Process | HTTP | gRPC (clients) | gRPC (internal) | debug/pprof |
|---------|------|----------------|-----------------|-------------|
| seq-db store | :9102 | — | **:9104** | :9201 |
| seq-proxy | **:9002** | **:9004** | — | :9200 |

Clients talk to the proxy: `http://localhost:9002` / `grpc://localhost:9004`.
The proxy talks to the store at `localhost:9104`.

## Running

Commands can be run either from the `dev/` directory or from the repository root
via `make -C dev …` (the `-C` flag points make at the directory with the
`Makefile`):

```bash
make -C dev up # build the debug binary and start store + proxy
make -C dev status # show process status and PIDs
make -C dev logs # tail both logs (or: make -C dev logs name=proxy)
make -C dev down # stop
make -C dev restart # down + up
```

Build only — `make -C dev build`.

Debugging requires `dlv` installed (VS Code attaches through it):
`go install github.com/go-delve/delve/cmd/dlv@latest`.

## Debugging in VS Code

1. `make -C dev up`
2. Set breakpoints in the code you need (store: `storeapi/`, `frac/`, `storage/`; proxy: `proxy/`, `proxyapi/`).
3. Run & Debug → **Attach to Go process (pick)**.
4. Pick a process from the list: store and proxy are distinguishable by name —
`seq-db-store` and `seq-db-proxy` (the name is set via `exec -a`; it's one binary).

To debug both processes at once — open a second VS Code window and attach to the
other process.

> Since the processes are started without headless-delve, you can attach to them
> by PID without debugger conflicts (on macOS only one debugger can trace a
> process at a time — a headless setup would block local-attach).

## Files

- `Makefile` — start/stop/logs/status targets (`up`, `down`, `status`, `logs`, `build`, `restart`)
- `_helpers.sh` — shared bash functions for the Makefile recipes (paths via env)
- `config.store.yaml` / `config.proxy.yaml` — local configs (ports are separated)
- `data-store/` — store data (created automatically, safe to delete)
- `logs/{store,proxy}.log` — process logs
- `run/{store,proxy}.pid` — pid files

## Notes

- The binary is built with `-gcflags="all=-N -l"` (no optimizations/inlining) — required for the debugger.
- The mapping is a local copy in `dev/mappings.yaml` (mirrors `quickstart/mappings.yaml`). Config paths are relative to the process CWD, which the `up` recipe sets to `dev/` — so `make -C dev up` works from anywhere.
- The topology matches prod: proxy and store are separate processes. For simplified single-process debugging, the root `Makefile` has `make debug` (mode `single`).
- The `Makefile` targets GNU Make 3.81 (macOS): bash function bodies live in
`_helpers.sh` and are loaded via `source`; the rest is written in recipes with
`; \`-continuations (`.ONESHELL` is unsupported in 3.81).
58 changes: 58 additions & 0 deletions dev/_helpers.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
#!/usr/bin/env bash
#
# dev/_helpers.sh — shared bash helpers for dev/Makefile.
#
# Not run on its own — only sourced from Makefile recipes. Reads paths from
# environment variables exported by the Makefile:
# RUN — pid-file directory (dev/run)
# LOG — process log directory (dev/logs)
# BIN — path to the built debug binary
#
# Processes are started WITHOUT dlv — attach later via `dlv attach` from the IDE.
# exec -a overrides argv[0] (process name): store/proxy are distinguishable in
# `ps` and in the VS Code process list (one binary, different names).
#

set -euo pipefail

log() { printf '\033[1;34m[dev]\033[0m %s\n' "$*"; }
err() { printf '\033[1;31m[dev]\033[0m %s\n' "$*" >&2; }

is_running() { [[ -f "$RUN/$1.pid" ]] && kill -0 "$(cat "$RUN/$1.pid")" 2>/dev/null; }

start_one() {
local name="$1" cfg="$2" mode="$3"
if is_running "$name"; then
log "$name already running (pid $(cat "$RUN/$name.pid"))"
return 0
fi
log "starting $name (mode=$mode)"
mkdir -p "$RUN" "$LOG"
exec -a "seq-db-$name" "$BIN" --mode="$mode" --config="$cfg" >"$LOG/$name.log" 2>&1 &
echo $! >"$RUN/$name.pid"
}

wait_port() {
local port="$1" name="$2" i=0
while ! nc -z 127.0.0.1 "$port" 2>/dev/null; do
i=$((i+1))
if [[ $i -ge 60 ]]; then
err "$name did not open port :$port within 60s. See $LOG/$name.log"
tail -n 40 "$LOG/$name.log" >&2 || true
exit 1
fi
sleep 1
done
}

stop_one() {
local name="$1"
if is_running "$name"; then
local pid; pid="$(cat "$RUN/$name.pid")"
log "stopping $name (pid $pid)"
kill "$pid" 2>/dev/null || true
sleep 1
kill -9 "$pid" 2>/dev/null || true
fi
rm -f "$RUN/$name.pid"
}
19 changes: 19 additions & 0 deletions dev/config.proxy.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# Local debug config for seq-proxy.
# Mode: --mode=proxy. Accepts HTTP/gRPC from clients, talks to store via gRPC.

address:
http: :9002 # client HTTP API (gRPC-gateway)
grpc: :9004 # client gRPC API (SeqProxyApi)
debug: :9200 # pprof / debug server

cluster:
hot_stores:
- localhost:9104

slow_logs:
bulk_threshold: 1s
search_threshold: 100ms
fetch_threshold: 100ms

mapping:
path: mappings.yaml
20 changes: 20 additions & 0 deletions dev/config.store.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# Local debug config for seq-db (store).
# Mode: --mode=store. Stores fractions, serves data to proxy via gRPC.

address:
http: :9102 # not used in store, but occupies the port from config
grpc: :9104 # proxy connects here (cluster.hot_stores)
debug: :9201 # pprof / debug server

storage:
data_dir: data-store
frac_size: 16MiB
total_size: 1GiB

slow_logs:
bulk_threshold: 1s
search_threshold: 100ms
fetch_threshold: 100ms

mapping:
path: mappings.yaml
24 changes: 24 additions & 0 deletions dev/mappings.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
mapping-list:
- type: "keyword"
name: "k8s_pod"
- type: "keyword"
name: "k8s_namespace"
- type: "keyword"
name: "k8s_container"
- type: "text"
name: "request"
- type: "path"
name: "request_uri"
- name: "message"
types:
- type: "text"
- title: "keyword"
type: "keyword"
size: 18
- type: "object"
name: "someobj"
mapping-list:
- type: "keyword"
name: "nested"
- type: "text"
name: "nestedtext"
Loading