Add pprof to ledger service

admin/README.md

@@ -91,6 +91,23 @@ curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"
 curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "set-config", "data": {"profiler-trigger": "1m"}}'
 ```
 
+### Ledger service profiler
+The standalone ledger service (`cmd/ledger`) can expose the same profiler configuration through its admin server. The admin server is disabled by default because `--admin-addr` defaults to an empty value; start the service with `--admin-addr=127.0.0.1:9003` to enable it. The `net/http/pprof` endpoints are registered on the admin server for on-demand profiling and are restricted to loopback addresses.
+
+```bash
+# Start the ledger service with the admin server enabled
+./ledger --triedir=/path/to/trie --admin-addr=127.0.0.1:9003
+
+# Enable the auto-profiler
+curl localhost:9003/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "set-config", "data": {"profiler-enabled": true}}'
+
+# Trigger a profile run
+curl localhost:9003/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "set-config", "data": {"profiler-trigger": "1m"}}'
+
+# Get a heap profile via pprof (only accessible from loopback)
+curl -o heap.prof localhost:9003/debug/pprof/heap
+```
+
 ### Set a stop height
 ```
 curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "stop-at-height", "data": { "height": 1111, "crash": false }}'

cmd/ledger/README.md

@@ -59,6 +59,10 @@ go build -o flow-ledger-service ./cmd/ledger
 - `-max-request-size`: Maximum request message size in bytes (default: 1 GiB)
 - `-max-response-size`: Maximum response message size in bytes (default: 1 GiB)
 - `-loglevel`: Log level (panic, fatal, error, warn, info, debug) (default: info)
+- `-profiler-enabled`: Whether to enable the auto-profiler (default: false)
+- `-profiler-dir`: Directory to create auto-profiler profiles (default: `profiler`)
+- `-profiler-interval`: Interval between auto-profiler runs (default: 15m)
+- `-profiler-duration`: Duration of each auto-profiler run (default: 10s)
 
 ## Admin Commands
 
@@ -69,6 +73,8 @@ When `-admin-addr` is provided, the service exposes an HTTP admin API for managi
 - `trigger-checkpoint`: Triggers a checkpoint to be created as soon as the current WAL segment file is finished writing. This is useful for manually creating checkpoints without waiting for the automatic checkpoint distance.
 - `ping`: Simple health check command to verify the admin server is responsive.
 - `list-commands`: Lists all available admin commands.
+- `get-config`: Returns the current value of a registered runtime config.
+- `set-config`: Updates the value of a registered runtime config.
 
 **Examples:**
 ```bash
@@ -86,6 +92,24 @@ curl -X POST http://localhost:9003/admin/run_command \
 curl -X POST http://localhost:9003/admin/run_command \
   -H "Content-Type: application/json" \
   -d '{"commandName": "list-commands", "data": {}}'
+
+# Get the current auto-profiler enabled state
+curl -X POST http://localhost:9003/admin/run_command \
+  -H "Content-Type: application/json" \
+  -d '{"commandName": "get-config", "data": "profiler-enabled"}'
+
+# Enable the auto-profiler
+curl -X POST http://localhost:9003/admin/run_command \
+  -H "Content-Type: application/json" \
+  -d '{"commandName": "set-config", "data": {"profiler-enabled": true}}'
+
+# Trigger a profile run manually
+curl -X POST http://localhost:9003/admin/run_command \
+  -H "Content-Type: application/json" \
+  -d '{"commandName": "set-config", "data": {"profiler-trigger": "1m"}}'
+
+# Get a heap profile via pprof (only accessible from loopback)
+curl -o heap.prof http://localhost:9003/debug/pprof/heap
 ```
 
 **Note:** When running an execution node with a remote ledger service (using `--ledger-service-addr`), the `trigger-checkpoint` command on the execution node is disabled. You must use the ledger service's admin endpoint to trigger checkpoints.

cmd/ledger/admin.go

@@ -3,10 +3,14 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"net"
 	"net/http"
+	"net/http/pprof"
 
 	"github.com/rs/zerolog"
 	"go.uber.org/atomic"
+
+	"github.com/onflow/flow-go/module/updatable_configs"
 )
 
 // adminRequest represents the JSON request body for admin commands.
@@ -29,19 +33,50 @@ type adminResponse struct {
 type adminHandler struct {
 	logger            zerolog.Logger
 	triggerCheckpoint *atomic.Bool
+	configManager     *updatable_configs.Manager
 	commands          []string
 }
 
+// requireLoopback returns an http.HandlerFunc that only serves requests originating
+// from the loopback interface. It is used to prevent sensitive profiling endpoints
+// from being exposed when the admin server is bound to a publicly reachable address.
+func requireLoopback(next http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		host, _, err := net.SplitHostPort(r.RemoteAddr)
+		if err != nil {
+			http.Error(w, "forbidden", http.StatusForbidden)
+			return
+		}
+		ip := net.ParseIP(host)
+		if ip == nil || !ip.IsLoopback() {
+			http.Error(w, "forbidden", http.StatusForbidden)
+			return
+		}
+		next(w, r)
+	}
+}
+
 // newAdminHandler creates a new admin HTTP handler.
-func newAdminHandler(logger zerolog.Logger, triggerCheckpoint *atomic.Bool) http.Handler {
+func newAdminHandler(logger zerolog.Logger, triggerCheckpoint *atomic.Bool, configManager *updatable_configs.Manager) http.Handler {
 	h := &adminHandler{
 		logger:            logger.With().Str("component", "admin").Logger(),
 		triggerCheckpoint: triggerCheckpoint,
-		commands:          []string{"ping", "list-commands", "trigger-checkpoint"},
+		configManager:     configManager,
+		commands:          []string{"ping", "list-commands", "trigger-checkpoint", "get-config", "set-config"},
 	}
 
 	mux := http.NewServeMux()
 	mux.HandleFunc("/admin/run_command", h.handleCommand)
+
+	// Register pprof handlers for profiling (CPU, heap, goroutine, etc.).
+	// These endpoints are restricted to loopback to avoid exposing profiling
+	// data when the admin server is bound to a public address.
+	mux.HandleFunc("/debug/pprof/", requireLoopback(pprof.Index))
+	mux.HandleFunc("/debug/pprof/cmdline", requireLoopback(pprof.Cmdline))
+	mux.HandleFunc("/debug/pprof/profile", requireLoopback(pprof.Profile))
+	mux.HandleFunc("/debug/pprof/symbol", requireLoopback(pprof.Symbol))
+	mux.HandleFunc("/debug/pprof/trace", requireLoopback(pprof.Trace))
+
 	return mux
 }
 
@@ -78,6 +113,54 @@ func (h *adminHandler) handleCommand(w http.ResponseWriter, r *http.Request) {
 			result = "checkpoint already triggered"
 		}
 
+	case "get-config":
+		var configName string
+		if err := json.Unmarshal(req.Data, &configName); err != nil {
+			h.writeError(w, http.StatusBadRequest, fmt.Sprintf("get-config data must be a string config name: %v", err))
+			return
+		}
+		field, ok := h.configManager.GetField(configName)
+		if !ok {
+			h.writeError(w, http.StatusBadRequest, fmt.Sprintf("unknown config field: %s", configName))
+			return
+		}
+		result = field.Get()
+
+	case "set-config":
+		var data map[string]any
+		if err := json.Unmarshal(req.Data, &data); err != nil {
+			h.writeError(w, http.StatusBadRequest, fmt.Sprintf("set-config data must be a JSON object: %v", err))
+			return
+		}
+		if len(data) != 1 {
+			h.writeError(w, http.StatusBadRequest, fmt.Sprintf("set-config data must have exactly one entry, got %d", len(data)))
+			return
+		}
+		var configName string
+		var configValue any
+		for k, v := range data {
+			configName = k
+			configValue = v
+		}
+		field, ok := h.configManager.GetField(configName)
+		if !ok {
+			h.writeError(w, http.StatusBadRequest, fmt.Sprintf("unknown config field: %s", configName))
+			return
+		}
+		oldValue := field.Get()
+		if err := field.Set(configValue); err != nil {
+			status := http.StatusInternalServerError
+			if updatable_configs.IsValidationError(err) {
+				status = http.StatusBadRequest
+			}
+			h.writeError(w, status, fmt.Sprintf("failed to set config %s: %v", configName, err))
+			return
+		}
+		result = map[string]any{
+			"oldValue": oldValue,
+			"newValue": configValue,
+		}
+
 	default:
 		h.writeError(w, http.StatusBadRequest, fmt.Sprintf("unknown command: %s", req.CommandName))
 		return

cmd/ledger/main.go

@@ -9,7 +9,9 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
+	"runtime"
 	"strings"
+	"sync"
 	"syscall"
 	"time"
 
@@ -23,6 +25,8 @@ import (
 	"github.com/onflow/flow-go/ledger/remote"
 	"github.com/onflow/flow-go/module/irrecoverable"
 	"github.com/onflow/flow-go/module/metrics"
+	"github.com/onflow/flow-go/module/profiler"
+	"github.com/onflow/flow-go/module/updatable_configs"
 )
 
 var (
@@ -37,6 +41,11 @@ var (
 	logLevel            = flag.String("loglevel", "info", "Log level (panic, fatal, error, warn, info, debug)")
 	maxRequestSize      = flag.Uint("max-request-size", 1<<30, "Maximum request message size in bytes (default: 1 GiB)")
 	maxResponseSize     = flag.Uint("max-response-size", 1<<30, "Maximum response message size in bytes (default: 1 GiB)")
+
+	profilerEnabled  = flag.Bool("profiler-enabled", false, "Whether to enable the auto-profiler")
+	profilerDir      = flag.String("profiler-dir", "profiler", "Directory to create auto-profiler profiles")
+	profilerInterval = flag.Duration("profiler-interval", 15*time.Minute, "Interval between auto-profiler runs")
+	profilerDuration = flag.Duration("profiler-duration", 10*time.Second, "Duration of each auto-profiler run")
 )
 
 func main() {
@@ -60,18 +69,88 @@ func main() {
 		Str("service", "ledger").
 		Logger()
 
+	if *profilerInterval <= 0 {
+		logger.Fatal().Dur("profiler_interval", *profilerInterval).Msg("profiler-interval must be positive")
+	}
+
 	// Validate that at least one address is provided
 	if *ledgerServiceTCP == "" && *ledgerServiceSocket == "" {
 		logger.Fatal().Msg("at least one of --ledger-service-tcp or --ledger-service-socket must be provided")
 	}
 
+	// Initialize updatable config manager and auto-profiler.
+	// The profiler is configured via admin get-config/set-config commands.
+	configManager := updatable_configs.NewManager()
+	profilerConfig := profiler.ProfilerConfig{
+		Enabled:         *profilerEnabled,
+		UploaderEnabled: false, // ledger service does not support remote profile upload
+		Dir:             *profilerDir,
+		Interval:        *profilerInterval,
+		Duration:        *profilerDuration,
+	}
+
+	autoProfiler, err := profiler.New(logger, &profiler.NoopUploader{}, profilerConfig)
+	if err != nil {
+		logger.Fatal().Err(err).Msg("failed to create auto-profiler")
+	}
+
+	err = configManager.RegisterBoolConfig("profiler-enabled", autoProfiler.Enabled, autoProfiler.SetEnabled)
+	if err != nil {
+		logger.Fatal().Err(err).Msg("failed to register profiler-enabled config")
+	}
+	err = configManager.RegisterDurationConfig(
+		"profiler-trigger",
+		func() time.Duration { return profilerConfig.Duration },
+		func(d time.Duration) error { return autoProfiler.TriggerRun(d) },
+	)
+	if err != nil {
+		logger.Fatal().Err(err).Msg("failed to register profiler-trigger config")
+	}
+	err = configManager.RegisterUintConfig(
+		"profiler-set-mem-profile-rate",
+		func() uint { return uint(runtime.MemProfileRate) },
+		func(r uint) error { runtime.MemProfileRate = int(r); return nil },
+	)
+	if err != nil {
+		logger.Fatal().Err(err).Msg("failed to register profiler-set-mem-profile-rate config")
+	}
+	var currentBlockRateMu sync.Mutex
+	var currentBlockRate uint
+	err = configManager.RegisterUintConfig(
+		"profiler-set-block-profile-rate",
+		func() uint {
+			currentBlockRateMu.Lock()
+			defer currentBlockRateMu.Unlock()
+			return currentBlockRate
+		},
+		func(r uint) error {
+			currentBlockRateMu.Lock()
+			defer currentBlockRateMu.Unlock()
+			runtime.SetBlockProfileRate(int(r))
+			currentBlockRate = r
+			return nil
+		},
+	)
+	if err != nil {
+		logger.Fatal().Err(err).Msg("failed to register profiler-set-block-profile-rate config")
+	}
+	err = configManager.RegisterUintConfig(
+		"profiler-set-mutex-profile-fraction",
+		func() uint { return uint(runtime.SetMutexProfileFraction(-1)) },
+		func(r uint) error { _ = runtime.SetMutexProfileFraction(int(r)); return nil },
+	)
+	if err != nil {
+		logger.Fatal().Err(err).Msg("failed to register profiler-set-mutex-profile-fraction config")
+	}
+
 	logger.Info().
 		Str("triedir", *triedir).
 		Str("ledger_service_tcp", *ledgerServiceTCP).
 		Str("ledger_service_socket", *ledgerServiceSocket).
 		Str("admin_addr", *adminAddr).
 		Uint("metrics_port", *metricsPort).
 		Int("mtrie_cache_size", *mtrieCacheSize).
+		Bool("profiler_enabled", *profilerEnabled).
 		Msg("starting ledger service")
 
 	// Create trigger for manual checkpointing (used by admin command)
@@ -229,10 +308,14 @@ func main() {
 	// This is a lightweight HTTP-only server (no gRPC proxy layer)
 	var adminServer *http.Server
 	if *adminAddr != "" {
-		adminHandler := newAdminHandler(logger, triggerCheckpointOnNextSegmentFinish)
+		adminHandler := newAdminHandler(logger, triggerCheckpointOnNextSegmentFinish, configManager)
 		adminServer = &http.Server{
-			Addr:    *adminAddr,
-			Handler: adminHandler,
+			Addr:              *adminAddr,
+			Handler:           adminHandler,
+			ReadHeaderTimeout: 10 * time.Second,
+			ReadTimeout:       10 * time.Second,
+			IdleTimeout:       60 * time.Second,
+			WriteTimeout:      2 * time.Minute,
 		}
 
 		go func() {
@@ -298,6 +381,10 @@ func main() {
 		logger.Info().Msg("metrics server stopped")
 	}
 
+	logger.Info().Msg("shutting down auto-profiler...")
+	<-autoProfiler.Done()
+	logger.Info().Msg("auto-profiler stopped")
+
 	logger.Info().Msg("waiting for ledger to stop...")
 	<-ledgerStorage.Done()