diff --git a/aggregator/cmd/main.go b/aggregator/cmd/main.go index 24c9e35be..092ea3bd8 100644 --- a/aggregator/cmd/main.go +++ b/aggregator/cmd/main.go @@ -148,8 +148,8 @@ func runServer(configPath string, lggr logger.Logger, sugaredLggr logger.Sugared } lggr.Infow("Successfully loaded configuration from environment variables") - var aggMonitoring common.AggregatorMonitoring = &monitoring.NoopAggregatorMonitoring{} - if config.Monitoring.Enabled && config.Monitoring.Type == "beholder" { + var aggMonitoring common.AggregatorMonitoring = monitoring.NewNoopAggregatorMonitoring() + if config.Monitoring.Beholder.Enabled { m, err := monitoring.InitMonitoring(beholder.Config{ InsecureConnection: config.Monitoring.Beholder.InsecureConnection, CACertFile: config.Monitoring.Beholder.CACertFile, diff --git a/aggregator/pkg/server.go b/aggregator/pkg/server.go index 4cf4e98c2..4553e88a7 100644 --- a/aggregator/pkg/server.go +++ b/aggregator/pkg/server.go @@ -314,7 +314,7 @@ func NewServer(l logger.SugaredLogger, config *model.AggregatorConfig, aggMonito l.Infow("Server configuration loaded", "storage_type", config.Storage.StorageType, - "monitoring_enabled", config.Monitoring.Enabled, + "monitoring_beholder_enabled", config.Monitoring.Beholder.Enabled, "rate_limiting_enabled", config.RateLimiting.Enabled, "health_check_enabled", config.HealthCheck.Enabled, "orphan_recovery_enabled", config.OrphanRecovery.Enabled, diff --git a/aggregator/tests/utils.go b/aggregator/tests/utils.go index d9af04097..a8b088ce5 100644 --- a/aggregator/tests/utils.go +++ b/aggregator/tests/utils.go @@ -173,9 +173,7 @@ func CreateServerOnlyWithMessageRulesControl(t *testing.T, options ...ConfigOpti Storage: &model.StorageConfig{ StorageType: model.StorageTypePostgreSQL, // Default to PostgreSQL }, - Monitoring: model.MonitoringConfig{ - Enabled: false, - }, + Monitoring: model.MonitoringConfig{}, APIClients: []*model.ClientConfig{ { ClientID: "test-client", diff --git a/bootstrap/bootstrap.go b/bootstrap/bootstrap.go index e4519e828..ac60b1914 100644 --- a/bootstrap/bootstrap.go +++ b/bootstrap/bootstrap.go @@ -13,6 +13,7 @@ import ( "time" "github.com/BurntSushi/toml" + "github.com/grafana/pyroscope-go" "github.com/jmoiron/sqlx" sdkmetric "go.opentelemetry.io/otel/sdk/metric" "go.uber.org/zap/zapcore" @@ -142,6 +143,8 @@ type Bootstrapper struct { // accCloser is set by startWithAppConfig; JD mode uses runner.accCloser instead. accCloser *AccessorCloserRegistry + // pyroscope is a saved reference to profiler to close it on stop + pyroscope *pyroscope.Profiler logLevel zapcore.Level } @@ -211,15 +214,20 @@ func NewBootstrapper( if b.config != nil && b.config.Monitoring != nil { mon = *b.config.Monitoring } - err := monitoring.SetupBeholder(mon, fac.MetricViews()) + err := monitoring.SetupBeholder(mon.Beholder, fac.MetricViews()) if err != nil { return nil, fmt.Errorf("failed to setup beholder: %w", err) } - lggr, err := logging.InitLogger(b.name, b.logLevel, mon) + lggr, err := logging.InitLogger(b.name, mon.LogLevel, mon) if err != nil { return nil, fmt.Errorf("failed to init logger: %w", err) } b.lggr = lggr + pyroscopeProfiler, err := monitoring.SetupPyroscope(lggr, b.name, mon.Pyroscope) + if err != nil { + return nil, fmt.Errorf("failed to setup pyroscope: %w", err) + } + b.pyroscope = pyroscopeProfiler lggr.Infow("Monitoring initialized", "config", mon) return b, nil @@ -449,6 +457,12 @@ func (b *Bootstrapper) Stop(ctx context.Context) error { return fmt.Errorf("failed to stop info server: %w", err) } } + if b.pyroscope != nil { + err := b.pyroscope.Stop() + if err != nil { + return fmt.Errorf("failed to stop pyroscope: %w", err) + } + } if b.appCfg != nil { var errs []error if err := b.fac.Stop(ctx); err != nil { diff --git a/bootstrap/config_test.go b/bootstrap/config_test.go index 930f66f8d..a9fb3f103 100644 --- a/bootstrap/config_test.go +++ b/bootstrap/config_test.go @@ -15,9 +15,8 @@ const validEd25519PublicKeyHex = "0123456789abcdef0123456789abcdef0123456789abcd // whose Validate() passes, for use in Config validation tests. func validBeholderMonitoring() *monitoring.Config { return &monitoring.Config{ - Enabled: true, - Type: "beholder", Beholder: monitoring.BeholderConfig{ + Enabled: true, MetricReaderInterval: 10, TraceSampleRatio: 1.0, TraceBatchTimeout: 5, @@ -274,7 +273,7 @@ func TestConfig_validate(t *testing.T) { name: "valid with monitoring present but disabled", config: &Config{ JD: validJD, Keystore: validKeystore, DB: validDB, Server: validServer, - Monitoring: &monitoring.Config{Enabled: false}, + Monitoring: &monitoring.Config{}, }, wantErr: false, }, @@ -286,23 +285,13 @@ func TestConfig_validate(t *testing.T) { }, wantErr: false, }, - { - name: "invalid monitoring: enabled without type", - config: &Config{ - JD: validJD, Keystore: validKeystore, DB: validDB, Server: validServer, - Monitoring: &monitoring.Config{Enabled: true}, - }, - wantErr: true, - errContains: []string{"failed to validate 'monitoring' section", "monitoring type is required"}, - }, { name: "invalid monitoring: enabled beholder with non-positive metric interval", config: &Config{ JD: validJD, Keystore: validKeystore, DB: validDB, Server: validServer, Monitoring: &monitoring.Config{ - Enabled: true, - Type: "beholder", Beholder: monitoring.BeholderConfig{ + Enabled: true, MetricReaderInterval: 0, TraceSampleRatio: 0.5, TraceBatchTimeout: 5, @@ -317,7 +306,7 @@ func TestConfig_validate(t *testing.T) { name: "aggregates errors across db and monitoring sections", config: &Config{ JD: validJD, Keystore: validKeystore, DB: DBConfig{URL: ""}, Server: validServer, - Monitoring: &monitoring.Config{Enabled: true}, + Monitoring: &monitoring.Config{LogLevel: "invalid"}, }, wantErr: true, errContains: []string{"failed to validate 'db' section", "failed to validate 'monitoring' section"}, diff --git a/build/devenv/components/committeeccv/component.go b/build/devenv/components/committeeccv/component.go index db17d0685..97687d881 100644 --- a/build/devenv/components/committeeccv/component.go +++ b/build/devenv/components/committeeccv/component.go @@ -12,6 +12,7 @@ import ( "github.com/pelletier/go-toml/v2" chainsel "github.com/smartcontractkit/chain-selectors" + "github.com/smartcontractkit/chainlink-ccv/bootstrap" "github.com/smartcontractkit/chainlink-ccv/build/devenv/cciptestinterfaces" "github.com/smartcontractkit/chainlink-ccv/build/devenv/chainreg" @@ -624,7 +625,6 @@ func buildVerifierJobSpecEffects( DefaultExecutorQualifier: devenvcommon.DefaultExecutorQualifier, NOPs: ccvchangesets.NOPInputsFromTopology(topology), Committee: ccvchangesets.CommitteeInputFromTopologyPerFamily(committee, family), - PyroscopeURL: obs.PyroscopeURL, TargetNOPs: verNOPAliases, DisableFinalityCheckers: disableFinalityCheckersPerFamily[family], // Consolidated topology: one verifier job per NOP writing to every aggregator. diff --git a/build/devenv/components/executor/component.go b/build/devenv/components/executor/component.go index ae44791e0..94cf4a1d8 100644 --- a/build/devenv/components/executor/component.go +++ b/build/devenv/components/executor/component.go @@ -310,7 +310,6 @@ func buildExecutorJobSpecs( NOPs: ccvchangesets.NOPInputsFromTopology(topology), Pool: ccvchangesets.ExecutorPoolInputFromTopology(pool), IndexerAddress: topology.IndexerAddress, - PyroscopeURL: obs.PyroscopeURL, TargetNOPs: ccvshared.ConvertStringToNopAliases(execNOPAliases), }) if err != nil { diff --git a/build/devenv/components/observability/component.go b/build/devenv/components/observability/component.go index 298be3838..835d550b5 100644 --- a/build/devenv/components/observability/component.go +++ b/build/devenv/components/observability/component.go @@ -21,8 +21,7 @@ const Version = 1 // and executor components to consume. The toml tags keep the serialized phased // output stable. type Observability struct { - PyroscopeURL string `toml:"pyroscope_url"` - Monitoring ccvdeployment.MonitoringConfig `toml:"monitoring"` + Monitoring ccvdeployment.MonitoringConfig `toml:"monitoring"` } func init() { @@ -79,7 +78,6 @@ func decode(raw any) (*Observability, error) { return nil, err } return &Observability{ - PyroscopeURL: cfg.PyroscopeURL, - Monitoring: cfg.Monitoring, + Monitoring: cfg.Monitoring, }, nil } diff --git a/build/devenv/components/observability/component_test.go b/build/devenv/components/observability/component_test.go index feeac6b1f..fcce01c5a 100644 --- a/build/devenv/components/observability/component_test.go +++ b/build/devenv/components/observability/component_test.go @@ -9,12 +9,15 @@ import ( func validConfig() map[string]any { return map[string]any{ - "version": int64(1), - "pyroscope_url": "http://host.docker.internal:4040", + "version": int64(1), "monitoring": map[string]any{ - "Enabled": true, - "Type": "beholder", + "LogLevel": "info", + "Pyroscope": map[string]any{ + "Enabled": true, + "URL": "http://host.docker.internal:4040", + }, "Beholder": map[string]any{ + "Enabled": true, "InsecureConnection": true, "OtelExporterHTTPEndpoint": "host.docker.internal:4318", }, @@ -43,7 +46,8 @@ func TestRunPhase1_PublishesObservability(t *testing.T) { obs, ok := out[Key].(*Observability) require.True(t, ok, "output %q should be *Observability", Key) - require.Equal(t, "http://host.docker.internal:4040", obs.PyroscopeURL) - require.True(t, obs.Monitoring.Enabled) + require.True(t, obs.Monitoring.Pyroscope.Enabled) + require.Equal(t, "http://host.docker.internal:4040", obs.Monitoring.Pyroscope.URL) + require.True(t, obs.Monitoring.Beholder.Enabled) require.Equal(t, "host.docker.internal:4318", obs.Monitoring.Beholder.OtelExporterHTTPEndpoint) } diff --git a/build/devenv/components/tokenverifier/component.go b/build/devenv/components/tokenverifier/component.go index 98e3f6efb..64f87a62c 100644 --- a/build/devenv/components/tokenverifier/component.go +++ b/build/devenv/components/tokenverifier/component.go @@ -89,16 +89,10 @@ func (c *component) RunPhase4( continue } - template, tmplErr := tvIn.GenerateTemplateConfig() - if tmplErr != nil { - return nil, nil, fmt.Errorf("tokenverifier: generating template config: %w", tmplErr) - } - cs := ccvchangesets.GenerateTokenVerifierConfig() output, csErr := cs.Apply(localEnv, ccvchangesets.GenerateTokenVerifierConfigInput{ ServiceIdentifier: "TokenVerifier", ChainSelectors: selectors, - PyroscopeURL: template.PyroscopeURL, Lombard: ccvchangesets.LombardConfigInput{ VerifierID: "LombardVerifier", Qualifier: devenvcommon.LombardVerifierResolverQualifier, diff --git a/build/devenv/config.go b/build/devenv/config.go index c047181eb..5ff03771d 100644 --- a/build/devenv/config.go +++ b/build/devenv/config.go @@ -62,8 +62,7 @@ func Load[T any](paths []string) (*T, error) { decoder.DisallowUnknownFields() if err := decoder.Decode(&config); err != nil { - var details *toml.StrictMissingError - if errors.As(err, &details) { + if details, ok := errors.AsType[*toml.StrictMissingError](err); ok { fmt.Println(details.String()) } return nil, fmt.Errorf("failed to decode TOML config, strict mode: %s", err) @@ -188,8 +187,7 @@ func loadLegacyCfg(data []byte) (*Cfg, error) { decoder := toml.NewDecoder(strings.NewReader(string(data))) decoder.DisallowUnknownFields() if err := decoder.Decode(&cfg); err != nil { - var details *toml.StrictMissingError - if errors.As(err, &details) { + if details, ok := errors.AsType[*toml.StrictMissingError](err); ok { fmt.Println(details.String()) } return nil, fmt.Errorf("failed to decode TOML config, strict mode: %s", err) diff --git a/build/devenv/env-cl-16.toml b/build/devenv/env-cl-16.toml index d36c400ba..034bcd97e 100644 --- a/build/devenv/env-cl-16.toml +++ b/build/devenv/env-cl-16.toml @@ -4,20 +4,23 @@ cl_nodes_funding_link = 50 ## Environment configuration define the topology. [environment_topology] indexer_address = ["http://indexer-1:8100"] -pyroscope_url = "http://host.docker.internal:4040" -[environment_topology.monitoring] +[environment_topology.Monitoring] +LogLevel = 'debug' + +[environment_topology.Monitoring.Pyroscope] Enabled = true -Type = "beholder" +URL = "http://host.docker.internal:4040" -[environment_topology.monitoring.Beholder] +[environment_topology.Monitoring.Beholder] +Enabled = true InsecureConnection = true OtelExporterHTTPEndpoint = "host.docker.internal:4318" LogStreamingEnabled = true MetricReaderInterval = 5 TraceSampleRatio = 1.0 TraceBatchTimeout = 10 -[environment_topology.monitoring.Beholder.TelemetryAttributes] +[environment_topology.Monitoring.Beholder.TelemetryAttributes] ccip_env = "devenv" [[environment_topology.nop_topology.nops]] @@ -971,10 +974,14 @@ execution_interval = 15_000_000_000 [indexer.indexer_config] [indexer.indexer_config.Monitoring] -Enabled = true -Type = 'beholder' +LogLevel = 'debug' + +[indexer.indexer_config.Monitoring.Pyroscope] +Enabled = false +URL = '' [indexer.indexer_config.Monitoring.Beholder] +Enabled = true InsecureConnection = true CACertFile = '' OtelExporterGRPCEndpoint = '' diff --git a/build/devenv/env-cl.toml b/build/devenv/env-cl.toml index cf8bb5516..277bad5fd 100644 --- a/build/devenv/env-cl.toml +++ b/build/devenv/env-cl.toml @@ -1,20 +1,23 @@ ## CL mode environment topology for 2-node setup [environment_topology] indexer_address = ["http://indexer-1:8100"] -pyroscope_url = "http://host.docker.internal:4040" -[environment_topology.monitoring] +[environment_topology.Monitoring] +LogLevel = 'debug' + +[environment_topology.Monitoring.Pyroscope] Enabled = true -Type = "beholder" +URL = "http://host.docker.internal:4040" -[environment_topology.monitoring.Beholder] +[environment_topology.Monitoring.Beholder] +Enabled = true InsecureConnection = true OtelExporterHTTPEndpoint = "host.docker.internal:4318" LogStreamingEnabled = true MetricReaderInterval = 5 TraceSampleRatio = 1.0 TraceBatchTimeout = 10 -[environment_topology.monitoring.Beholder.TelemetryAttributes] +[environment_topology.Monitoring.Beholder.TelemetryAttributes] ccip_env = "devenv" [[environment_topology.nop_topology.nops]] diff --git a/build/devenv/env-phased.toml b/build/devenv/env-phased.toml index e54ccc089..378edc857 100644 --- a/build/devenv/env-phased.toml +++ b/build/devenv/env-phased.toml @@ -14,20 +14,22 @@ version = 1 ## the top-level [environment_topology] section in env.toml. [observability] version = 1 -pyroscope_url = "http://host.docker.internal:4040" +[observability.Monitoring] +LogLevel = 'debug' -[observability.monitoring] -Enabled = true -Type = "beholder" +[observability.Monitoring.Pyroscope] +Enabled = false +URL = '' -[observability.monitoring.Beholder] +[observability.Monitoring.Beholder] +Enabled = true InsecureConnection = true OtelExporterHTTPEndpoint = "host.docker.internal:4318" LogStreamingEnabled = true MetricReaderInterval = 5 TraceSampleRatio = 1.0 TraceBatchTimeout = 10 -[observability.monitoring.Beholder.TelemetryAttributes] +[observability.Monitoring.Beholder.TelemetryAttributes] ccip_env = "devenv" [protocol_contracts] @@ -394,13 +396,17 @@ execution_interval = 15_000_000_000 # Optional: full DB connection string (container-to-container). When set, used for storage config and output. [indexer.indexer_config] -LogLevel = 'info' +LogLevel = 'debug' [indexer.indexer_config.Monitoring] -Enabled = true -Type = 'beholder' +LogLevel = 'debug' + +[indexer.indexer_config.Monitoring.Pyroscope] +Enabled = false +URL = '' [indexer.indexer_config.Monitoring.Beholder] +Enabled = true InsecureConnection = true CACertFile = '' OtelExporterGRPCEndpoint = '' diff --git a/build/devenv/env.toml b/build/devenv/env.toml index b5a75d564..eb550a6d7 100644 --- a/build/devenv/env.toml +++ b/build/devenv/env.toml @@ -13,20 +13,23 @@ high_availability = false ## Environment configuration define the topology. Eventually the topology should be the only config that is needed. [environment_topology] indexer_address = ["http://indexer-1:8100"] -pyroscope_url = "http://host.docker.internal:4040" -[environment_topology.monitoring] +[environment_topology.Monitoring] +LogLevel = 'debug' + +[environment_topology.Monitoring.Pyroscope] Enabled = true -Type = "beholder" +URL = "http://host.docker.internal:4040" -[environment_topology.monitoring.Beholder] +[environment_topology.Monitoring.Beholder] +Enabled = true InsecureConnection = true OtelExporterHTTPEndpoint = "host.docker.internal:4318" LogStreamingEnabled = true MetricReaderInterval = 5 TraceSampleRatio = 1.0 TraceBatchTimeout = 10 -[environment_topology.monitoring.Beholder.TelemetryAttributes] +[environment_topology.Monitoring.Beholder.TelemetryAttributes] ccip_env = "devenv" [[environment_topology.nop_topology.nops]] @@ -350,13 +353,17 @@ execution_interval = 15_000_000_000 # Optional: full DB connection string (container-to-container). When set, used for storage config and output. [indexer.indexer_config] -LogLevel = 'info' +LogLevel = 'debug' [indexer.indexer_config.Monitoring] -Enabled = true -Type = 'beholder' +LogLevel = 'debug' + +[indexer.indexer_config.Monitoring.Pyroscope] +Enabled = false +URL = '' [indexer.indexer_config.Monitoring.Beholder] +Enabled = true InsecureConnection = true CACertFile = '' OtelExporterGRPCEndpoint = '' diff --git a/build/devenv/environment_monolith.go b/build/devenv/environment_monolith.go index 75170df12..1f63e0ea6 100644 --- a/build/devenv/environment_monolith.go +++ b/build/devenv/environment_monolith.go @@ -14,6 +14,7 @@ import ( "github.com/ethereum/go-ethereum/common" chainsel "github.com/smartcontractkit/chain-selectors" + "github.com/smartcontractkit/chainlink-ccv/bootstrap" "github.com/smartcontractkit/chainlink-ccv/build/devenv/cciptestinterfaces" "github.com/smartcontractkit/chainlink-ccv/build/devenv/chainreg" @@ -719,32 +720,11 @@ func NewEnvironment() (in *Cfg, err error) { return nil, fmt.Errorf("fake data provider is required for token verifiers to provide attestation API endpoints, but it was not created successfully") } - template, err := tokenVerifierInput.GenerateTemplateConfig() - if err != nil { - return nil, fmt.Errorf("failed to generate template config for token verifier: %w", err) - } - // Use changeset to generate token verifier config from on-chain state cs := ccvchangesets.GenerateTokenVerifierConfig() output, err := cs.Apply(*e, ccvchangesets.GenerateTokenVerifierConfigInput{ ServiceIdentifier: "TokenVerifier", ChainSelectors: selectors, - PyroscopeURL: template.PyroscopeURL, - Monitoring: ccvdeployment.MonitoringConfig{ - Enabled: template.Monitoring.Enabled, - Type: template.Monitoring.Type, - Beholder: ccvdeployment.BeholderConfig{ - InsecureConnection: template.Monitoring.Beholder.InsecureConnection, - CACertFile: template.Monitoring.Beholder.CACertFile, - OtelExporterGRPCEndpoint: template.Monitoring.Beholder.OtelExporterGRPCEndpoint, - OtelExporterHTTPEndpoint: template.Monitoring.Beholder.OtelExporterHTTPEndpoint, - LogStreamingEnabled: template.Monitoring.Beholder.LogStreamingEnabled, - MetricReaderInterval: template.Monitoring.Beholder.MetricReaderInterval, - TraceSampleRatio: template.Monitoring.Beholder.TraceSampleRatio, - TraceBatchTimeout: template.Monitoring.Beholder.TraceBatchTimeout, - TelemetryAttributes: template.Monitoring.Beholder.TelemetryAttributes, - }, - }, Lombard: ccvchangesets.LombardConfigInput{ VerifierID: "LombardVerifier", Qualifier: devenvcommon.LombardVerifierResolverQualifier, diff --git a/build/devenv/fakes/go.mod b/build/devenv/fakes/go.mod index 69461c9ea..31f11b457 100644 --- a/build/devenv/fakes/go.mod +++ b/build/devenv/fakes/go.mod @@ -72,6 +72,8 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.3 // indirect + github.com/grafana/pyroscope-go v1.2.8 // indirect + github.com/grafana/pyroscope-go/godeltaprof v0.1.9 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/holiman/uint256 v1.3.2 // indirect github.com/invopop/jsonschema v0.13.0 // indirect diff --git a/build/devenv/fakes/go.sum b/build/devenv/fakes/go.sum index d7cf3ecf7..277abb4f7 100644 --- a/build/devenv/fakes/go.sum +++ b/build/devenv/fakes/go.sum @@ -166,6 +166,10 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grafana/pyroscope-go v1.2.8 h1:UvCwIhlx9DeV7F6TW/z8q1Mi4PIm3vuUJ2ZlCEvmA4M= +github.com/grafana/pyroscope-go v1.2.8/go.mod h1:SSi59eQ1/zmKoY/BKwa5rSFsJaq+242Bcrr4wPix1g8= +github.com/grafana/pyroscope-go/godeltaprof v0.1.9 h1:c1Us8i6eSmkW+Ez05d3co8kasnuOY813tbMN8i/a3Og= +github.com/grafana/pyroscope-go/godeltaprof v0.1.9/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= github.com/holiman/uint256 v1.3.2 h1:a9EgMPSC1AAaj1SZL5zIQD3WbwTuHrMGOerLjGmM/TA= diff --git a/build/devenv/services/aggregator.template.toml b/build/devenv/services/aggregator.template.toml index 3f5dcdbf4..8c675f9f9 100644 --- a/build/devenv/services/aggregator.template.toml +++ b/build/devenv/services/aggregator.template.toml @@ -1,5 +1,3 @@ -pyroscope_url = "http://pyroscope:4040" - [server] address = ":50051" requestTimeout = "5s" @@ -79,9 +77,14 @@ key_prefix = "ratelimit" [Monitoring] +LogLevel = 'debug' + +[Monitoring.Pyroscope] Enabled = true -Type = "beholder" +URL = "http://pyroscope:4040" + [Monitoring.Beholder] +Enabled = true InsecureConnection = true CACertFile = "" OtelExporterGRPCEndpoint = "" @@ -90,6 +93,7 @@ LogStreamingEnabled = true MetricReaderInterval = 5 TraceSampleRatio = 1.0 TraceBatchTimeout = 10 + [Monitoring.Beholder.TelemetryAttributes] ccip_env = "devenv" diff --git a/build/devenv/services/bootstrap_test.go b/build/devenv/services/bootstrap_test.go index da15445d5..32eee4c86 100644 --- a/build/devenv/services/bootstrap_test.go +++ b/build/devenv/services/bootstrap_test.go @@ -13,7 +13,7 @@ import ( // Bootstrap.Monitoring before launch, after which ApplyBootstrapDefaults runs again internally; if a // future default clobbered Monitoring, those values would silently disappear from the bootstrap config. func TestApplyBootstrapDefaults_PreservesMonitoring(t *testing.T) { - mon := &monitoring.Config{Enabled: true, Type: "beholder"} + mon := &monitoring.Config{} out := ApplyBootstrapDefaults(BootstrapInput{Monitoring: mon}) diff --git a/build/devenv/services/tokenVerifier.go b/build/devenv/services/tokenVerifier.go index e6c2680c5..da9c991b1 100644 --- a/build/devenv/services/tokenVerifier.go +++ b/build/devenv/services/tokenVerifier.go @@ -35,9 +35,6 @@ const ( DefaultTokenVerifierDBPort = 8450 ) -//go:embed tokenVerifier.template.toml -var tokenVerifierConfigTemplate string - type TokenVerifierDBInput struct { Image string `toml:"image"` Name string `toml:"name"` @@ -299,11 +296,3 @@ func (v *TokenVerifierInput) GenerateConfigWithBlockchainInfos(blockchainInfos c } return cfg, nil } - -func (v *TokenVerifierInput) GenerateTemplateConfig() (*token.Config, error) { - var config *token.Config - if _, err := toml.Decode(tokenVerifierConfigTemplate, &config); err != nil { - return nil, fmt.Errorf("failed to decode verifier config template: %w", err) - } - return config, nil -} diff --git a/build/devenv/services/tokenVerifier.template.toml b/build/devenv/services/tokenVerifier.template.toml deleted file mode 100644 index e916b6fd0..000000000 --- a/build/devenv/services/tokenVerifier.template.toml +++ /dev/null @@ -1 +0,0 @@ -pyroscope_url = "http://pyroscope:4040" diff --git a/cmd/executor/common.go b/cmd/executor/common.go index deb7058d4..77d7910f8 100644 --- a/cmd/executor/common.go +++ b/cmd/executor/common.go @@ -15,7 +15,7 @@ import ( // SetupMonitoring configures executor monitoring via Beholder or returns a noop implementation. func SetupMonitoring(config executor.MonitoringConfig) executor.Monitoring { - if !config.Enabled || config.Type != "beholder" { + if !config.Beholder.Enabled { return monitoring.NewNoopExecutorMonitoring() } diff --git a/cmd/executor/service_test.go b/cmd/executor/service_test.go index a53cd9830..24a0e61ca 100644 --- a/cmd/executor/service_test.go +++ b/cmd/executor/service_test.go @@ -252,11 +252,6 @@ func TestStartPyroscope_EmptyAddress(t *testing.T) { } func TestSetupMonitoring_Disabled(t *testing.T) { - m := SetupMonitoring(executorsvc.MonitoringConfig{Enabled: false}) - require.NotNil(t, m) -} - -func TestSetupMonitoring_EnabledButNotBeholder(t *testing.T) { - m := SetupMonitoring(executorsvc.MonitoringConfig{Enabled: true, Type: "noop"}) + m := SetupMonitoring(executorsvc.MonitoringConfig{}) require.NotNil(t, m) } diff --git a/cmd/verifier/common.go b/cmd/verifier/common.go index c08001e6c..8e1806ad5 100644 --- a/cmd/verifier/common.go +++ b/cmd/verifier/common.go @@ -41,7 +41,7 @@ const ( func SetupMonitoring(config verifier.MonitoringConfig, verifierServiceName string) verifier.Monitoring { // If monitoring is not enabled, return a fake monitoring implementation that does nothing. - if !config.Enabled { + if !config.Beholder.Enabled { verifierMonitoring := monitoring.NewFakeVerifierMonitoring() return verifierMonitoring } diff --git a/common/monitoring/config.go b/common/monitoring/config.go index 7367de731..54d5cace4 100644 --- a/common/monitoring/config.go +++ b/common/monitoring/config.go @@ -1,19 +1,32 @@ package monitoring -import "fmt" +import ( + "fmt" + + "go.uber.org/zap/zapcore" +) // Config provides monitoring configuration for CCV services. type Config struct { - // Enabled enables the monitoring system. - Enabled bool `json:"enabled" toml:"Enabled"` - // Type is the type of monitoring system to use (beholder, noop). - Type string `json:"type" toml:"Type"` - // Beholder is the configuration for the beholder client (Not required if type is noop). + // LogLevel specifies the logging level for service logger + LogLevel string `json:"logLevel" toml:"LogLevel"` + // Pyroscope is the configuration for pyroscope performance monitoring tool + Pyroscope PyroscopeConfig `json:"pyroscope" toml:"Pyroscope"` + // Beholder is the configuration for the beholder client. Beholder BeholderConfig `json:"beholder" toml:"Beholder"` } +type PyroscopeConfig struct { + // Enabled enables the pyroscope telemetry. + Enabled bool `json:"enabled" toml:"Enabled"` + // URL specifies the remote endpoint of pyroscope service + URL string `json:"url" toml:"URL"` +} + // BeholderConfig wraps OpenTelemetry configuration for the beholder client. type BeholderConfig struct { + // Enabled enables the beholder telemetry. + Enabled bool `json:"enabled" toml:"Enabled"` // InsecureConnection disables TLS for the beholder client. InsecureConnection bool `json:"insecure_connection" toml:"InsecureConnection"` // CACertFile is the path to the CA certificate file for the beholder client. @@ -38,11 +51,20 @@ type BeholderConfig struct { // Validate performs validation on the monitoring configuration. func (m *Config) Validate() error { - if m.Enabled && m.Type == "" { - return fmt.Errorf("monitoring type is required when monitoring is enabled") + if m.LogLevel != "" { + _, err := zapcore.ParseLevel(m.LogLevel) + if err != nil { + return fmt.Errorf("monitoring log_level is invalid: %w", err) + } + } + + if m.Pyroscope.Enabled { + if err := m.Pyroscope.Validate(); err != nil { + return fmt.Errorf("pyroscope config validation failed: %w", err) + } } - if m.Enabled && m.Type == "beholder" { + if m.Beholder.Enabled { if err := m.Beholder.Validate(); err != nil { return fmt.Errorf("beholder config validation failed: %w", err) } @@ -51,8 +73,24 @@ func (m *Config) Validate() error { return nil } +// Validate performs validation on the pyroscope configuration. +func (p *PyroscopeConfig) Validate() error { + if len(p.URL) == 0 { + return fmt.Errorf("url is missing") + } + + return nil +} + // Validate performs validation on the beholder configuration. func (b *BeholderConfig) Validate() error { + if b.LogStreamingLevel != "" { + _, err := zapcore.ParseLevel(b.LogStreamingLevel) + if err != nil { + return fmt.Errorf("beholder log_streaming_level is invalid: %w", err) + } + } + if b.MetricReaderInterval <= 0 { return fmt.Errorf("metric_reader_interval must be positive, got %d", b.MetricReaderInterval) } diff --git a/common/monitoring/logging/logging.go b/common/monitoring/logging/logging.go index 0b1b4290b..fc6e4ebc6 100644 --- a/common/monitoring/logging/logging.go +++ b/common/monitoring/logging/logging.go @@ -13,14 +13,21 @@ import ( ) // InitLogger creates a named logger with the base core and optional Beholder log streaming. -func InitLogger(name string, baseLogLevel zapcore.Level, config monitoring.Config) (logger.Logger, error) { +func InitLogger(name, baseLogLevel string, config monitoring.Config) (logger.Logger, error) { loggerCores := make([]zapcore.Core, 0, 2) - baseCore, err := logger.NewCore(zaplog.GetLogProfile(baseLogLevel)) + if baseLogLevel == "" { + baseLogLevel = "info" + } + baseZapLogLevel, err := zapcore.ParseLevel(baseLogLevel) + if err != nil { + return nil, fmt.Errorf("failed to parse log level: %w", err) + } + baseCore, err := logger.NewCore(zaplog.GetLogProfile(baseZapLogLevel)) if err != nil { return nil, fmt.Errorf("failed to initialize base core: %w", err) } loggerCores = append(loggerCores, baseCore) - if config.Enabled && config.Type == "beholder" && config.Beholder.LogStreamingEnabled { + if config.Beholder.Enabled && config.Beholder.LogStreamingEnabled { if config.Beholder.LogStreamingLevel == "" { config.Beholder.LogStreamingLevel = "info" } diff --git a/common/monitoring/setup.go b/common/monitoring/setup.go index 3492c78a5..d0ff6c9d8 100644 --- a/common/monitoring/setup.go +++ b/common/monitoring/setup.go @@ -4,41 +4,43 @@ import ( "fmt" "time" + "github.com/grafana/pyroscope-go" "go.opentelemetry.io/otel/attribute" sdkmetric "go.opentelemetry.io/otel/sdk/metric" "go.uber.org/zap/zapcore" "github.com/smartcontractkit/chainlink-common/pkg/beholder" + "github.com/smartcontractkit/chainlink-common/pkg/logger" ) -func SetupBeholder(config Config, metricViews []sdkmetric.View) error { - if !config.Enabled || config.Type != "beholder" { +func SetupBeholder(config BeholderConfig, metricViews []sdkmetric.View) error { + if !config.Enabled { return nil } var err error logLevel := zapcore.InfoLevel - if config.Beholder.LogStreamingLevel != "" { - logLevel, err = zapcore.ParseLevel(config.Beholder.LogStreamingLevel) + if config.LogStreamingLevel != "" { + logLevel, err = zapcore.ParseLevel(config.LogStreamingLevel) if err != nil { return fmt.Errorf("failed to parse log level: %w", err) } } beholderConfig := beholder.Config{ - InsecureConnection: config.Beholder.InsecureConnection, - CACertFile: config.Beholder.CACertFile, - OtelExporterHTTPEndpoint: config.Beholder.OtelExporterHTTPEndpoint, - OtelExporterGRPCEndpoint: config.Beholder.OtelExporterGRPCEndpoint, - LogStreamingEnabled: config.Beholder.LogStreamingEnabled, + InsecureConnection: config.InsecureConnection, + CACertFile: config.CACertFile, + OtelExporterHTTPEndpoint: config.OtelExporterHTTPEndpoint, + OtelExporterGRPCEndpoint: config.OtelExporterGRPCEndpoint, + LogStreamingEnabled: config.LogStreamingEnabled, LogLevel: logLevel, - MetricReaderInterval: time.Second * time.Duration(config.Beholder.MetricReaderInterval), - TraceSampleRatio: config.Beholder.TraceSampleRatio, - TraceBatchTimeout: time.Second * time.Duration(config.Beholder.TraceBatchTimeout), + MetricReaderInterval: time.Second * time.Duration(config.MetricReaderInterval), + TraceSampleRatio: config.TraceSampleRatio, + TraceBatchTimeout: time.Second * time.Duration(config.TraceBatchTimeout), } - if len(config.Beholder.TelemetryAttributes) > 0 { - attrs := make([]attribute.KeyValue, 0, len(config.Beholder.TelemetryAttributes)) - for k, v := range config.Beholder.TelemetryAttributes { + if len(config.TelemetryAttributes) > 0 { + attrs := make([]attribute.KeyValue, 0, len(config.TelemetryAttributes)) + for k, v := range config.TelemetryAttributes { attrs = append(attrs, attribute.String(k, v)) } beholderConfig.ResourceAttributes = attrs @@ -61,3 +63,23 @@ func SetupBeholder(config Config, metricViews []sdkmetric.View) error { return nil } + +// SetupPyroscope starts the Pyroscope profiler. +func SetupPyroscope(lggr logger.Logger, name string, config PyroscopeConfig) (*pyroscope.Profiler, error) { + if !config.Enabled { + return nil, nil + } + return pyroscope.Start(pyroscope.Config{ + ApplicationName: name, + ServerAddress: config.URL, + Logger: lggr, + ProfileTypes: []pyroscope.ProfileType{ + pyroscope.ProfileCPU, + pyroscope.ProfileAllocObjects, + pyroscope.ProfileAllocSpace, + pyroscope.ProfileGoroutines, + pyroscope.ProfileBlockDuration, + pyroscope.ProfileMutexDuration, + }, + }) +} diff --git a/deployment/changesets/generate_token_verifier_config.go b/deployment/changesets/generate_token_verifier_config.go index 9fa83596d..05ba1b77b 100644 --- a/deployment/changesets/generate_token_verifier_config.go +++ b/deployment/changesets/generate_token_verifier_config.go @@ -66,8 +66,6 @@ type CCTPConfigInput struct { type GenerateTokenVerifierConfigInput struct { ServiceIdentifier string ChainSelectors []uint64 - PyroscopeURL string - Monitoring ccvdeployment.MonitoringConfig Lombard LombardConfigInput CCTP CCTPConfigInput // ReplaceExisting controls how the result is written to env metadata. By @@ -137,13 +135,11 @@ func GenerateTokenVerifierConfig() deployment.ChangeSetV2[GenerateTokenVerifierC } tvConfig := &token.Config{ - PyroscopeURL: cfg.PyroscopeURL, CommitteeConfig: chainaccess.CommitteeConfig{ OnRampAddresses: onRampAddresses, RMNRemoteAddresses: rmnRemoteAddresses, }, TokenVerifiers: []token.VerifierConfig{}, - Monitoring: cfg.Monitoring, } if len(cctpVerifierAddresses) > 0 { diff --git a/executor/config_test.go b/executor/config_test.go index c53fa9179..1e65a4e90 100644 --- a/executor/config_test.go +++ b/executor/config_test.go @@ -229,13 +229,13 @@ func TestConfiguration_Validate(t *testing.T) { wantErrContains: "execution_interval must not be negative", }, { - name: "monitoring_enabled_without_type_fails", + name: "monitoring_enabled_invalid_log_level_fails", config: func() Configuration { c := validConfig() - c.Monitoring = MonitoringConfig{Enabled: true, Type: ""} + c.Monitoring = MonitoringConfig{LogLevel: "invalid"} return c }(), - wantErrContains: "monitoring type is required", + wantErrContains: "log_level is invalid", }, } diff --git a/indexer/cmd/main.go b/indexer/cmd/main.go index 458f64df9..eadaee451 100644 --- a/indexer/cmd/main.go +++ b/indexer/cmd/main.go @@ -64,7 +64,7 @@ func main() { // Setup OTEL Monitoring (via beholder) var indexerMonitoring common.IndexerMonitoring - if config.Monitoring.Enabled { + if config.Monitoring.Beholder.Enabled { indexerMonitoring, err = monitoring.InitMonitoring(beholder.Config{ InsecureConnection: config.Monitoring.Beholder.InsecureConnection, CACertFile: config.Monitoring.Beholder.CACertFile, diff --git a/indexer/cmd/oapigen/go.sum b/indexer/cmd/oapigen/go.sum index e7f91defc..dc8cff6b6 100644 --- a/indexer/cmd/oapigen/go.sum +++ b/indexer/cmd/oapigen/go.sum @@ -126,6 +126,10 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grafana/pyroscope-go v1.2.8 h1:UvCwIhlx9DeV7F6TW/z8q1Mi4PIm3vuUJ2ZlCEvmA4M= +github.com/grafana/pyroscope-go v1.2.8/go.mod h1:SSi59eQ1/zmKoY/BKwa5rSFsJaq+242Bcrr4wPix1g8= +github.com/grafana/pyroscope-go/godeltaprof v0.1.9 h1:c1Us8i6eSmkW+Ez05d3co8kasnuOY813tbMN8i/a3Og= +github.com/grafana/pyroscope-go/godeltaprof v0.1.9/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= github.com/holiman/uint256 v1.3.2 h1:a9EgMPSC1AAaj1SZL5zIQD3WbwTuHrMGOerLjGmM/TA= diff --git a/indexer/config.example.toml b/indexer/config.example.toml index 8ab5fde55..7c92f465b 100644 --- a/indexer/config.example.toml +++ b/indexer/config.example.toml @@ -1,8 +1,12 @@ [Monitoring] +LogLevel = 'debug' + +[Monitoring.Pyroscope] Enabled = true -Type = "beholder" +URL = "http://pyroscope:4040" [Monitoring.Beholder] +Enabled = true InsecureConnection = true CACertFile = "" OtelExporterGRPCEndpoint = "" diff --git a/indexer/pkg/config/config.go b/indexer/pkg/config/config.go index 6462fdcbb..696102496 100644 --- a/indexer/pkg/config/config.go +++ b/indexer/pkg/config/config.go @@ -323,15 +323,8 @@ func (c *Config) Validate() error { return fmt.Errorf("storage config validation failed: %w", err) } - if c.Monitoring.Enabled && c.Monitoring.Type == "" { - return fmt.Errorf("monitoring type is required when monitoring is enabled") - } - - // Validate beholder config if monitoring is enabled and type is beholder - if c.Monitoring.Enabled && c.Monitoring.Type == "beholder" { - if err := c.Monitoring.Beholder.Validate(); err != nil { - return fmt.Errorf("beholder config validation failed: %w", err) - } + if err := c.Monitoring.Validate(); err != nil { + return fmt.Errorf("monitoring config validation failed: %w", err) } return nil