Skip to content
176 changes: 169 additions & 7 deletions adapter/admin_grpc.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ type KeyVizSampler interface {
type AdminGroup interface {
Status() raftengine.Status
Configuration(ctx context.Context) (raftengine.Configuration, error)
SnapshotEvery() uint64
}

// NodeIdentity is the value form of the protobuf NodeIdentity message used for
Expand All @@ -55,6 +56,53 @@ func (n NodeIdentity) toProto() *pb.NodeIdentity {
return &pb.NodeIdentity{NodeId: n.NodeID, GrpcAddress: n.GRPCAddress}
}

// LeaderVersionProbe fetches the Admin service version for a peer address.
// Implementations must honor ctx for the 500ms async GetRaftGroups probe
// budget and any auth metadata copied from the inbound Admin request.
type LeaderVersionProbe func(ctx context.Context, grpcAddress string) (string, error)

// AdminOption adjusts optional AdminServer behavior without changing existing
// test construction call sites.
type AdminOption func(*AdminServer)

func WithAdminNodeVersion(version string) AdminOption {
return func(s *AdminServer) {
s.nodeVersion = version
}
}

func WithAdminLeaderVersionProbe(probe LeaderVersionProbe) AdminOption {
return func(s *AdminServer) {
s.leaderVersionProbe = probe
}
}

func WithAdminLeaderVersionProbeTimeout(timeout time.Duration) AdminOption {
return func(s *AdminServer) {
if timeout > 0 {
s.leaderVersionProbeTimeout = timeout
}
}
}

func WithAdminLeaderVersionCacheTTL(ttl time.Duration) AdminOption {
return func(s *AdminServer) {
if ttl > 0 {
s.leaderVersionCacheTTL = ttl
}
}
}

type versionCacheEntry struct {
version string
fetchedAt time.Time
}

const (
defaultAdminLeaderVersionProbeTimeout = 500 * time.Millisecond
defaultAdminLeaderVersionCacheTTL = 10 * time.Second
)

// AdminServer implements the node-side Admin gRPC service described in
// docs/admin_ui_key_visualizer_design.md §4 (Layer A). Phase 0 only implements
// GetClusterOverview and GetRaftGroups; remaining RPCs return Unimplemented so
Expand All @@ -78,6 +126,12 @@ type AdminServer struct {
// pairs atomically with concurrent RPC reads.
sampler KeyVizSampler

nodeVersion string
leaderVersionProbe LeaderVersionProbe
leaderVersionProbeTimeout time.Duration
leaderVersionCacheTTL time.Duration
versionCache sync.Map

pb.UnimplementedAdminServer
}

Expand All @@ -86,14 +140,22 @@ type AdminServer struct {
// snapshot shipped to the admin binary; callers that already have a membership
// source may pass nil and let the admin binary's fan-out layer discover peers
// by other means.
func NewAdminServer(self NodeIdentity, members []NodeIdentity) *AdminServer {
func NewAdminServer(self NodeIdentity, members []NodeIdentity, opts ...AdminOption) *AdminServer {
cloned := append([]NodeIdentity(nil), members...)
return &AdminServer{
self: self,
members: cloned,
groups: make(map[uint64]AdminGroup),
now: time.Now,
srv := &AdminServer{
self: self,
members: cloned,
groups: make(map[uint64]AdminGroup),
now: time.Now,
leaderVersionProbeTimeout: defaultAdminLeaderVersionProbeTimeout,
leaderVersionCacheTTL: defaultAdminLeaderVersionCacheTTL,
}
for _, opt := range opts {
if opt != nil {
opt(srv)
}
}
return srv
}

// SetClock overrides the clock used by GetRaftGroups, letting tests inject a
Expand Down Expand Up @@ -380,7 +442,7 @@ func mergeSeedMembers(seeds []NodeIdentity, selfID string, live *liveMembers) {
// GetRaftGroups returns per-group state snapshots. Phase 0 wires commit/applied
// indices only; per-follower contact and term history land in later phases.
func (s *AdminServer) GetRaftGroups(
_ context.Context,
ctx context.Context,
_ *pb.GetRaftGroupsRequest,
) (*pb.GetRaftGroupsResponse, error) {
s.groupsMu.RLock()
Expand Down Expand Up @@ -410,11 +472,111 @@ func (s *AdminServer) GetRaftGroups(
CommitIndex: st.CommitIndex,
AppliedIndex: st.AppliedIndex,
LastContactUnixMs: lastContactUnixMs,
LeaderNodeVersion: s.leaderNodeVersion(ctx, st.Leader),

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

Pass the mockable clock time now (computed on line 452 using s.now()) into leaderNodeVersion instead of calling time.Now() internally. This ensures consistency with the rest of the GetRaftGroups response (such as LastContactUnixMs) and allows deterministic testing of the cache TTL using the mock clock.

Suggested change
LeaderNodeVersion: s.leaderNodeVersion(ctx, st.Leader),
LeaderNodeVersion: s.leaderNodeVersion(ctx, st.Leader, now),

})
}
return &pb.GetRaftGroupsResponse{Groups: out}, nil
}

func (s *AdminServer) GetNodeVersion(
context.Context,
*pb.GetNodeVersionRequest,
) (*pb.GetNodeVersionResponse, error) {
return &pb.GetNodeVersionResponse{NodeVersion: s.nodeVersion}, nil
}

func (s *AdminServer) leaderNodeVersion(ctx context.Context, leader raftengine.LeaderInfo) string {
if version, ok := s.localLeaderVersion(leader); ok {
return version
}
key := leaderVersionCacheKey(leader)
now := time.Now()
if version, ok := s.cachedLeaderVersion(key, now); ok {
return version
}

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

Update the signature of leaderNodeVersion to accept now time.Time from the caller. This avoids redundant time.Now() syscalls in the loop and allows deterministic testing of the cache TTL.

func (s *AdminServer) leaderNodeVersion(ctx context.Context, leader raftengine.LeaderInfo, now time.Time) string {
	if version, ok := s.localLeaderVersion(leader); ok {
		return version
	}
	key := leaderVersionCacheKey(leader)
	if version, ok := s.cachedLeaderVersion(key, now); ok {
		return version
	}

if s.leaderVersionProbe == nil || leader.Address == "" {
return ""
}
if version, ok := s.reserveLeaderVersionProbe(key, now); ok {
return version
}
s.probeLeaderVersionAsync(ctx, key, leader.Address)
return ""
}

func (s *AdminServer) localLeaderVersion(leader raftengine.LeaderInfo) (string, bool) {
if leader.ID == "" && leader.Address == "" {
return "", true
}
if leader.ID == s.self.NodeID || (leader.Address != "" && leader.Address == s.self.GRPCAddress) {
return s.nodeVersion, true
}
return "", false
}

func leaderVersionCacheKey(leader raftengine.LeaderInfo) string {
if leader.ID != "" {
return leader.ID
Comment thread
bootjp marked this conversation as resolved.
}
return leader.Address
}

func (s *AdminServer) cachedLeaderVersion(key string, now time.Time) (string, bool) {
if key == "" || s.leaderVersionCacheTTL <= 0 {
return "", false
}
actual, ok := s.versionCache.Load(key)
if !ok {
return "", false
}
entry, ok := actual.(versionCacheEntry)
if !ok {
s.versionCache.Delete(key)
return "", false
}
if now.Sub(entry.fetchedAt) > s.leaderVersionCacheTTL {
s.versionCache.Delete(key)
return "", false
}
return entry.version, true
}

func (s *AdminServer) reserveLeaderVersionProbe(key string, now time.Time) (string, bool) {
marker := versionCacheEntry{fetchedAt: now}
actual, loaded := s.versionCache.LoadOrStore(key, marker)
if !loaded {
return "", false
}
entry, ok := actual.(versionCacheEntry)
switch {
case !ok:
s.versionCache.Store(key, marker)
return "", false
case now.Sub(entry.fetchedAt) <= s.leaderVersionCacheTTL:
return entry.version, true
default:
s.versionCache.Store(key, marker)
return "", false
}
}

func (s *AdminServer) probeLeaderVersionAsync(ctx context.Context, key, address string) {
probe := s.leaderVersionProbe
timeout := s.leaderVersionProbeTimeout
go func() {
probeCtx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
if md, ok := metadata.FromIncomingContext(ctx); ok {
probeCtx = metadata.NewOutgoingContext(probeCtx, md.Copy())
}
version, err := probe(probeCtx, address)
if err != nil {
version = ""
}
s.versionCache.Store(key, versionCacheEntry{version: version, fetchedAt: time.Now()})

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P3 Badge Use AdminServer clock for version-cache stamps

When tests or callers install a custom clock with SetClock, the cache freshness checks compare entries against that injected clock, but async probe completions are stamped with wall time here. If the injected clock is behind real time, now.Sub(entry.fetchedAt) stays negative and cached empty/error versions do not expire on the configured TTL; if it is ahead, a just-fetched version can expire immediately. Stamp cache entries with the same clock used by GetRaftGroups so TTL behavior remains consistent.

Useful? React with 👍 / 👎.

}()
}

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

In probeLeaderVersionAsync, the incoming gRPC request context ctx is passed directly into a background goroutine, and metadata.FromIncomingContext(ctx) is called asynchronously inside that goroutine.

Since the background goroutine runs asynchronously, the parent GetRaftGroups request may have already completed and returned. Once the request returns, the gRPC server cancels and cleans up the request context. Accessing ctx asynchronously after the handler has returned can lead to race conditions, undefined behavior, or accessing recycled metadata.

To ensure safety and avoid race conditions, extract the metadata synchronously from ctx before spawning the goroutine.

func (s *AdminServer) probeLeaderVersionAsync(ctx context.Context, key, address string) {
	probe := s.leaderVersionProbe
	timeout := s.leaderVersionProbeTimeout
	var md metadata.MD
	if incomingMd, ok := metadata.FromIncomingContext(ctx); ok {
		md = incomingMd.Copy()
	}
	go func() {
		probeCtx, cancel := context.WithTimeout(context.Background(), timeout)
		defer cancel()
		if md != nil {
			probeCtx = metadata.NewOutgoingContext(probeCtx, md)
		}
		version, err := probe(probeCtx, address)
		if err != nil {
			version = ""
		}
		s.versionCache.Store(key, versionCacheEntry{version: version, fetchedAt: time.Now()})
	}()
}


func (s *AdminServer) snapshotLeaders() []*pb.GroupLeader {
s.groupsMu.RLock()
defer s.groupsMu.RUnlock()
Expand Down
Loading
Loading