-
Notifications
You must be signed in to change notification settings - Fork 2
backup: add admin version API scaffolding #1059
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
2c44879
8f78f4b
6ece87b
ab741c2
7aece27
5cb7ae0
259d59b
b9686b1
a0e32f6
fd9ea80
e8c0cb2
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -41,6 +41,7 @@ type KeyVizSampler interface { | |
| type AdminGroup interface { | ||
| Status() raftengine.Status | ||
| Configuration(ctx context.Context) (raftengine.Configuration, error) | ||
| SnapshotEvery() uint64 | ||
| } | ||
|
|
||
| // NodeIdentity is the value form of the protobuf NodeIdentity message used for | ||
|
|
@@ -55,6 +56,53 @@ func (n NodeIdentity) toProto() *pb.NodeIdentity { | |
| return &pb.NodeIdentity{NodeId: n.NodeID, GrpcAddress: n.GRPCAddress} | ||
| } | ||
|
|
||
| // LeaderVersionProbe fetches the Admin service version for a peer address. | ||
| // Implementations must honor ctx for the 500ms async GetRaftGroups probe | ||
| // budget and any auth metadata copied from the inbound Admin request. | ||
| type LeaderVersionProbe func(ctx context.Context, grpcAddress string) (string, error) | ||
|
|
||
| // AdminOption adjusts optional AdminServer behavior without changing existing | ||
| // test construction call sites. | ||
| type AdminOption func(*AdminServer) | ||
|
|
||
| func WithAdminNodeVersion(version string) AdminOption { | ||
| return func(s *AdminServer) { | ||
| s.nodeVersion = version | ||
| } | ||
| } | ||
|
|
||
| func WithAdminLeaderVersionProbe(probe LeaderVersionProbe) AdminOption { | ||
| return func(s *AdminServer) { | ||
| s.leaderVersionProbe = probe | ||
| } | ||
| } | ||
|
|
||
| func WithAdminLeaderVersionProbeTimeout(timeout time.Duration) AdminOption { | ||
| return func(s *AdminServer) { | ||
| if timeout > 0 { | ||
| s.leaderVersionProbeTimeout = timeout | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func WithAdminLeaderVersionCacheTTL(ttl time.Duration) AdminOption { | ||
| return func(s *AdminServer) { | ||
| if ttl > 0 { | ||
| s.leaderVersionCacheTTL = ttl | ||
| } | ||
| } | ||
| } | ||
|
|
||
| type versionCacheEntry struct { | ||
| version string | ||
| fetchedAt time.Time | ||
| } | ||
|
|
||
| const ( | ||
| defaultAdminLeaderVersionProbeTimeout = 500 * time.Millisecond | ||
| defaultAdminLeaderVersionCacheTTL = 10 * time.Second | ||
| ) | ||
|
|
||
| // AdminServer implements the node-side Admin gRPC service described in | ||
| // docs/admin_ui_key_visualizer_design.md §4 (Layer A). Phase 0 only implements | ||
| // GetClusterOverview and GetRaftGroups; remaining RPCs return Unimplemented so | ||
|
|
@@ -78,6 +126,12 @@ type AdminServer struct { | |
| // pairs atomically with concurrent RPC reads. | ||
| sampler KeyVizSampler | ||
|
|
||
| nodeVersion string | ||
| leaderVersionProbe LeaderVersionProbe | ||
| leaderVersionProbeTimeout time.Duration | ||
| leaderVersionCacheTTL time.Duration | ||
| versionCache sync.Map | ||
|
|
||
| pb.UnimplementedAdminServer | ||
| } | ||
|
|
||
|
|
@@ -86,14 +140,22 @@ type AdminServer struct { | |
| // snapshot shipped to the admin binary; callers that already have a membership | ||
| // source may pass nil and let the admin binary's fan-out layer discover peers | ||
| // by other means. | ||
| func NewAdminServer(self NodeIdentity, members []NodeIdentity) *AdminServer { | ||
| func NewAdminServer(self NodeIdentity, members []NodeIdentity, opts ...AdminOption) *AdminServer { | ||
| cloned := append([]NodeIdentity(nil), members...) | ||
| return &AdminServer{ | ||
| self: self, | ||
| members: cloned, | ||
| groups: make(map[uint64]AdminGroup), | ||
| now: time.Now, | ||
| srv := &AdminServer{ | ||
| self: self, | ||
| members: cloned, | ||
| groups: make(map[uint64]AdminGroup), | ||
| now: time.Now, | ||
| leaderVersionProbeTimeout: defaultAdminLeaderVersionProbeTimeout, | ||
| leaderVersionCacheTTL: defaultAdminLeaderVersionCacheTTL, | ||
| } | ||
| for _, opt := range opts { | ||
| if opt != nil { | ||
| opt(srv) | ||
| } | ||
| } | ||
| return srv | ||
| } | ||
|
|
||
| // SetClock overrides the clock used by GetRaftGroups, letting tests inject a | ||
|
|
@@ -380,7 +442,7 @@ func mergeSeedMembers(seeds []NodeIdentity, selfID string, live *liveMembers) { | |
| // GetRaftGroups returns per-group state snapshots. Phase 0 wires commit/applied | ||
| // indices only; per-follower contact and term history land in later phases. | ||
| func (s *AdminServer) GetRaftGroups( | ||
| _ context.Context, | ||
| ctx context.Context, | ||
| _ *pb.GetRaftGroupsRequest, | ||
| ) (*pb.GetRaftGroupsResponse, error) { | ||
| s.groupsMu.RLock() | ||
|
|
@@ -410,11 +472,111 @@ func (s *AdminServer) GetRaftGroups( | |
| CommitIndex: st.CommitIndex, | ||
| AppliedIndex: st.AppliedIndex, | ||
| LastContactUnixMs: lastContactUnixMs, | ||
| LeaderNodeVersion: s.leaderNodeVersion(ctx, st.Leader), | ||
| }) | ||
| } | ||
| return &pb.GetRaftGroupsResponse{Groups: out}, nil | ||
| } | ||
|
|
||
| func (s *AdminServer) GetNodeVersion( | ||
| context.Context, | ||
| *pb.GetNodeVersionRequest, | ||
| ) (*pb.GetNodeVersionResponse, error) { | ||
| return &pb.GetNodeVersionResponse{NodeVersion: s.nodeVersion}, nil | ||
| } | ||
|
|
||
| func (s *AdminServer) leaderNodeVersion(ctx context.Context, leader raftengine.LeaderInfo) string { | ||
| if version, ok := s.localLeaderVersion(leader); ok { | ||
| return version | ||
| } | ||
| key := leaderVersionCacheKey(leader) | ||
| now := time.Now() | ||
| if version, ok := s.cachedLeaderVersion(key, now); ok { | ||
| return version | ||
| } | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Update the signature of func (s *AdminServer) leaderNodeVersion(ctx context.Context, leader raftengine.LeaderInfo, now time.Time) string {
if version, ok := s.localLeaderVersion(leader); ok {
return version
}
key := leaderVersionCacheKey(leader)
if version, ok := s.cachedLeaderVersion(key, now); ok {
return version
} |
||
| if s.leaderVersionProbe == nil || leader.Address == "" { | ||
| return "" | ||
| } | ||
| if version, ok := s.reserveLeaderVersionProbe(key, now); ok { | ||
| return version | ||
| } | ||
| s.probeLeaderVersionAsync(ctx, key, leader.Address) | ||
| return "" | ||
| } | ||
|
|
||
| func (s *AdminServer) localLeaderVersion(leader raftengine.LeaderInfo) (string, bool) { | ||
| if leader.ID == "" && leader.Address == "" { | ||
| return "", true | ||
| } | ||
| if leader.ID == s.self.NodeID || (leader.Address != "" && leader.Address == s.self.GRPCAddress) { | ||
| return s.nodeVersion, true | ||
| } | ||
| return "", false | ||
| } | ||
|
|
||
| func leaderVersionCacheKey(leader raftengine.LeaderInfo) string { | ||
| if leader.ID != "" { | ||
| return leader.ID | ||
|
bootjp marked this conversation as resolved.
|
||
| } | ||
| return leader.Address | ||
| } | ||
|
|
||
| func (s *AdminServer) cachedLeaderVersion(key string, now time.Time) (string, bool) { | ||
| if key == "" || s.leaderVersionCacheTTL <= 0 { | ||
| return "", false | ||
| } | ||
| actual, ok := s.versionCache.Load(key) | ||
| if !ok { | ||
| return "", false | ||
| } | ||
| entry, ok := actual.(versionCacheEntry) | ||
| if !ok { | ||
| s.versionCache.Delete(key) | ||
| return "", false | ||
| } | ||
| if now.Sub(entry.fetchedAt) > s.leaderVersionCacheTTL { | ||
| s.versionCache.Delete(key) | ||
| return "", false | ||
| } | ||
| return entry.version, true | ||
| } | ||
|
|
||
| func (s *AdminServer) reserveLeaderVersionProbe(key string, now time.Time) (string, bool) { | ||
| marker := versionCacheEntry{fetchedAt: now} | ||
| actual, loaded := s.versionCache.LoadOrStore(key, marker) | ||
| if !loaded { | ||
| return "", false | ||
| } | ||
| entry, ok := actual.(versionCacheEntry) | ||
| switch { | ||
| case !ok: | ||
| s.versionCache.Store(key, marker) | ||
| return "", false | ||
| case now.Sub(entry.fetchedAt) <= s.leaderVersionCacheTTL: | ||
| return entry.version, true | ||
| default: | ||
| s.versionCache.Store(key, marker) | ||
| return "", false | ||
| } | ||
| } | ||
|
|
||
| func (s *AdminServer) probeLeaderVersionAsync(ctx context.Context, key, address string) { | ||
| probe := s.leaderVersionProbe | ||
| timeout := s.leaderVersionProbeTimeout | ||
| go func() { | ||
| probeCtx, cancel := context.WithTimeout(context.Background(), timeout) | ||
| defer cancel() | ||
| if md, ok := metadata.FromIncomingContext(ctx); ok { | ||
| probeCtx = metadata.NewOutgoingContext(probeCtx, md.Copy()) | ||
| } | ||
| version, err := probe(probeCtx, address) | ||
| if err != nil { | ||
| version = "" | ||
| } | ||
| s.versionCache.Store(key, versionCacheEntry{version: version, fetchedAt: time.Now()}) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When tests or callers install a custom clock with Useful? React with 👍 / 👎. |
||
| }() | ||
| } | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. In Since the background goroutine runs asynchronously, the parent To ensure safety and avoid race conditions, extract the metadata synchronously from func (s *AdminServer) probeLeaderVersionAsync(ctx context.Context, key, address string) {
probe := s.leaderVersionProbe
timeout := s.leaderVersionProbeTimeout
var md metadata.MD
if incomingMd, ok := metadata.FromIncomingContext(ctx); ok {
md = incomingMd.Copy()
}
go func() {
probeCtx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
if md != nil {
probeCtx = metadata.NewOutgoingContext(probeCtx, md)
}
version, err := probe(probeCtx, address)
if err != nil {
version = ""
}
s.versionCache.Store(key, versionCacheEntry{version: version, fetchedAt: time.Now()})
}()
} |
||
|
|
||
| func (s *AdminServer) snapshotLeaders() []*pb.GroupLeader { | ||
| s.groupsMu.RLock() | ||
| defer s.groupsMu.RUnlock() | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Pass the mockable clock time
now(computed on line 452 usings.now()) intoleaderNodeVersioninstead of callingtime.Now()internally. This ensures consistency with the rest of theGetRaftGroupsresponse (such asLastContactUnixMs) and allows deterministic testing of the cache TTL using the mock clock.