Skip to content

DAOS-18825 pool: include excluded ranks as DOWNOUT at creation time #18659

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wangshilong wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

DAOS-18825 pool: include excluded ranks as DOWNOUT at creation time #18659

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions src/common/pool_map.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1670,12 +1670,14 @@ add_domain_tree_to_pool_buf(struct pool_map *map, struct pool_buf *map_buf, int
*/
int
gen_pool_buf(struct pool_map *map, struct pool_buf **map_buf_out, int map_version, int ndomains,
int nnodes, int ntargets, const uint32_t *domains, uint32_t dss_tgt_nr)
int nnodes, int ntargets, const uint32_t *domains, uint32_t dss_tgt_nr,
d_rank_list_t *downout_ranks)
{
struct pool_component map_comp;
struct pool_buf *map_buf;
uint32_t num_comps;
uint8_t new_status;
uint8_t target_status;
int i, rc;
uint32_t num_domain_comps = 0;
d_rank_list_t *ordered_ranks;
Expand Down Expand Up @@ -1720,9 +1722,20 @@ gen_pool_buf(struct pool_map *map, struct pool_buf **map_buf_out, int map_versio
for (i = 0; i < ordered_ranks->rl_nr; i++) {
int j;

/*
* Asymmetric pool create: for a fresh pool (map == NULL), if the rank
* is already excluded / admin-excluded in system membership, all its
* targets are inserted as DOWNOUT so that a subsequent reint can bring
* them back naturally without extending the pool map.
*/
target_status = new_status;
if (map == NULL && downout_ranks != NULL &&
d_rank_in_rank_list(downout_ranks, ordered_ranks->rl_ranks[i]))
target_status = PO_COMP_ST_DOWNOUT;

for (j = 0; j < dss_tgt_nr; j++) {
map_comp.co_type = PO_COMP_TP_TARGET;
map_comp.co_status = new_status;
map_comp.co_status = target_status;
map_comp.co_index = j;
map_comp.co_padding = 0;
map_comp.co_id = (i * dss_tgt_nr + j) + num_comps;
Expand Down
2,013 changes: 689 additions & 1,324 deletions src/control/common/proto/mgmt/pool.pb.go
View file
Open in desktop

Large diffs are not rendered by default.

59 changes: 39 additions & 20 deletions src/control/lib/control/pool.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -223,18 +223,19 @@ type (
// PoolCreateReq contains the parameters for a pool create request.
PoolCreateReq struct {
poolRequest
UUID uuid.UUID `json:"uuid,omitempty"` // Optional UUID; auto-generate if not supplied
User string `json:"user"`
UserGroup string `json:"user_group"`
ACL *AccessControlList `json:"-"`
NumSvcReps uint32 `json:"num_svc_reps"`
Properties []*daos.PoolProperty `json:"-"`
TotalBytes uint64 `json:"total_bytes"` // Auto-sizing param
TierRatio []float64 `json:"tier_ratio"` // Auto-sizing param
NumRanks uint32 `json:"num_ranks"` // Auto-sizing param
Ranks []ranklist.Rank `json:"ranks"` // Manual-sizing param
TierBytes []uint64 `json:"tier_bytes"` // Per-rank values
MemRatio float32 `json:"mem_ratio"` // mem_file_size:meta_blob_size
UUID uuid.UUID `json:"uuid,omitempty"` // Optional UUID; auto-generate if not supplied
User string `json:"user"`
UserGroup string `json:"user_group"`
ACL *AccessControlList `json:"-"`
NumSvcReps uint32 `json:"num_svc_reps"`
Properties []*daos.PoolProperty `json:"-"`
TotalBytes uint64 `json:"total_bytes"` // Auto-sizing param
TierRatio []float64 `json:"tier_ratio"` // Auto-sizing param
NumRanks uint32 `json:"num_ranks"` // Auto-sizing param
Ranks []ranklist.Rank `json:"ranks"` // Manual-sizing param
TierBytes []uint64 `json:"tier_bytes"` // Per-rank values
MemRatio float32 `json:"mem_ratio"` // mem_file_size:meta_blob_size
RanksAutoSelected bool `json:"-"` // Internal: Ranks were selected from joined membership
}

// PoolCreateResp contains the response from a pool create request.
Expand Down Expand Up @@ -1296,28 +1297,46 @@ func getMaxPoolSize(ctx context.Context, rpcClient UnaryInvoker, createReq *Pool
return 0, 0, errors.Wrap(err, "getMaxPoolSize: SystemQuery")
}
joinedRanks := ranklist.RankList{}
excludedRanks := ranklist.RankList{}
for _, member := range queryResp.Members {
if member.State == system.MemberStateJoined {
switch member.State {
case system.MemberStateJoined:
joinedRanks = append(joinedRanks, member.Rank)
case system.MemberStateExcluded, system.MemberStateAdminExcluded:
excludedRanks = append(excludedRanks, member.Rank)
}
}

// Refuse if any requested ranks are not joined, update ranklist to contain only joined ranks.
// Refuse if any requested ranks are not known to the system. Use only joined ranks for the
// storage scan. If no ranks were requested, keep the existing control behavior of filling
// createReq.Ranks with the joined ranks only; the management service may add excluded ranks
// to the pool map based on its authoritative membership view.
filterRanks := ranklist.RankList{}
if len(createReq.Ranks) == 0 {
filterRanks = joinedRanks
createReq.Ranks = append(ranklist.RankList{}, joinedRanks...)
createReq.RanksAutoSelected = true
} else {
createReq.RanksAutoSelected = false
knownRanks := append(ranklist.RankList{}, joinedRanks...)
knownRanks = append(knownRanks, excludedRanks...)

for _, rank := range createReq.Ranks {
if !rank.InList(joinedRanks) {
return 0, 0, errors.Errorf("specified rank %d is not joined", rank)
if !rank.InList(knownRanks) {
return 0, 0, errors.Errorf("specified rank %d is not a known system member", rank)
}
if rank.InList(joinedRanks) {
filterRanks = append(filterRanks, rank)
}
filterRanks = append(filterRanks, rank)
}
}
slices.Sort(filterRanks)
rpcClient.Debugf("requested/joined/filter ranks: %v/%v/%v", createReq.Ranks, joinedRanks,
filterRanks)
createReq.Ranks = filterRanks
if len(createReq.Ranks) > 0 {
createReq.Ranks = ranklist.RankSetFromRanks(createReq.Ranks).Ranks()
slices.Sort(createReq.Ranks)
}
rpcClient.Debugf("requested/joined/excluded/filter ranks: %v/%v/%v/%v", createReq.Ranks,
joinedRanks, excludedRanks, filterRanks)

scanReq := &StorageScanReq{
Usage: true,
Expand Down
28 changes: 25 additions & 3 deletions src/control/lib/control/pool_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3696,7 +3696,7 @@ func TestControl_getMaxPoolSize(t *testing.T) {
memberStates: map[ranklist.Rank]system.MemberState{
0: system.MemberStateStopped,
},
expError: errors.New("specified rank 0 is not joined"),
expError: errors.New("specified rank 0 is not a known system member"),
},
"multiple requested ranks not joined": {
hostsConfigArray: []MockHostStorageConfig{
Expand All @@ -3717,7 +3717,29 @@ func TestControl_getMaxPoolSize(t *testing.T) {
1: system.MemberStateStopped,
2: system.MemberStateExcluded,
},
expError: errors.New("specified rank 1 is not joined"),
expError: errors.New("specified rank 1 is not a known system member"),
},
"requested excluded rank retained but not scanned": {
hostsConfigArray: []MockHostStorageConfig{
{
HostName: "foo",
ScmConfig: []MockScmConfig{newScmCfg(0)},
NvmeConfig: []MockNvmeConfig{newNvmeCfg(0, 0)},
},
{
HostName: "bar",
ScmConfig: []MockScmConfig{newScmCfg(1, 50*humanize.GByte)},
NvmeConfig: []MockNvmeConfig{newNvmeCfg(1, 0, 500*humanize.GByte)},
},
},
tgtRanks: []ranklist.Rank{0, 1},
memberStates: map[ranklist.Rank]system.MemberState{
0: system.MemberStateJoined,
1: system.MemberStateExcluded,
},
expCreateReqRanks: ranklist.RankList{0, 1},
expScmBytes: 100 * humanize.GByte,
expNvmeBytes: humanize.TByte,
},
"all requested ranks joined": {
hostsConfigArray: []MockHostStorageConfig{
Expand All @@ -3741,7 +3763,7 @@ func TestControl_getMaxPoolSize(t *testing.T) {
expScmBytes: 100 * humanize.GByte,
expNvmeBytes: humanize.TByte,
},
"no requested ranks; filters to joined ranks only": {
"no requested ranks; records joined ranks only": {
hostsConfigArray: []MockHostStorageConfig{
{
HostName: "foo",
Expand Down
104 changes: 86 additions & 18 deletions src/control/server/mgmt_pool.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
//
// (C) Copyright 2020-2024 Intel Corporation.
// (C) Copyright 2025 Hewlett Packard Enterprise Development LP
// (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP
//
// SPDX-License-Identifier: BSD-2-Clause-Patent
//
Expand Down Expand Up @@ -198,6 +198,14 @@ func (svc *mgmtSvc) calculateCreateStorage(req *mgmtpb.PoolCreateReq) error {

nvmeMissing := !instances[0].GetStorage().HasBlockDevices()

// Determine the number of ranks that will actually host storage. Ranks
// admitted into the pool as DOWNOUT do not receive VOS/blob-store
// allocation, so they must not be counted when dividing total bytes.
activeRanks := len(req.GetRanks()) - len(req.GetDownoutRanks())
if activeRanks <= 0 {
return errors.New("no active ranks in calculateCreateStorage()")
}

// As this is an exclusive interface between control-API and server, accept only known
// request parameter combinations.

Expand All @@ -223,18 +231,19 @@ func (svc *mgmtSvc) calculateCreateStorage(req *mgmtpb.PoolCreateReq) error {
for tierIdx := range req.TierBytes {
req.TierBytes[tierIdx] =
uint64(float64(req.TotalBytes)*req.TierRatio[tierIdx]) /
uint64(len(req.GetRanks()))
svc.log.Infof("%s = (%s*%f) / %d", humanize.IBytes(req.TierBytes[tierIdx]),
uint64(activeRanks)
svc.log.Infof("%s = (%s*%f) / %d active-ranks",
humanize.IBytes(req.TierBytes[tierIdx]),
humanize.IBytes(req.TotalBytes), req.TierRatio[tierIdx],
len(req.GetRanks()))
activeRanks)
}

default:
return errors.Errorf("unexpected pool create params in request: %+v", req)
}

// Sanity check tier bytes are greater than the minimums.
tgts, ranks := uint64(instances[0].GetTargetCount()), uint64(len(req.GetRanks()))
tgts, ranks := uint64(instances[0].GetTargetCount()), uint64(activeRanks)
if tgts == 0 {
return errors.New("zero target count")
}
Expand Down Expand Up @@ -353,27 +362,58 @@ func (svc *mgmtSvc) poolCreate(parent context.Context, req *mgmtpb.PoolCreateReq
return nil, err
}

// Asymmetric pool create: also enumerate ranks that are currently
// Excluded or AdminExcluded in the system. If any of the pool's target
// ranks fall into those states, they are still admitted into the pool map
// but marked DOWNOUT so that a later reint can bring them back naturally.
excludedRanks, err := svc.sysdb.MemberRanks(system.MemberStateExcluded |
system.MemberStateAdminExcluded)
if err != nil {
return nil, err
}
excludedSet := make(map[uint32]struct{}, len(excludedRanks))
for _, r := range excludedRanks {
excludedSet[r.Uint32()] = struct{}{}
}

if len(req.GetRanks()) > 0 {
// If the request supplies a specific rank list, use it. Note that
// the rank list may include downed ranks, in which case the create
// will fail with an error.
// If the request supplies a rank list, use it. Note that the rank list
// may include downed ranks, in which case the create will fail with an
// error.
reqRanks := ranklist.RanksFromUint32(req.GetRanks())
// Create a RankSet to sort/dedupe the ranks.
reqRanks = ranklist.RankSetFromRanks(reqRanks).Ranks()

if invalid := ranklist.CheckRankMembership(allRanks, reqRanks); len(invalid) > 0 {
// Consider the full set of "known" ranks (available + excluded) as
// valid membership candidates; excluded ranks passed in are OK; they
// will be flagged as DOWNOUT in the pool map.
knownRanks := append(append([]ranklist.Rank{}, allRanks...), excludedRanks...)
if invalid := ranklist.CheckRankMembership(knownRanks, reqRanks); len(invalid) > 0 {
return nil, FaultPoolInvalidRanks(invalid)
}

// The control client auto-sizing path preserves the existing behavior
// of sending joined ranks. Treat that internal marker as the default
// full-cluster create and add excluded ranks so they enter the pool map
// as DOWNOUT. Explicit user-supplied ranks are not expanded.
if req.GetRanksAutoSelected() {
reqRanks = append(reqRanks, excludedRanks...)
reqRanks = ranklist.RankSetFromRanks(reqRanks).Ranks()
}

req.Ranks = ranklist.RanksToUint32(reqRanks)
} else {
// Otherwise, create the pool across the requested number of
// available ranks in the system (if the request does not
// specify a number of ranks, all are used).
//
// Asymmetric pool create: excluded / admin-excluded ranks are also
// admitted into the default rank set so they enter the pool map as
// DOWNOUT. This keeps rank IDs stable across the cluster so that a
// later reint can bring these ranks back naturally.
nAllRanks := len(allRanks)
nRanks := nAllRanks
if req.GetNumRanks() > 0 {
nRanks = int(req.GetNumRanks())
nRanks := int(req.GetNumRanks())

if nRanks > nAllRanks {
return nil, FaultPoolInvalidNumRanks(nRanks, nAllRanks)
Expand All @@ -387,11 +427,21 @@ func (svc *mgmtSvc) poolCreate(parent context.Context, req *mgmtpb.PoolCreateReq
rand.Shuffle(nAllRanks, func(i, j int) {
allRanks[i], allRanks[j] = allRanks[j], allRanks[i]
})
}

req.Ranks = make([]uint32, nRanks)
for i := 0; i < nRanks; i++ {
req.Ranks[i] = allRanks[i].Uint32()
// With an explicit num_ranks, do NOT auto-admit excluded
// ranks: the user asked for a specific size.
req.Ranks = make([]uint32, nRanks)
for i := 0; i < nRanks; i++ {
req.Ranks[i] = allRanks[i].Uint32()
}
} else {
// Full-cluster default: include every joined + excluded rank.
req.Ranks = make([]uint32, 0, nAllRanks+len(excludedRanks))
for _, r := range allRanks {
req.Ranks = append(req.Ranks, r.Uint32())
}
for _, r := range excludedRanks {
req.Ranks = append(req.Ranks, r.Uint32())
}
}
sort.Slice(req.Ranks, func(i, j int) bool { return req.Ranks[i] < req.Ranks[j] })
}
Expand All @@ -400,14 +450,31 @@ func (svc *mgmtSvc) poolCreate(parent context.Context, req *mgmtpb.PoolCreateReq
return nil, errors.New("pool request contains zero target ranks")
}

// Compute DownoutRanks as req.Ranks intersected with excludedSet. Always
// overwrite the field because clients are not trusted to supply the
// correct membership state.
downout := make([]uint32, 0)
for _, r := range req.Ranks {
if _, ok := excludedSet[r]; ok {
downout = append(downout, r)
}
}
req.DownoutRanks = downout
if len(downout) > 0 {
svc.log.Noticef("pool create: %d rank(s) are excluded and will "+
"enter the pool map as DOWNOUT: %v", len(downout), downout)
}

// Clamp the maximum allowed svc replicas to the smaller of requested
// storage ranks or MaxPoolServiceReps.
// storage ranks or MaxPoolServiceReps. DOWNOUT ranks are not eligible
// as PS replicas, so they must not inflate the ceiling.
activeRankCount := len(req.GetRanks()) - len(req.GetDownoutRanks())
maxSvcReps := func(allRanks int) uint32 {
if allRanks > MaxPoolServiceReps {
return uint32(MaxPoolServiceReps)
}
return uint32(allRanks)
}(len(req.GetRanks()))
}(activeRankCount)

// If NumSvcReps is not specified, daos_engine will choose a value.
if req.GetNumSvcReps() > maxSvcReps {
Expand Down Expand Up @@ -468,6 +535,7 @@ func (svc *mgmtSvc) poolCreate(parent context.Context, req *mgmtpb.PoolCreateReq
}
}()

req.RanksAutoSelected = false
dResp, err := svc.harness.CallDrpc(ctx, daos.MethodPoolCreate, req)
if err != nil {
svc.log.Errorf("pool create dRPC call failed: %s", err)
Expand Down
Loading
Loading